From 46d420f30947f77991e59b22a13354d85dda723d Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Thu, 22 Jul 2021 09:06:47 +0800 Subject: [PATCH 01/51] [Refactor] Main code modification for coordinate system refactor (#677) --- configs/_base_/models/fcos3d.py | 1 + .../_base_/models/hv_pointpillars_fpn_nus.py | 7 +- .../models/hv_pointpillars_secfpn_kitti.py | 2 +- .../models/hv_pointpillars_secfpn_waymo.py | 9 +- .../_base_/models/hv_second_secfpn_kitti.py | 2 +- .../_base_/models/hv_second_secfpn_waymo.py | 9 +- configs/_base_/models/parta2.py | 2 +- ...pn_4x8_cyclic_80e_pcdet_kitti-3d-3class.py | 2 +- ...lars_secfpn_3x8_100e_det3d_kitti-3d-car.py | 2 +- ...rs_secfpn_4x8_80e_pcdet_kitti-3d-3class.py | 2 +- ...nd_secfpn_4x8_80e_pcdet_kitti-3d-3class.py | 2 +- configs/free_anchor/README.md | 7 +- ...s_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py | 7 +- configs/imvoxelnet/imvoxelnet_kitti-3d-car.py | 2 +- ...nd_secfpn_adamw_2x8_80e_kitti-3d-3class.py | 2 +- ...rtA2_secfpn_2x8_cyclic_80e_kitti-3d-car.py | 2 +- ...intpillars_secfpn_6x8_160e_kitti-3d-car.py | 2 +- ...ntpillars_secfpn_sbn-all_2x8_2x_lyft-3d.py | 18 +- ...intpillars_secfpn_sbn-all_4x8_2x_nus-3d.py | 14 +- ..._secfpn_sbn-all_range100_2x8_2x_lyft-3d.py | 18 +- ...pillars_secfpn_sbn_2x16_2x_waymo-3d-car.py | 2 +- ...llars_secfpn_sbn_2x16_2x_waymoD5-3d-car.py | 2 +- ...net-400mf_secfpn_sbn-all_2x8_2x_lyft-3d.py | 18 +- ...gnet-400mf_secfpn_sbn-all_4x8_2x_nus-3d.py | 14 +- ..._secfpn_sbn-all_range100_2x8_2x_lyft-3d.py | 18 +- .../hv_second_secfpn_6x8_80e_kitti-3d-car.py | 2 +- ...nd_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py | 5 +- .../hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py | 20 +- .../hv_ssn_secfpn_sbn-all_2x16_2x_nus-3d.py | 22 +- mmdet3d/apis/inference.py | 10 +- mmdet3d/core/anchor/anchor_3d_generator.py | 6 +- mmdet3d/core/bbox/box_np_ops.py | 174 +--- .../bbox/iou_calculators/iou3d_calculator.py | 18 +- .../samplers/iou_neg_piecewise_sampler.py | 8 +- mmdet3d/core/bbox/structures/base_box3d.py | 70 +- mmdet3d/core/bbox/structures/box_3d_mode.py | 55 +- mmdet3d/core/bbox/structures/cam_box3d.py | 104 ++- mmdet3d/core/bbox/structures/coord_3d_mode.py | 159 ++-- mmdet3d/core/bbox/structures/depth_box3d.py | 76 +- mmdet3d/core/bbox/structures/lidar_box3d.py | 69 +- mmdet3d/core/bbox/structures/utils.py | 164 ++-- mmdet3d/core/bbox/transforms.py | 4 +- mmdet3d/core/points/base_points.py | 32 +- mmdet3d/core/utils/__init__.py | 6 +- mmdet3d/core/utils/array_converter.py | 320 ++++++++ mmdet3d/core/visualizer/show_result.py | 6 +- mmdet3d/datasets/kitti_dataset.py | 2 - mmdet3d/datasets/lyft_dataset.py | 8 +- mmdet3d/datasets/nuscenes_dataset.py | 8 +- .../datasets/pipelines/data_augment_utils.py | 20 +- mmdet3d/datasets/pipelines/formating.py | 3 +- mmdet3d/datasets/pipelines/transforms_3d.py | 4 +- mmdet3d/datasets/waymo_dataset.py | 1 - mmdet3d/models/dense_heads/anchor3d_head.py | 6 +- .../dense_heads/anchor_free_mono3d_head.py | 1 + .../models/dense_heads/fcos_mono3d_head.py | 9 +- .../models/dense_heads/free_anchor3d_head.py | 1 + .../models/dense_heads/groupfree3d_head.py | 2 +- mmdet3d/models/dense_heads/parta2_rpn_head.py | 6 +- mmdet3d/models/dense_heads/ssd_3d_head.py | 32 +- mmdet3d/models/dense_heads/train_mixins.py | 4 +- mmdet3d/models/dense_heads/vote_head.py | 4 +- .../roi_heads/bbox_heads/h3d_bbox_head.py | 2 +- .../roi_heads/bbox_heads/parta2_bbox_head.py | 8 +- .../roi_heads/mask_heads/primitive_head.py | 2 +- mmdet3d/ops/iou3d/src/iou3d_kernel.cu | 8 +- .../ops/roiaware_pool3d/points_in_boxes.py | 48 +- .../src/points_in_boxes_cpu.cpp | 14 +- .../src/points_in_boxes_cuda.cu | 26 +- .../src/roiaware_pool3d_kernel.cu | 34 +- tests/data/kitti/kitti_dbinfos_train.pkl | Bin 647 -> 752 bytes tests/data/lyft/lyft_infos.pkl | Bin 7300 -> 7287 bytes tests/data/lyft/lyft_infos_val.pkl | Bin 10022 -> 11926 bytes tests/data/lyft/sample_results.pkl | Bin 1539 -> 1764 bytes tests/data/sunrgbd/sunrgbd_infos.pkl | Bin 2211 -> 2217 bytes .../kitti_format/waymo_dbinfos_train.pkl | Bin 615 -> 616 bytes .../test_datasets/test_kitti_dataset.py | 47 +- .../test_datasets/test_lyft_dataset.py | 35 +- .../test_datasets/test_sunrgbd_dataset.py | 3 + .../test_datasets/test_waymo_dataset.py | 29 +- .../test_augmentations/test_transforms_3d.py | 18 +- .../test_pipelines/test_indoor_pipeline.py | 20 +- .../test_pipelines/test_outdoor_pipeline.py | 114 ++- .../test_roiaware_pool3d.py | 46 +- .../test_heads/test_parta2_bbox_head.py | 31 +- .../test_heads/test_roi_extractors.py | 5 +- .../test_heads/test_semantic_heads.py | 4 +- tests/test_utils/test_anchors.py | 44 +- tests/test_utils/test_box3d.py | 758 ++++++++++++------ tests/test_utils/test_box_np_ops.py | 28 +- tests/test_utils/test_coord_3d_mode.py | 106 ++- tests/test_utils/test_utils.py | 170 +++- tools/create_data.py | 2 +- tools/data_converter/kitti_converter.py | 4 +- tools/data_converter/lyft_converter.py | 6 +- tools/data_converter/nuscenes_converter.py | 8 +- tools/data_converter/sunrgbd_data_utils.py | 17 +- 97 files changed, 2101 insertions(+), 1143 deletions(-) create mode 100644 mmdet3d/core/utils/array_converter.py diff --git a/configs/_base_/models/fcos3d.py b/configs/_base_/models/fcos3d.py index 92ea907605..1465b81a72 100644 --- a/configs/_base_/models/fcos3d.py +++ b/configs/_base_/models/fcos3d.py @@ -29,6 +29,7 @@ pred_attrs=True, pred_velo=True, dir_offset=0.7854, # pi/4 + dir_limit_offset=0, strides=[8, 16, 32, 64, 128], group_reg_dims=(2, 1, 3, 1, 2), # offset, depth, size, rot, velo cls_branch=(256, ), diff --git a/configs/_base_/models/hv_pointpillars_fpn_nus.py b/configs/_base_/models/hv_pointpillars_fpn_nus.py index e153f6c6e6..be29269ded 100644 --- a/configs/_base_/models/hv_pointpillars_fpn_nus.py +++ b/configs/_base_/models/hv_pointpillars_fpn_nus.py @@ -49,8 +49,8 @@ ranges=[[-50, -50, -1.8, 50, 50, -1.8]], scales=[1, 2, 4], sizes=[ - [0.8660, 2.5981, 1.], # 1.5/sqrt(3) - [0.5774, 1.7321, 1.], # 1/sqrt(3) + [2.5981, 0.8660, 1.], # 1.5 / sqrt(3) + [1.7321, 0.5774, 1.], # 1 / sqrt(3) [1., 1., 1.], [0.4, 0.4, 1], ], @@ -59,8 +59,7 @@ reshape_out=True), assigner_per_size=False, diff_rad_by_sin=True, - dir_offset=0.7854, # pi/4 - dir_limit_offset=0, + dir_offset=-0.7854, # -pi / 4 bbox_coder=dict(type='DeltaXYZWLHRBBoxCoder', code_size=9), loss_cls=dict( type='FocalLoss', diff --git a/configs/_base_/models/hv_pointpillars_secfpn_kitti.py b/configs/_base_/models/hv_pointpillars_secfpn_kitti.py index 85076d0798..bb2014729f 100644 --- a/configs/_base_/models/hv_pointpillars_secfpn_kitti.py +++ b/configs/_base_/models/hv_pointpillars_secfpn_kitti.py @@ -41,7 +41,7 @@ [0, -39.68, -0.6, 70.4, 39.68, -0.6], [0, -39.68, -1.78, 70.4, 39.68, -1.78], ], - sizes=[[0.6, 0.8, 1.73], [0.6, 1.76, 1.73], [1.6, 3.9, 1.56]], + sizes=[[0.8, 0.6, 1.73], [1.76, 0.6, 1.73], [3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=False), diff_rad_by_sin=True, diff --git a/configs/_base_/models/hv_pointpillars_secfpn_waymo.py b/configs/_base_/models/hv_pointpillars_secfpn_waymo.py index 14873ead47..30e23e9560 100644 --- a/configs/_base_/models/hv_pointpillars_secfpn_waymo.py +++ b/configs/_base_/models/hv_pointpillars_secfpn_waymo.py @@ -48,15 +48,14 @@ [-74.88, -74.88, -0.1188, 74.88, 74.88, -0.1188], [-74.88, -74.88, 0, 74.88, 74.88, 0]], sizes=[ - [2.08, 4.73, 1.77], # car - [0.84, 1.81, 1.77], # cyclist - [0.84, 0.91, 1.74] # pedestrian + [4.73, 2.08, 1.77], # car + [1.81, 0.84, 1.77], # cyclist + [0.91, 0.84, 1.74] # pedestrian ], rotations=[0, 1.57], reshape_out=False), diff_rad_by_sin=True, - dir_offset=0.7854, # pi/4 - dir_limit_offset=0, + dir_offset=-0.7854, # -pi / 4 bbox_coder=dict(type='DeltaXYZWLHRBBoxCoder', code_size=7), loss_cls=dict( type='FocalLoss', diff --git a/configs/_base_/models/hv_second_secfpn_kitti.py b/configs/_base_/models/hv_second_secfpn_kitti.py index 6bf18abe1d..e7d569a527 100644 --- a/configs/_base_/models/hv_second_secfpn_kitti.py +++ b/configs/_base_/models/hv_second_secfpn_kitti.py @@ -37,7 +37,7 @@ [0, -40.0, -0.6, 70.4, 40.0, -0.6], [0, -40.0, -1.78, 70.4, 40.0, -1.78], ], - sizes=[[0.6, 0.8, 1.73], [0.6, 1.76, 1.73], [1.6, 3.9, 1.56]], + sizes=[[0.8, 0.6, 1.73], [1.76, 0.6, 1.73], [3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=False), diff_rad_by_sin=True, diff --git a/configs/_base_/models/hv_second_secfpn_waymo.py b/configs/_base_/models/hv_second_secfpn_waymo.py index eb9bd3ae5c..0fa39e1505 100644 --- a/configs/_base_/models/hv_second_secfpn_waymo.py +++ b/configs/_base_/models/hv_second_secfpn_waymo.py @@ -42,15 +42,14 @@ [-76.8, -51.2, 0, 76.8, 51.2, 0], [-76.8, -51.2, -0.1188, 76.8, 51.2, -0.1188]], sizes=[ - [2.08, 4.73, 1.77], # car - [0.84, 0.91, 1.74], # pedestrian - [0.84, 1.81, 1.77] # cyclist + [4.73, 2.08, 1.77], # car + [0.91, 0.84, 1.74], # pedestrian + [1.81, 0.84, 1.77] # cyclist ], rotations=[0, 1.57], reshape_out=False), diff_rad_by_sin=True, - dir_offset=0.7854, # pi/4 - dir_limit_offset=0, + dir_offset=-0.7854, # -pi / 4 bbox_coder=dict(type='DeltaXYZWLHRBBoxCoder', code_size=7), loss_cls=dict( type='FocalLoss', diff --git a/configs/_base_/models/parta2.py b/configs/_base_/models/parta2.py index 6c5ae9a663..aa1556789f 100644 --- a/configs/_base_/models/parta2.py +++ b/configs/_base_/models/parta2.py @@ -38,7 +38,7 @@ ranges=[[0, -40.0, -0.6, 70.4, 40.0, -0.6], [0, -40.0, -0.6, 70.4, 40.0, -0.6], [0, -40.0, -1.78, 70.4, 40.0, -1.78]], - sizes=[[0.6, 0.8, 1.73], [0.6, 1.76, 1.73], [1.6, 3.9, 1.56]], + sizes=[[0.8, 0.6, 1.73], [1.76, 0.6, 1.73], [3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=False), diff_rad_by_sin=True, diff --git a/configs/benchmark/hv_PartA2_secfpn_4x8_cyclic_80e_pcdet_kitti-3d-3class.py b/configs/benchmark/hv_PartA2_secfpn_4x8_cyclic_80e_pcdet_kitti-3d-3class.py index 19862097a3..398a19cd2f 100644 --- a/configs/benchmark/hv_PartA2_secfpn_4x8_cyclic_80e_pcdet_kitti-3d-3class.py +++ b/configs/benchmark/hv_PartA2_secfpn_4x8_cyclic_80e_pcdet_kitti-3d-3class.py @@ -38,7 +38,7 @@ ranges=[[0, -40.0, -0.6, 70.4, 40.0, -0.6], [0, -40.0, -0.6, 70.4, 40.0, -0.6], [0, -40.0, -1.78, 70.4, 40.0, -1.78]], - sizes=[[0.6, 0.8, 1.73], [0.6, 1.76, 1.73], [1.6, 3.9, 1.56]], + sizes=[[0.8, 0.6, 1.73], [1.76, 0.6, 1.73], [3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=False), diff_rad_by_sin=True, diff --git a/configs/benchmark/hv_pointpillars_secfpn_3x8_100e_det3d_kitti-3d-car.py b/configs/benchmark/hv_pointpillars_secfpn_3x8_100e_det3d_kitti-3d-car.py index 42a31a2170..72c7372451 100644 --- a/configs/benchmark/hv_pointpillars_secfpn_3x8_100e_det3d_kitti-3d-car.py +++ b/configs/benchmark/hv_pointpillars_secfpn_3x8_100e_det3d_kitti-3d-car.py @@ -37,7 +37,7 @@ anchor_generator=dict( type='Anchor3DRangeGenerator', ranges=[[0, -39.68, -1.78, 69.12, 39.68, -1.78]], - sizes=[[1.6, 3.9, 1.56]], + sizes=[[3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=True), diff_rad_by_sin=True, diff --git a/configs/benchmark/hv_pointpillars_secfpn_4x8_80e_pcdet_kitti-3d-3class.py b/configs/benchmark/hv_pointpillars_secfpn_4x8_80e_pcdet_kitti-3d-3class.py index 76ddd69a06..02eed9fb18 100644 --- a/configs/benchmark/hv_pointpillars_secfpn_4x8_80e_pcdet_kitti-3d-3class.py +++ b/configs/benchmark/hv_pointpillars_secfpn_4x8_80e_pcdet_kitti-3d-3class.py @@ -48,7 +48,7 @@ [0, -40.0, -0.6, 70.4, 40.0, -0.6], [0, -40.0, -1.78, 70.4, 40.0, -1.78], ], - sizes=[[0.6, 0.8, 1.73], [0.6, 1.76, 1.73], [1.6, 3.9, 1.56]], + sizes=[[0.8, 0.6, 1.73], [1.76, 0.6, 1.73], [3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=False), diff_rad_by_sin=True, diff --git a/configs/benchmark/hv_second_secfpn_4x8_80e_pcdet_kitti-3d-3class.py b/configs/benchmark/hv_second_secfpn_4x8_80e_pcdet_kitti-3d-3class.py index 1f2b109bf8..d61a050fb1 100644 --- a/configs/benchmark/hv_second_secfpn_4x8_80e_pcdet_kitti-3d-3class.py +++ b/configs/benchmark/hv_second_secfpn_4x8_80e_pcdet_kitti-3d-3class.py @@ -39,7 +39,7 @@ [0, -40.0, -0.6, 70.4, 40.0, -0.6], [0, -40.0, -1.78, 70.4, 40.0, -1.78], ], - sizes=[[0.6, 0.8, 1.73], [0.6, 1.76, 1.73], [1.6, 3.9, 1.56]], + sizes=[[0.8, 0.6, 1.73], [1.76, 0.6, 1.73], [3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=False), diff_rad_by_sin=True, diff --git a/configs/free_anchor/README.md b/configs/free_anchor/README.md index b38cf3c2f9..e88c024871 100644 --- a/configs/free_anchor/README.md +++ b/configs/free_anchor/README.md @@ -49,8 +49,8 @@ model = dict( ranges=[[-50, -50, -1.8, 50, 50, -1.8]], scales=[1, 2, 4], sizes=[ - [0.8660, 2.5981, 1.], # 1.5/sqrt(3) - [0.5774, 1.7321, 1.], # 1/sqrt(3) + [2.5981, 0.8660, 1.], # 1.5 / sqrt(3) + [1.7321, 0.5774, 1.], # 1 / sqrt(3) [1., 1., 1.], [0.4, 0.4, 1], ], @@ -59,8 +59,7 @@ model = dict( reshape_out=True), assigner_per_size=False, diff_rad_by_sin=True, - dir_offset=0.7854, # pi/4 - dir_limit_offset=0, + dir_offset=-0.7854, # -pi / 4 bbox_coder=dict(type='DeltaXYZWLHRBBoxCoder', code_size=9), loss_cls=dict( type='FocalLoss', diff --git a/configs/free_anchor/hv_pointpillars_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py b/configs/free_anchor/hv_pointpillars_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py index d0a989f121..7412b93085 100644 --- a/configs/free_anchor/hv_pointpillars_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py +++ b/configs/free_anchor/hv_pointpillars_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py @@ -21,8 +21,8 @@ ranges=[[-50, -50, -1.8, 50, 50, -1.8]], scales=[1, 2, 4], sizes=[ - [0.8660, 2.5981, 1.], # 1.5/sqrt(3) - [0.5774, 1.7321, 1.], # 1/sqrt(3) + [2.5981, 0.8660, 1.], # 1.5 / sqrt(3) + [1.7321, 0.5774, 1.], # 1 / sqrt(3) [1., 1., 1.], [0.4, 0.4, 1], ], @@ -31,8 +31,7 @@ reshape_out=True), assigner_per_size=False, diff_rad_by_sin=True, - dir_offset=0.7854, # pi/4 - dir_limit_offset=0, + dir_offset=-0.7854, # -pi / 4 bbox_coder=dict(type='DeltaXYZWLHRBBoxCoder', code_size=9), loss_cls=dict( type='FocalLoss', diff --git a/configs/imvoxelnet/imvoxelnet_kitti-3d-car.py b/configs/imvoxelnet/imvoxelnet_kitti-3d-car.py index 47932d7f6d..06ebe62a2a 100644 --- a/configs/imvoxelnet/imvoxelnet_kitti-3d-car.py +++ b/configs/imvoxelnet/imvoxelnet_kitti-3d-car.py @@ -25,7 +25,7 @@ anchor_generator=dict( type='AlignedAnchor3DRangeGenerator', ranges=[[-0.16, -39.68, -1.78, 68.96, 39.68, -1.78]], - sizes=[[1.6, 3.9, 1.56]], + sizes=[[3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=True), diff_rad_by_sin=True, diff --git a/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py b/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py index 213b626dcb..e9f592f5f5 100644 --- a/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py +++ b/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py @@ -74,7 +74,7 @@ [0, -40.0, -0.6, 70.4, 40.0, -0.6], [0, -40.0, -1.78, 70.4, 40.0, -1.78], ], - sizes=[[0.6, 0.8, 1.73], [0.6, 1.76, 1.73], [1.6, 3.9, 1.56]], + sizes=[[0.8, 0.6, 1.73], [1.76, 0.6, 1.73], [3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=False), assigner_per_size=True, diff --git a/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-car.py b/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-car.py index 91cf983d9c..89be085d84 100644 --- a/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-car.py +++ b/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-car.py @@ -10,7 +10,7 @@ _delete_=True, type='Anchor3DRangeGenerator', ranges=[[0, -40.0, -1.78, 70.4, 40.0, -1.78]], - sizes=[[1.6, 3.9, 1.56]], + sizes=[[3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=False)), roi_head=dict( diff --git a/configs/pointpillars/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car.py b/configs/pointpillars/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car.py index 1e0f0faf9b..50b89d6aae 100644 --- a/configs/pointpillars/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car.py +++ b/configs/pointpillars/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car.py @@ -10,7 +10,7 @@ _delete_=True, type='Anchor3DRangeGenerator', ranges=[[0, -39.68, -1.78, 69.12, 39.68, -1.78]], - sizes=[[1.6, 3.9, 1.56]], + sizes=[[3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=True)), # model training and testing settings diff --git a/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_2x8_2x_lyft-3d.py b/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_2x8_2x_lyft-3d.py index 46d7b06723..1a0400eb33 100644 --- a/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_2x8_2x_lyft-3d.py +++ b/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_2x8_2x_lyft-3d.py @@ -29,15 +29,15 @@ [-80, -80, -0.9122268, 80, 80, -0.9122268], [-80, -80, -1.8012227, 80, 80, -1.8012227]], sizes=[ - [1.92, 4.75, 1.71], # car - [2.84, 10.24, 3.44], # truck - [2.92, 12.70, 3.42], # bus - [2.42, 6.52, 2.34], # emergency vehicle - [2.75, 8.17, 3.20], # other vehicle - [0.96, 2.35, 1.59], # motorcycle - [0.63, 1.76, 1.44], # bicycle - [0.76, 0.80, 1.76], # pedestrian - [0.35, 0.73, 0.50] # animal + [4.75, 1.92, 1.71], # car + [10.24, 2.84, 3.44], # truck + [12.70, 2.92, 3.42], # bus + [6.52, 2.42, 2.34], # emergency vehicle + [8.17, 2.75, 3.20], # other vehicle + [2.35, 0.96, 1.59], # motorcycle + [1.76, 0.63, 1.44], # bicycle + [0.80, 0.76, 1.76], # pedestrian + [0.73, 0.35, 0.50] # animal ], rotations=[0, 1.57], reshape_out=True))) diff --git a/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_4x8_2x_nus-3d.py b/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_4x8_2x_nus-3d.py index 868c7ff8c9..afff99c630 100644 --- a/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_4x8_2x_nus-3d.py +++ b/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_4x8_2x_nus-3d.py @@ -29,13 +29,13 @@ [-49.6, -49.6, -1.763965, 49.6, 49.6, -1.763965], ], sizes=[ - [1.95017717, 4.60718145, 1.72270761], # car - [2.4560939, 6.73778078, 2.73004906], # truck - [2.87427237, 12.01320693, 3.81509561], # trailer - [0.60058911, 1.68452161, 1.27192197], # bicycle - [0.66344886, 0.7256437, 1.75748069], # pedestrian - [0.39694519, 0.40359262, 1.06232151], # traffic_cone - [2.49008838, 0.48578221, 0.98297065], # barrier + [4.60718145, 1.95017717, 1.72270761], # car + [6.73778078, 2.4560939, 2.73004906], # truck + [12.01320693, 2.87427237, 3.81509561], # trailer + [1.68452161, 0.60058911, 1.27192197], # bicycle + [0.7256437, 0.66344886, 1.75748069], # pedestrian + [0.40359262, 0.39694519, 1.06232151], # traffic_cone + [0.48578221, 2.49008838, 0.98297065], # barrier ], custom_values=[0, 0], rotations=[0, 1.57], diff --git a/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_range100_2x8_2x_lyft-3d.py b/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_range100_2x8_2x_lyft-3d.py index fa18aca2c3..7964b79982 100644 --- a/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_range100_2x8_2x_lyft-3d.py +++ b/configs/pointpillars/hv_pointpillars_secfpn_sbn-all_range100_2x8_2x_lyft-3d.py @@ -28,15 +28,15 @@ [-100, -100, -0.9122268, 100, 100, -0.9122268], [-100, -100, -1.8012227, 100, 100, -1.8012227]], sizes=[ - [1.92, 4.75, 1.71], # car - [2.84, 10.24, 3.44], # truck - [2.92, 12.70, 3.42], # bus - [2.42, 6.52, 2.34], # emergency vehicle - [2.75, 8.17, 3.20], # other vehicle - [0.96, 2.35, 1.59], # motorcycle - [0.63, 1.76, 1.44], # bicycle - [0.76, 0.80, 1.76], # pedestrian - [0.35, 0.73, 0.50] # animal + [4.75, 1.92, 1.71], # car + [10.24, 2.84, 3.44], # truck + [12.70, 2.92, 3.42], # bus + [6.52, 2.42, 2.34], # emergency vehicle + [8.17, 2.75, 3.20], # other vehicle + [2.35, 0.96, 1.59], # motorcycle + [1.76, 0.63, 1.44], # bicycle + [0.80, 0.76, 1.76], # pedestrian + [0.73, 0.35, 0.50] # animal ], rotations=[0, 1.57], reshape_out=True))) diff --git a/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymo-3d-car.py b/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymo-3d-car.py index aeac750d9e..90f2a42c53 100644 --- a/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymo-3d-car.py +++ b/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymo-3d-car.py @@ -17,7 +17,7 @@ anchor_generator=dict( type='AlignedAnchor3DRangeGenerator', ranges=[[-74.88, -74.88, -0.0345, 74.88, 74.88, -0.0345]], - sizes=[[2.08, 4.73, 1.77]], + sizes=[[4.73, 2.08, 1.77]], rotations=[0, 1.57], reshape_out=True)), # model training and testing settings diff --git a/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-car.py b/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-car.py index 1fe32fd404..3a3e326698 100644 --- a/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-car.py +++ b/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-car.py @@ -14,7 +14,7 @@ anchor_generator=dict( type='AlignedAnchor3DRangeGenerator', ranges=[[-74.88, -74.88, -0.0345, 74.88, 74.88, -0.0345]], - sizes=[[2.08, 4.73, 1.77]], + sizes=[[4.73, 2.08, 1.77]], rotations=[0, 1.57], reshape_out=True)), # model training and testing settings diff --git a/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_2x8_2x_lyft-3d.py b/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_2x8_2x_lyft-3d.py index f1a3f4d828..fb330d7855 100644 --- a/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_2x8_2x_lyft-3d.py +++ b/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_2x8_2x_lyft-3d.py @@ -25,15 +25,15 @@ [-80, -80, -0.9122268, 80, 80, -0.9122268], [-80, -80, -1.8012227, 80, 80, -1.8012227]], sizes=[ - [1.92, 4.75, 1.71], # car - [2.84, 10.24, 3.44], # truck - [2.92, 12.70, 3.42], # bus - [2.42, 6.52, 2.34], # emergency vehicle - [2.75, 8.17, 3.20], # other vehicle - [0.96, 2.35, 1.59], # motorcycle - [0.63, 1.76, 1.44], # bicycle - [0.76, 0.80, 1.76], # pedestrian - [0.35, 0.73, 0.50] # animal + [4.75, 1.92, 1.71], # car + [10.24, 2.84, 3.44], # truck + [12.70, 2.92, 3.42], # bus + [6.52, 2.42, 2.34], # emergency vehicle + [8.17, 2.75, 3.20], # other vehicle + [2.35, 0.96, 1.59], # motorcycle + [1.76, 0.63, 1.44], # bicycle + [0.80, 0.76, 1.76], # pedestrian + [0.73, 0.35, 0.50] # animal ], rotations=[0, 1.57], reshape_out=True))) diff --git a/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_4x8_2x_nus-3d.py b/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_4x8_2x_nus-3d.py index 0f9e031f90..ef8996a18a 100644 --- a/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_4x8_2x_nus-3d.py +++ b/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_4x8_2x_nus-3d.py @@ -25,13 +25,13 @@ [-49.6, -49.6, -1.763965, 49.6, 49.6, -1.763965], ], sizes=[ - [1.95017717, 4.60718145, 1.72270761], # car - [2.4560939, 6.73778078, 2.73004906], # truck - [2.87427237, 12.01320693, 3.81509561], # trailer - [0.60058911, 1.68452161, 1.27192197], # bicycle - [0.66344886, 0.7256437, 1.75748069], # pedestrian - [0.39694519, 0.40359262, 1.06232151], # traffic_cone - [2.49008838, 0.48578221, 0.98297065], # barrier + [4.60718145, 1.95017717, 1.72270761], # car + [6.73778078, 2.4560939, 2.73004906], # truck + [12.01320693, 2.87427237, 3.81509561], # trailer + [1.68452161, 0.60058911, 1.27192197], # bicycle + [0.7256437, 0.66344886, 1.75748069], # pedestrian + [0.40359262, 0.39694519, 1.06232151], # traffic_cone + [0.48578221, 2.49008838, 0.98297065], # barrier ], custom_values=[0, 0], rotations=[0, 1.57], diff --git a/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_range100_2x8_2x_lyft-3d.py b/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_range100_2x8_2x_lyft-3d.py index c7bc8f16eb..2af3719c9b 100644 --- a/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_range100_2x8_2x_lyft-3d.py +++ b/configs/regnet/hv_pointpillars_regnet-400mf_secfpn_sbn-all_range100_2x8_2x_lyft-3d.py @@ -26,15 +26,15 @@ [-100, -100, -0.9122268, 100, 100, -0.9122268], [-100, -100, -1.8012227, 100, 100, -1.8012227]], sizes=[ - [1.92, 4.75, 1.71], # car - [2.84, 10.24, 3.44], # truck - [2.92, 12.70, 3.42], # bus - [2.42, 6.52, 2.34], # emergency vehicle - [2.75, 8.17, 3.20], # other vehicle - [0.96, 2.35, 1.59], # motorcycle - [0.63, 1.76, 1.44], # bicycle - [0.76, 0.80, 1.76], # pedestrian - [0.35, 0.73, 0.50] # animal + [4.75, 1.92, 1.71], # car + [10.24, 2.84, 3.44], # truck + [12.70, 2.92, 3.42], # bus + [6.52, 2.42, 2.34], # emergency vehicle + [8.17, 2.75, 3.20], # other vehicle + [2.35, 0.96, 1.59], # motorcycle + [1.76, 0.63, 1.44], # bicycle + [0.80, 0.76, 1.76], # pedestrian + [0.73, 0.35, 0.50] # animal ], rotations=[0, 1.57], reshape_out=True))) diff --git a/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-car.py b/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-car.py index c4f2ffd51a..9ab7350ac4 100644 --- a/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-car.py +++ b/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-car.py @@ -12,7 +12,7 @@ _delete_=True, type='Anchor3DRangeGenerator', ranges=[[0, -40.0, -1.78, 70.4, 40.0, -1.78]], - sizes=[[1.6, 3.9, 1.56]], + sizes=[[3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=True)), # model training and testing settings diff --git a/configs/second/hv_second_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py b/configs/second/hv_second_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py index aae54b33a8..6412f535d5 100644 --- a/configs/second/hv_second_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py +++ b/configs/second/hv_second_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py @@ -21,7 +21,10 @@ classes=class_names, sample_groups=dict(Car=15, Pedestrian=10, Cyclist=10), points_loader=dict( - type='LoadPointsFromFile', load_dim=5, use_dim=[0, 1, 2, 3, 4])) + type='LoadPointsFromFile', + coord_type='LIDAR', + load_dim=5, + use_dim=[0, 1, 2, 3, 4])) train_pipeline = [ dict(type='LoadPointsFromFile', coord_type='LIDAR', load_dim=6, use_dim=5), diff --git a/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py b/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py index 8a70d2a770..50b33c8019 100644 --- a/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py +++ b/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py @@ -96,15 +96,15 @@ [-100, -100, -0.6276341, 100, 100, -0.6276341], [-100, -100, -0.3033737, 100, 100, -0.3033737]], sizes=[ - [0.63, 1.76, 1.44], # bicycle - [0.96, 2.35, 1.59], # motorcycle - [0.76, 0.80, 1.76], # pedestrian - [0.35, 0.73, 0.50], # animal - [1.92, 4.75, 1.71], # car - [2.42, 6.52, 2.34], # emergency vehicle - [2.92, 12.70, 3.42], # bus - [2.75, 8.17, 3.20], # other vehicle - [2.84, 10.24, 3.44] # truck + [1.76, 0.63, 1.44], # bicycle + [2.35, 0.96, 1.59], # motorcycle + [0.80, 0.76, 1.76], # pedestrian + [0.73, 0.35, 0.50], # animal + [4.75, 1.92, 1.71], # car + [6.52, 2.42, 2.34], # emergency vehicle + [12.70, 2.92, 3.42], # bus + [8.17, 2.75, 3.20], # other vehicle + [10.24, 2.84, 3.44] # truck ], custom_values=[], rotations=[0, 1.57], @@ -137,7 +137,7 @@ ], assign_per_class=True, diff_rad_by_sin=True, - dir_offset=0.7854, # pi/4 + dir_offset=-0.7854, # -pi/4 dir_limit_offset=0, bbox_coder=dict(type='DeltaXYZWLHRBBoxCoder', code_size=7), loss_cls=dict( diff --git a/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_nus-3d.py b/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_nus-3d.py index 18b658b0c3..8550201416 100644 --- a/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_nus-3d.py +++ b/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_nus-3d.py @@ -94,16 +94,16 @@ [-50, -50, -1.80673031, 50, 50, -1.80673031], [-50, -50, -1.64824291, 50, 50, -1.64824291]], sizes=[ - [0.60058911, 1.68452161, 1.27192197], # bicycle - [0.76279481, 2.09973778, 1.44403034], # motorcycle - [0.66344886, 0.72564370, 1.75748069], # pedestrian - [0.39694519, 0.40359262, 1.06232151], # traffic cone - [2.49008838, 0.48578221, 0.98297065], # barrier - [1.95017717, 4.60718145, 1.72270761], # car - [2.45609390, 6.73778078, 2.73004906], # truck - [2.87427237, 12.01320693, 3.81509561], # trailer - [2.94046906, 11.1885991, 3.47030982], # bus - [2.73050468, 6.38352896, 3.13312415] # construction vehicle + [1.68452161, 0.60058911, 1.27192197], # bicycle + [2.09973778, 0.76279481, 1.44403034], # motorcycle + [0.72564370, 0.66344886, 1.75748069], # pedestrian + [0.40359262, 0.39694519, 1.06232151], # traffic cone + [0.48578221, 2.49008838, 0.98297065], # barrier + [4.60718145, 1.95017717, 1.72270761], # car + [6.73778078, 2.45609390, 2.73004906], # truck + [12.01320693, 2.87427237, 3.81509561], # trailer + [11.1885991, 2.94046906, 3.47030982], # bus + [6.38352896, 2.73050468, 3.13312415] # construction vehicle ], custom_values=[0, 0], rotations=[0, 1.57], @@ -144,7 +144,7 @@ ], assign_per_class=True, diff_rad_by_sin=True, - dir_offset=0.7854, # pi/4 + dir_offset=-0.7854, # -pi/4 dir_limit_offset=0, bbox_coder=dict(type='DeltaXYZWLHRBBoxCoder', code_size=9), loss_cls=dict( diff --git a/mmdet3d/apis/inference.py b/mmdet3d/apis/inference.py index ea109208b5..cbd4b14fe6 100644 --- a/mmdet3d/apis/inference.py +++ b/mmdet3d/apis/inference.py @@ -7,10 +7,9 @@ from mmcv.runner import load_checkpoint from os import path as osp -from mmdet3d.core import (Box3DMode, CameraInstance3DBoxes, - DepthInstance3DBoxes, LiDARInstance3DBoxes, - show_multi_modality_result, show_result, - show_seg_result) +from mmdet3d.core import (Box3DMode, Coord3DMode, DepthInstance3DBoxes, + LiDARInstance3DBoxes, show_multi_modality_result, + show_result, show_seg_result) from mmdet3d.core.bbox import get_box_type from mmdet3d.datasets.pipelines import Compose from mmdet3d.models import build_model @@ -315,8 +314,7 @@ def show_det_result_meshlab(data, # for now we convert points into depth mode box_mode = data['img_metas'][0][0]['box_mode_3d'] if box_mode != Box3DMode.DEPTH: - points = points[..., [1, 0, 2]] - points[..., 0] *= -1 + points = Coord3DMode.convert(points, box_mode, Coord3DMode.DEPTH) show_bboxes = Box3DMode.convert(pred_bboxes, box_mode, Box3DMode.DEPTH) else: show_bboxes = deepcopy(pred_bboxes) diff --git a/mmdet3d/core/anchor/anchor_3d_generator.py b/mmdet3d/core/anchor/anchor_3d_generator.py index f9343ef996..752a5cbcba 100644 --- a/mmdet3d/core/anchor/anchor_3d_generator.py +++ b/mmdet3d/core/anchor/anchor_3d_generator.py @@ -31,7 +31,7 @@ class Anchor3DRangeGenerator(object): def __init__(self, ranges, - sizes=[[1.6, 3.9, 1.56]], + sizes=[[3.9, 1.6, 1.56]], scales=[1], rotations=[0, 1.5707963], custom_values=(), @@ -148,7 +148,7 @@ def anchors_single_range(self, feature_size, anchor_range, scale=1, - sizes=[[1.6, 3.9, 1.56]], + sizes=[[3.9, 1.6, 1.56]], rotations=[0, 1.5707963], device='cuda'): """Generate anchors in a single range. @@ -244,7 +244,7 @@ def anchors_single_range(self, feature_size, anchor_range, scale, - sizes=[[1.6, 3.9, 1.56]], + sizes=[[3.9, 1.6, 1.56]], rotations=[0, 1.5707963], device='cuda'): """Generate anchors in a single range. diff --git a/mmdet3d/core/bbox/box_np_ops.py b/mmdet3d/core/bbox/box_np_ops.py index 256c436a4e..c8d55d6654 100644 --- a/mmdet3d/core/bbox/box_np_ops.py +++ b/mmdet3d/core/bbox/box_np_ops.py @@ -1,13 +1,17 @@ # TODO: clean the functions in this file and move the APIs into box structures # in the future - import numba import numpy as np +from .structures.utils import limit_period, points_cam2img, rotation_3d_in_axis + def camera_to_lidar(points, r_rect, velo2cam): """Convert points in camera coordinate to lidar coordinate. + Note: + This function is for KITTI only. + Args: points (np.ndarray, shape=[N, 3]): Points in camera coordinate. r_rect (np.ndarray, shape=[4, 4]): Matrix to project points in @@ -26,7 +30,10 @@ def camera_to_lidar(points, r_rect, velo2cam): def box_camera_to_lidar(data, r_rect, velo2cam): - """Covert boxes in camera coordinate to lidar coordinate. + """Convert boxes in camera coordinate to lidar coordinate. + + Note: + This function is for KITTI only. Args: data (np.ndarray, shape=[N, 7]): Boxes in camera coordinate. @@ -39,10 +46,13 @@ def box_camera_to_lidar(data, r_rect, velo2cam): np.ndarray, shape=[N, 3]: Boxes in lidar coordinate. """ xyz = data[:, 0:3] - l, h, w = data[:, 3:4], data[:, 4:5], data[:, 5:6] + dx, dy, dz = data[:, 3:4], data[:, 4:5], data[:, 5:6] r = data[:, 6:7] xyz_lidar = camera_to_lidar(xyz, r_rect, velo2cam) - return np.concatenate([xyz_lidar, w, l, h, r], axis=1) + # yaw and dims also needs to be converted + r_new = -r - np.pi / 2 + r_new = limit_period(r_new, period=np.pi * 2) + return np.concatenate([xyz_lidar, dx, dz, dy, r_new], axis=1) def corners_nd(dims, origin=0.5): @@ -79,23 +89,6 @@ def corners_nd(dims, origin=0.5): return corners -def rotation_2d(points, angles): - """Rotation 2d points based on origin point clockwise when angle positive. - - Args: - points (np.ndarray): Points to be rotated with shape \ - (N, point_size, 2). - angles (np.ndarray): Rotation angle with shape (N). - - Returns: - np.ndarray: Same shape as points. - """ - rot_sin = np.sin(angles) - rot_cos = np.cos(angles) - rot_mat_T = np.stack([[rot_cos, -rot_sin], [rot_sin, rot_cos]]) - return np.einsum('aij,jka->aik', points, rot_mat_T) - - def center_to_corner_box2d(centers, dims, angles=None, origin=0.5): """Convert kitti locations, dimensions and angles to corners. format: center(xy), dims(xy), angles(clockwise when positive) @@ -117,7 +110,7 @@ def center_to_corner_box2d(centers, dims, angles=None, origin=0.5): corners = corners_nd(dims, origin=origin) # corners: [N, 4, 2] if angles is not None: - corners = rotation_2d(corners, angles) + corners = rotation_3d_in_axis(corners, angles) corners += centers.reshape([-1, 1, 2]) return corners @@ -171,37 +164,6 @@ def depth_to_lidar_points(depth, trunc_pixel, P2, r_rect, velo2cam): return lidar_points -def rotation_3d_in_axis(points, angles, axis=0): - """Rotate points in specific axis. - - Args: - points (np.ndarray, shape=[N, point_size, 3]]): - angles (np.ndarray, shape=[N]]): - axis (int, optional): Axis to rotate at. Defaults to 0. - - Returns: - np.ndarray: Rotated points. - """ - # points: [N, point_size, 3] - rot_sin = np.sin(angles) - rot_cos = np.cos(angles) - ones = np.ones_like(rot_cos) - zeros = np.zeros_like(rot_cos) - if axis == 1: - rot_mat_T = np.stack([[rot_cos, zeros, -rot_sin], [zeros, ones, zeros], - [rot_sin, zeros, rot_cos]]) - elif axis == 2 or axis == -1: - rot_mat_T = np.stack([[rot_cos, -rot_sin, zeros], - [rot_sin, rot_cos, zeros], [zeros, zeros, ones]]) - elif axis == 0: - rot_mat_T = np.stack([[zeros, rot_cos, -rot_sin], - [zeros, rot_sin, rot_cos], [ones, zeros, zeros]]) - else: - raise ValueError('axis should in range') - - return np.einsum('aij,jka->aik', points, rot_mat_T) - - def center_to_corner_box3d(centers, dims, angles=None, @@ -258,8 +220,8 @@ def box2d_to_corner_jit(boxes): rot_sin = np.sin(boxes[i, -1]) rot_cos = np.cos(boxes[i, -1]) rot_mat_T[0, 0] = rot_cos - rot_mat_T[0, 1] = -rot_sin - rot_mat_T[1, 0] = rot_sin + rot_mat_T[0, 1] = rot_sin + rot_mat_T[1, 0] = -rot_sin rot_mat_T[1, 1] = rot_cos box_corners[i] = corners[i] @ rot_mat_T + boxes[i, :2] return box_corners @@ -326,15 +288,15 @@ def rotation_points_single_angle(points, angle, axis=0): rot_cos = np.cos(angle) if axis == 1: rot_mat_T = np.array( - [[rot_cos, 0, -rot_sin], [0, 1, 0], [rot_sin, 0, rot_cos]], + [[rot_cos, 0, rot_sin], [0, 1, 0], [-rot_sin, 0, rot_cos]], dtype=points.dtype) elif axis == 2 or axis == -1: rot_mat_T = np.array( - [[rot_cos, -rot_sin, 0], [rot_sin, rot_cos, 0], [0, 0, 1]], + [[rot_cos, rot_sin, 0], [-rot_sin, rot_cos, 0], [0, 0, 1]], dtype=points.dtype) elif axis == 0: rot_mat_T = np.array( - [[1, 0, 0], [0, rot_cos, -rot_sin], [0, rot_sin, rot_cos]], + [[1, 0, 0], [0, rot_cos, rot_sin], [0, -rot_sin, rot_cos]], dtype=points.dtype) else: raise ValueError('axis should in range') @@ -342,44 +304,6 @@ def rotation_points_single_angle(points, angle, axis=0): return points @ rot_mat_T, rot_mat_T -def points_cam2img(points_3d, proj_mat, with_depth=False): - """Project points in camera coordinates to image coordinates. - - Args: - points_3d (np.ndarray): Points in shape (N, 3) - proj_mat (np.ndarray): Transformation matrix between coordinates. - with_depth (bool, optional): Whether to keep depth in the output. - Defaults to False. - - Returns: - np.ndarray: Points in image coordinates with shape [N, 2]. - """ - points_shape = list(points_3d.shape) - points_shape[-1] = 1 - - assert len(proj_mat.shape) == 2, 'The dimension of the projection'\ - f' matrix should be 2 instead of {len(proj_mat.shape)}.' - d1, d2 = proj_mat.shape[:2] - assert (d1 == 3 and d2 == 3) or (d1 == 3 and d2 == 4) or ( - d1 == 4 and d2 == 4), 'The shape of the projection matrix'\ - f' ({d1}*{d2}) is not supported.' - if d1 == 3: - proj_mat_expanded = np.eye(4, dtype=proj_mat.dtype) - proj_mat_expanded[:d1, :d2] = proj_mat - proj_mat = proj_mat_expanded - - points_4 = np.concatenate([points_3d, np.ones(points_shape)], axis=-1) - point_2d = points_4 @ proj_mat.T - point_2d_res = point_2d[..., :2] / point_2d[..., 2:3] - - if with_depth: - points_2d_depth = np.concatenate([point_2d_res, point_2d[..., 2:3]], - axis=-1) - return points_2d_depth - - return point_2d_res - - def box3d_to_bbox(box3d, P2): """Convert box3d in camera coordinates to bbox in image coordinates. @@ -460,25 +384,9 @@ def minmax_to_corner_2d(minmax_box): return center_to_corner_box2d(center, dims, origin=0.0) -def limit_period(val, offset=0.5, period=np.pi): - """Limit the value into a period for periodic function. - - Args: - val (np.ndarray): The value to be converted. - offset (float, optional): Offset to set the value range. \ - Defaults to 0.5. - period (float, optional): Period of the value. Defaults to np.pi. - - Returns: - torch.Tensor: Value in the range of \ - [-offset * period, (1-offset) * period] - """ - return val - np.floor(val / period + offset) * period - - def create_anchors_3d_range(feature_size, anchor_range, - sizes=((1.6, 3.9, 1.56), ), + sizes=((3.9, 1.6, 1.56), ), rotations=(0, np.pi / 2), dtype=np.float32): """Create anchors 3d by range. @@ -491,14 +399,14 @@ def create_anchors_3d_range(feature_size, (x_min, y_min, z_min, x_max, y_max, z_max). sizes (list[list] | np.ndarray | torch.Tensor, optional): Anchor size with shape [N, 3], in order of x, y, z. - Defaults to ((1.6, 3.9, 1.56), ). + Defaults to ((3.9, 1.6, 1.56), ). rotations (list[float] | np.ndarray | torch.Tensor, optional): Rotations of anchors in a single feature grid. Defaults to (0, np.pi / 2). dtype (type, optional): Data type. Default to np.float32. Returns: - np.ndarray: Range based anchors with shape of \ + np.ndarray: Range based anchors with shape of (*feature_size, num_sizes, num_rots, 7). """ anchor_range = np.array(anchor_range, dtype) @@ -549,7 +457,7 @@ def rbbox2d_to_near_bbox(rbboxes): """convert rotated bbox to nearest 'standing' or 'lying' bbox. Args: - rbboxes (np.ndarray): Rotated bboxes with shape of \ + rbboxes (np.ndarray): Rotated bboxes with shape of (N, 5(x, y, xdim, ydim, rad)). Returns: @@ -840,8 +748,8 @@ def boxes3d_to_corners3d_lidar(boxes3d, bottom_center=True): Args: boxes3d (np.ndarray): Boxes with shape of (N, 7) - [x, y, z, w, l, h, ry] in LiDAR coords, see the definition of ry - in KITTI dataset. + [x, y, z, dx, dy, dz, ry] in LiDAR coords, see the definition of + ry in KITTI dataset. bottom_center (bool, optional): Whether z is on the bottom center of object. Defaults to True. @@ -849,19 +757,25 @@ def boxes3d_to_corners3d_lidar(boxes3d, bottom_center=True): np.ndarray: Box corners with the shape of [N, 8, 3]. """ boxes_num = boxes3d.shape[0] - w, l, h = boxes3d[:, 3], boxes3d[:, 4], boxes3d[:, 5] - x_corners = np.array( - [w / 2., -w / 2., -w / 2., w / 2., w / 2., -w / 2., -w / 2., w / 2.], - dtype=np.float32).T - y_corners = np.array( - [-l / 2., -l / 2., l / 2., l / 2., -l / 2., -l / 2., l / 2., l / 2.], - dtype=np.float32).T + dx, dy, dz = boxes3d[:, 3], boxes3d[:, 4], boxes3d[:, 5] + x_corners = np.array([ + dx / 2., -dx / 2., -dx / 2., dx / 2., dx / 2., -dx / 2., -dx / 2., + dx / 2. + ], + dtype=np.float32).T + y_corners = np.array([ + -dy / 2., -dy / 2., dy / 2., dy / 2., -dy / 2., -dy / 2., dy / 2., + dy / 2. + ], + dtype=np.float32).T if bottom_center: z_corners = np.zeros((boxes_num, 8), dtype=np.float32) - z_corners[:, 4:8] = h.reshape(boxes_num, 1).repeat(4, axis=1) # (N, 8) + z_corners[:, 4:8] = dz.reshape(boxes_num, 1).repeat( + 4, axis=1) # (N, 8) else: z_corners = np.array([ - -h / 2., -h / 2., -h / 2., -h / 2., h / 2., h / 2., h / 2., h / 2. + -dz / 2., -dz / 2., -dz / 2., -dz / 2., dz / 2., dz / 2., dz / 2., + dz / 2. ], dtype=np.float32).T @@ -869,9 +783,9 @@ def boxes3d_to_corners3d_lidar(boxes3d, bottom_center=True): zeros, ones = np.zeros( ry.size, dtype=np.float32), np.ones( ry.size, dtype=np.float32) - rot_list = np.array([[np.cos(ry), -np.sin(ry), zeros], - [np.sin(ry), np.cos(ry), zeros], [zeros, zeros, - ones]]) # (3, 3, N) + rot_list = np.array([[np.cos(ry), np.sin(ry), zeros], + [-np.sin(ry), np.cos(ry), zeros], + [zeros, zeros, ones]]) # (3, 3, N) R_list = np.transpose(rot_list, (2, 0, 1)) # (N, 3, 3) temp_corners = np.concatenate((x_corners.reshape( diff --git a/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py b/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py index bbc3447751..6a930afe41 100644 --- a/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py +++ b/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py @@ -30,8 +30,10 @@ def __call__(self, bboxes1, bboxes2, mode='iou', is_aligned=False): between each aligned pair of bboxes1 and bboxes2. Args: - bboxes1 (torch.Tensor): shape (N, 7+N) [x, y, z, h, w, l, ry, v]. - bboxes2 (torch.Tensor): shape (M, 7+N) [x, y, z, h, w, l, ry, v]. + bboxes1 (torch.Tensor): shape (N, 7+N) + [x, y, z, dx, dy, dz, ry, v]. + bboxes2 (torch.Tensor): shape (M, 7+N) + [x, y, z, dx, dy, dz, ry, v]. mode (str): "iou" (intersection over union) or iof (intersection over foreground). is_aligned (bool): Whether the calculation is aligned. @@ -73,8 +75,8 @@ def __call__(self, bboxes1, bboxes2, mode='iou'): calculate the actual 3D IoUs of boxes. Args: - bboxes1 (torch.Tensor): shape (N, 7+C) [x, y, z, h, w, l, ry]. - bboxes2 (torch.Tensor): shape (M, 7+C) [x, y, z, h, w, l, ry]. + bboxes1 (torch.Tensor): shape (N, 7+C) [x, y, z, dx, dy, dz, ry]. + bboxes2 (torch.Tensor): shape (M, 7+C) [x, y, z, dx, dy, dz, ry]. mode (str): "iou" (intersection over union) or iof (intersection over foreground). @@ -109,8 +111,8 @@ def bbox_overlaps_nearest_3d(bboxes1, aligned pair of bboxes1 and bboxes2. Args: - bboxes1 (torch.Tensor): shape (N, 7+C) [x, y, z, h, w, l, ry, v]. - bboxes2 (torch.Tensor): shape (M, 7+C) [x, y, z, h, w, l, ry, v]. + bboxes1 (torch.Tensor): shape (N, 7+C) [x, y, z, dx, dy, dz, ry, v]. + bboxes2 (torch.Tensor): shape (M, 7+C) [x, y, z, dx, dy, dz, ry, v]. mode (str): "iou" (intersection over union) or iof (intersection over foreground). is_aligned (bool): Whether the calculation is aligned @@ -147,8 +149,8 @@ def bbox_overlaps_3d(bboxes1, bboxes2, mode='iou', coordinate='camera'): calculate the actual IoUs of boxes. Args: - bboxes1 (torch.Tensor): shape (N, 7+C) [x, y, z, h, w, l, ry]. - bboxes2 (torch.Tensor): shape (M, 7+C) [x, y, z, h, w, l, ry]. + bboxes1 (torch.Tensor): shape (N, 7+C) [x, y, z, dx, dy, dz, ry]. + bboxes2 (torch.Tensor): shape (M, 7+C) [x, y, z, dx, dy, dz, ry]. mode (str): "iou" (intersection over union) or iof (intersection over foreground). coordinate (str): 'camera' or 'lidar' coordinate system. diff --git a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py index 8a309ce2f2..955f1546df 100644 --- a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py +++ b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py @@ -8,8 +8,8 @@ class IoUNegPiecewiseSampler(RandomSampler): """IoU Piece-wise Sampling. - Sampling negtive proposals according to a list of IoU thresholds. - The negtive proposals are divided into several pieces according + Sampling negative proposals according to a list of IoU thresholds. + The negative proposals are divided into several pieces according to `neg_iou_piece_thrs`. And the ratio of each piece is indicated by `neg_piece_fractions`. @@ -17,11 +17,11 @@ class IoUNegPiecewiseSampler(RandomSampler): num (int): Number of proposals. pos_fraction (float): The fraction of positive proposals. neg_piece_fractions (list): A list contains fractions that indicates - the ratio of each piece of total negtive samplers. + the ratio of each piece of total negative samplers. neg_iou_piece_thrs (list): A list contains IoU thresholds that indicate the upper bound of this piece. neg_pos_ub (float): The total ratio to limit the upper bound - number of negtive samples. + number of negative samples. add_gt_as_proposals (bool): Whether to add gt as proposals. """ diff --git a/mmdet3d/core/bbox/structures/base_box3d.py b/mmdet3d/core/bbox/structures/base_box3d.py index 1c6549532b..b3ee156e85 100644 --- a/mmdet3d/core/bbox/structures/base_box3d.py +++ b/mmdet3d/core/bbox/structures/base_box3d.py @@ -2,6 +2,7 @@ import torch from abc import abstractmethod +from mmdet3d.ops import points_in_boxes_batch, points_in_boxes_gpu from mmdet3d.ops.iou3d import iou3d_cuda from .utils import limit_period, xywhr2xyxyr @@ -130,8 +131,8 @@ def corners(self): @abstractmethod def rotate(self, angle, points=None): - """Rotate boxes with points (optional) with the given angle or \ - rotation matrix. + """Rotate boxes with points (optional) with the given angle or rotation + matrix. Args: angle (float | torch.Tensor | np.ndarray): @@ -169,7 +170,7 @@ def in_range_3d(self, box_range): polygon, we try to reduce the burden for simpler cases. Returns: - torch.Tensor: A binary vector indicating whether each box is \ + torch.Tensor: A binary vector indicating whether each box is inside the reference range. """ in_range_flags = ((self.tensor[:, 0] > box_range[0]) @@ -189,7 +190,7 @@ def in_range_bev(self, box_range): in order of (x_min, y_min, x_max, y_max). Returns: - torch.Tensor: Indicating whether each box is inside \ + torch.Tensor: Indicating whether each box is inside the reference range. """ pass @@ -207,7 +208,7 @@ def convert_to(self, dst, rt_mat=None): to LiDAR. This requires a transformation matrix. Returns: - :obj:`BaseInstance3DBoxes`: The converted box of the same type \ + :obj:`BaseInstance3DBoxes`: The converted box of the same type in the `dst` mode. """ pass @@ -240,7 +241,7 @@ def nonempty(self, threshold: float = 0.0): threshold (float): The threshold of minimal sizes. Returns: - torch.Tensor: A binary vector which represents whether each \ + torch.Tensor: A binary vector which represents whether each box is empty (False) or non-empty (True). """ box = self.tensor @@ -266,8 +267,8 @@ def __getitem__(self, item): subject to Pytorch's indexing semantics. Returns: - :obj:`BaseInstance3DBoxes`: A new object of \ - :class:`BaseInstances3DBoxes` after indexing. + :obj:`BaseInstance3DBoxes`: A new object of + :class:`BaseInstance3DBoxes` after indexing. """ original_type = type(self) if isinstance(item, int): @@ -318,7 +319,7 @@ def to(self, device): device (str | :obj:`torch.device`): The name of the device. Returns: - :obj:`BaseInstance3DBoxes`: A new boxes object on the \ + :obj:`BaseInstance3DBoxes`: A new boxes object on the specific device. """ original_type = type(self) @@ -331,7 +332,7 @@ def clone(self): """Clone the Boxes. Returns: - :obj:`BaseInstance3DBoxes`: Box object with the same properties \ + :obj:`BaseInstance3DBoxes`: Box object with the same properties as self. """ original_type = type(self) @@ -443,14 +444,14 @@ def overlaps(cls, boxes1, boxes2, mode='iou'): def new_box(self, data): """Create a new box object with data. - The new box and its tensor has the similar properties \ + The new box and its tensor has the similar properties as self and self.tensor, respectively. Args: data (torch.Tensor | numpy.array | list): Data to be copied. Returns: - :obj:`BaseInstance3DBoxes`: A new bbox object with ``data``, \ + :obj:`BaseInstance3DBoxes`: A new bbox object with ``data``, the object's other properties are similar to ``self``. """ new_tensor = self.tensor.new_tensor(data) \ @@ -458,3 +459,48 @@ def new_box(self, data): original_type = type(self) return original_type( new_tensor, box_dim=self.box_dim, with_yaw=self.with_yaw) + + def points_in_boxes(self, points, boxes_override=None): + """Find the box which the points are in. + + Args: + points (torch.Tensor): Points in shape (N, 3). + + Returns: + torch.Tensor: The index of box where each point are in. + """ + if boxes_override is not None: + boxes = boxes_override + else: + boxes = self.tensor + box_idx = points_in_boxes_gpu( + points.unsqueeze(0), + boxes.unsqueeze(0).to(points.device)).squeeze(0) + return box_idx + + def points_in_boxes_batch(self, points, boxes_override=None): + """Find points that are in boxes (CUDA). + + Args: + points (torch.Tensor): Points in shape [1, M, 3] or [M, 3], + 3 dimensions are [x, y, z] in LiDAR coordinate. + + Returns: + torch.Tensor: The index of boxes each point lies in with shape + of (B, M, T). + """ + if boxes_override is not None: + boxes = boxes_override + else: + boxes = self.tensor + + points_clone = points.clone()[..., :3] + if points_clone.dim() == 2: + points_clone = points_clone.unsqueeze(0) + else: + assert points_clone.dim() == 3 and points_clone.shape[0] == 1 + + boxes = boxes.to(points_clone.device).unsqueeze(0) + box_idxs_of_pts = points_in_boxes_batch(points_clone, boxes) + + return box_idxs_of_pts.squeeze(0) diff --git a/mmdet3d/core/bbox/structures/box_3d_mode.py b/mmdet3d/core/bbox/structures/box_3d_mode.py index 0b318c36ff..0e98cd8112 100644 --- a/mmdet3d/core/bbox/structures/box_3d_mode.py +++ b/mmdet3d/core/bbox/structures/box_3d_mode.py @@ -6,6 +6,7 @@ from .cam_box3d import CameraInstance3DBoxes from .depth_box3d import DepthInstance3DBoxes from .lidar_box3d import LiDARInstance3DBoxes +from .utils import limit_period @unique @@ -60,12 +61,12 @@ class Box3DMode(IntEnum): DEPTH = 2 @staticmethod - def convert(box, src, dst, rt_mat=None): + def convert(box, src, dst, rt_mat=None, with_yaw=True): """Convert boxes from `src` mode to `dst` mode. Args: box (tuple | list | np.ndarray | - torch.Tensor | BaseInstance3DBoxes): + torch.Tensor | :obj:`BaseInstance3DBoxes`): Can be a k-tuple, k-list or an Nxk array/tensor, where k = 7. src (:obj:`Box3DMode`): The src Box mode. dst (:obj:`Box3DMode`): The target Box mode. @@ -74,9 +75,13 @@ def convert(box, src, dst, rt_mat=None): The conversion from `src` coordinates to `dst` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. + with_yaw (bool): If `box` is an instance of + :obj:`BaseInstance3DBoxes`, whether or not it has a yaw angle. + Defaults to True. Returns: - (tuple | list | np.ndarray | torch.Tensor | BaseInstance3DBoxes): \ + (tuple | list | np.ndarray | torch.Tensor | + :obj:`BaseInstance3DBoxes`): The converted box of the same type. """ if src == dst: @@ -99,32 +104,53 @@ def convert(box, src, dst, rt_mat=None): else: arr = box.clone() + if is_Instance3DBoxes: + with_yaw = box.with_yaw + # convert box from `src` mode to `dst` mode. x_size, y_size, z_size = arr[..., 3:4], arr[..., 4:5], arr[..., 5:6] + if with_yaw: + yaw = arr[..., 6:7] if src == Box3DMode.LIDAR and dst == Box3DMode.CAM: if rt_mat is None: rt_mat = arr.new_tensor([[0, -1, 0], [0, 0, -1], [1, 0, 0]]) - xyz_size = torch.cat([y_size, z_size, x_size], dim=-1) + xyz_size = torch.cat([x_size, z_size, y_size], dim=-1) + if with_yaw: + yaw = -yaw - np.pi / 2 + yaw = limit_period(yaw, period=np.pi * 2) elif src == Box3DMode.CAM and dst == Box3DMode.LIDAR: if rt_mat is None: rt_mat = arr.new_tensor([[0, 0, 1], [-1, 0, 0], [0, -1, 0]]) - xyz_size = torch.cat([z_size, x_size, y_size], dim=-1) + xyz_size = torch.cat([x_size, z_size, y_size], dim=-1) + if with_yaw: + yaw = -yaw - np.pi / 2 + yaw = limit_period(yaw, period=np.pi * 2) elif src == Box3DMode.DEPTH and dst == Box3DMode.CAM: if rt_mat is None: rt_mat = arr.new_tensor([[1, 0, 0], [0, 0, 1], [0, -1, 0]]) xyz_size = torch.cat([x_size, z_size, y_size], dim=-1) + if with_yaw: + yaw = -yaw elif src == Box3DMode.CAM and dst == Box3DMode.DEPTH: if rt_mat is None: rt_mat = arr.new_tensor([[1, 0, 0], [0, 0, -1], [0, 1, 0]]) xyz_size = torch.cat([x_size, z_size, y_size], dim=-1) + if with_yaw: + yaw = -yaw elif src == Box3DMode.LIDAR and dst == Box3DMode.DEPTH: if rt_mat is None: rt_mat = arr.new_tensor([[0, -1, 0], [1, 0, 0], [0, 0, 1]]) - xyz_size = torch.cat([y_size, x_size, z_size], dim=-1) + xyz_size = torch.cat([x_size, y_size, z_size], dim=-1) + if with_yaw: + yaw = yaw + np.pi / 2 + yaw = limit_period(yaw, period=np.pi * 2) elif src == Box3DMode.DEPTH and dst == Box3DMode.LIDAR: if rt_mat is None: rt_mat = arr.new_tensor([[0, 1, 0], [-1, 0, 0], [0, 0, 1]]) - xyz_size = torch.cat([y_size, x_size, z_size], dim=-1) + xyz_size = torch.cat([x_size, y_size, z_size], dim=-1) + if with_yaw: + yaw = yaw - np.pi / 2 + yaw = limit_period(yaw, period=np.pi * 2) else: raise NotImplementedError( f'Conversion from Box3DMode {src} to {dst} ' @@ -134,13 +160,17 @@ def convert(box, src, dst, rt_mat=None): rt_mat = arr.new_tensor(rt_mat) if rt_mat.size(1) == 4: extended_xyz = torch.cat( - [arr[:, :3], arr.new_ones(arr.size(0), 1)], dim=-1) + [arr[..., :3], arr.new_ones(arr.size(0), 1)], dim=-1) xyz = extended_xyz @ rt_mat.t() else: - xyz = arr[:, :3] @ rt_mat.t() + xyz = arr[..., :3] @ rt_mat.t() - remains = arr[..., 6:] - arr = torch.cat([xyz[:, :3], xyz_size, remains], dim=-1) + if with_yaw: + remains = arr[..., 7:] + arr = torch.cat([xyz[..., :3], xyz_size, yaw, remains], dim=-1) + else: + remains = arr[..., 6:] + arr = torch.cat([xyz[..., :3], xyz_size, remains], dim=-1) # convert arr to the original type original_type = type(box) @@ -159,7 +189,6 @@ def convert(box, src, dst, rt_mat=None): raise NotImplementedError( f'Conversion to {dst} through {original_type}' ' is not supported yet') - return target_type( - arr, box_dim=arr.size(-1), with_yaw=box.with_yaw) + return target_type(arr, box_dim=arr.size(-1), with_yaw=with_yaw) else: return arr diff --git a/mmdet3d/core/bbox/structures/cam_box3d.py b/mmdet3d/core/bbox/structures/cam_box3d.py index 9969037f11..7ac29619ed 100644 --- a/mmdet3d/core/bbox/structures/cam_box3d.py +++ b/mmdet3d/core/bbox/structures/cam_box3d.py @@ -1,7 +1,7 @@ import numpy as np import torch -from mmdet3d.core.points import BasePoints +from ...points import BasePoints from .base_box3d import BaseInstance3DBoxes from .utils import limit_period, rotation_3d_in_axis @@ -37,6 +37,7 @@ class CameraInstance3DBoxes(BaseInstance3DBoxes): with_yaw (bool): If True, the value of yaw will be set to 0 as minmax boxes. """ + YAW_AXIS = 1 def __init__(self, tensor, @@ -116,16 +117,16 @@ def corners(self): / | / | (x0, y0, z0) + ----------- + + (x1, y1, z1) | / . | / - | / oriign | / + | / origin | / (x0, y1, z0) + ----------- + -------> x right | (x1, y1, z0) | v down y """ - # TODO: rotation_3d_in_axis function do not support - # empty tensor currently. - assert len(self.tensor) != 0 + if self.tensor.numel() == 0: + return torch.empty([0, 8, 3], device=self.tensor.device) + dims = self.dims corners_norm = torch.from_numpy( np.stack(np.unravel_index(np.arange(8), [2] * 3), axis=1)).to( @@ -136,8 +137,11 @@ def corners(self): corners_norm = corners_norm - dims.new_tensor([0.5, 1, 0.5]) corners = dims.view([-1, 1, 3]) * corners_norm.reshape([1, 8, 3]) - # rotate around y axis - corners = rotation_3d_in_axis(corners, self.tensor[:, 6], axis=1) + # positive direction of the gravity axis + # in cam coord system points to the earth + # so the rotation is clockwise if viewed from above + corners = rotation_3d_in_axis( + corners, self.tensor[:, 6], axis=self.YAW_AXIS, clockwise=True) corners += self.tensor[:, :3].view(-1, 1, 3) return corners @@ -145,7 +149,12 @@ def corners(self): def bev(self): """torch.Tensor: A n x 5 tensor of 2D BEV box of each box with rotation in XYWHR format.""" - return self.tensor[:, [0, 2, 3, 5, 6]] + bev = self.tensor[:, [0, 2, 3, 5, 6]].clone() + # positive direction of the gravity axis + # in cam coord system points to the earth + # so the bev yaw angle needs to be reversed + bev[:, -1] = -bev[:, -1] + return bev @property def nearest_bev(self): @@ -169,8 +178,8 @@ def nearest_bev(self): return bev_boxes def rotate(self, angle, points=None): - """Rotate boxes with points (optional) with the given angle or \ - rotation matrix. + """Rotate boxes with points (optional) with the given angle or rotation + matrix. Args: angle (float | torch.Tensor | np.ndarray): @@ -179,39 +188,43 @@ def rotate(self, angle, points=None): Points to rotate. Defaults to None. Returns: - tuple or None: When ``points`` is None, the function returns \ - None, otherwise it returns the rotated points and the \ + tuple or None: When ``points`` is None, the function returns + None, otherwise it returns the rotated points and the rotation matrix ``rot_mat_T``. """ if not isinstance(angle, torch.Tensor): angle = self.tensor.new_tensor(angle) + assert angle.shape == torch.Size([3, 3]) or angle.numel() == 1, \ f'invalid rotation angle shape {angle.shape}' if angle.numel() == 1: - rot_sin = torch.sin(angle) - rot_cos = torch.cos(angle) - rot_mat_T = self.tensor.new_tensor([[rot_cos, 0, -rot_sin], - [0, 1, 0], - [rot_sin, 0, rot_cos]]) + self.tensor[:, 0:3], rot_mat_T = rotation_3d_in_axis( + self.tensor[:, 0:3], + angle, + axis=self.YAW_AXIS, + return_mat=True, + # positive direction of the gravity axis + # in cam coord system points to the earth + # so the rotation is clockwise if viewed from above + clockwise=True) else: rot_mat_T = angle rot_sin = rot_mat_T[2, 0] rot_cos = rot_mat_T[0, 0] angle = np.arctan2(rot_sin, rot_cos) + self.tensor[:, 0:3] = self.tensor[:, 0:3] @ rot_mat_T - self.tensor[:, :3] = self.tensor[:, :3] @ rot_mat_T self.tensor[:, 6] += angle if points is not None: if isinstance(points, torch.Tensor): points[:, :3] = points[:, :3] @ rot_mat_T elif isinstance(points, np.ndarray): - rot_mat_T = rot_mat_T.numpy() + rot_mat_T = rot_mat_T.cpu().numpy() points[:, :3] = np.dot(points[:, :3], rot_mat_T) elif isinstance(points, BasePoints): - # clockwise - points.rotate(-angle) + points.rotate(rot_mat_T) else: raise ValueError return points, rot_mat_T @@ -263,7 +276,7 @@ def in_range_bev(self, box_range): polygon, we reduce the burden for simpler cases. Returns: - torch.Tensor: Indicating whether each box is inside \ + torch.Tensor: Indicating whether each box is inside the reference range. """ in_range_flags = ((self.tensor[:, 0] > box_range[0]) @@ -295,8 +308,8 @@ def height_overlaps(cls, boxes1, boxes2, mode='iou'): boxes2_top_height = boxes2.top_height.view(1, -1) boxes2_bottom_height = boxes2.bottom_height.view(1, -1) - # In camera coordinate system - # from up to down is the positive direction + # positive direction of the gravity axis + # in cam coord system points to the earth heighest_of_bottom = torch.min(boxes1_bottom_height, boxes2_bottom_height) lowest_of_top = torch.max(boxes1_top_height, boxes2_top_height) @@ -315,9 +328,50 @@ def convert_to(self, dst, rt_mat=None): to LiDAR. This requires a transformation matrix. Returns: - :obj:`BaseInstance3DBoxes`: \ + :obj:`BaseInstance3DBoxes`: The converted box of the same type in the ``dst`` mode. """ from .box_3d_mode import Box3DMode return Box3DMode.convert( box=self, src=Box3DMode.CAM, dst=dst, rt_mat=rt_mat) + + def points_in_boxes(self, points): + """Find the box which the points are in. + + Args: + points (torch.Tensor): Points in shape (N, 3). + + Returns: + torch.Tensor: The index of box where each point are in. + """ + from .coord_3d_mode import Coord3DMode + + points_lidar = Coord3DMode.convert(points, Coord3DMode.CAM, + Coord3DMode.LIDAR) + boxes_lidar = Coord3DMode.convert(self.tensor, Coord3DMode.CAM, + Coord3DMode.LIDAR) + + box_idx = super().points_in_boxes(self, points_lidar, boxes_lidar) + return box_idx + + def points_in_boxes_batch(self, points): + """Find points that are in boxes (CUDA). + + Args: + points (torch.Tensor): Points in shape [1, M, 3] or [M, 3], + 3 dimensions are [x, y, z] in LiDAR coordinate. + + Returns: + torch.Tensor: The index of boxes each point lies in with shape + of (B, M, T). + """ + from .coord_3d_mode import Coord3DMode + + points_lidar = Coord3DMode.convert(points, Coord3DMode.CAM, + Coord3DMode.LIDAR) + boxes_lidar = Coord3DMode.convert(self.tensor, Coord3DMode.CAM, + Coord3DMode.LIDAR) + + box_idx = super().points_in_boxes_batch(self, points_lidar, + boxes_lidar) + return box_idx diff --git a/mmdet3d/core/bbox/structures/coord_3d_mode.py b/mmdet3d/core/bbox/structures/coord_3d_mode.py index edd5f00d3d..b42726461c 100644 --- a/mmdet3d/core/bbox/structures/coord_3d_mode.py +++ b/mmdet3d/core/bbox/structures/coord_3d_mode.py @@ -2,12 +2,9 @@ import torch from enum import IntEnum, unique -from mmdet3d.core.points import (BasePoints, CameraPoints, DepthPoints, - LiDARPoints) +from ...points import BasePoints, CameraPoints, DepthPoints, LiDARPoints from .base_box3d import BaseInstance3DBoxes -from .cam_box3d import CameraInstance3DBoxes -from .depth_box3d import DepthInstance3DBoxes -from .lidar_box3d import LiDARInstance3DBoxes +from .box_3d_mode import Box3DMode @unique @@ -63,119 +60,73 @@ class Coord3DMode(IntEnum): DEPTH = 2 @staticmethod - def convert(input, src, dst, rt_mat=None): - """Convert boxes or points from `src` mode to `dst` mode.""" + def convert(input, src, dst, rt_mat=None, with_yaw=True, is_point=True): + """Convert boxes or points from `src` mode to `dst` mode. + + Args: + input (tuple | list | np.ndarray | torch.Tensor | + :obj:`BaseInstance3DBoxes` | :obj:`BasePoints`): + Can be a k-tuple, k-list or an Nxk array/tensor, where k = 7. + src (:obj:`Box3DMode` | :obj:`Coord3DMode`): The source mode. + dst (:obj:`Box3DMode` | :obj:`Coord3DMode`): The target mode. + rt_mat (np.ndarray | torch.Tensor): The rotation and translation + matrix between different coordinates. Defaults to None. + The conversion from `src` coordinates to `dst` coordinates + usually comes along the change of sensors, e.g., from camera + to LiDAR. This requires a transformation matrix. + with_yaw (bool): If `box` is an instance of + :obj:`BaseInstance3DBoxes`, whether or not it has a yaw angle. + Defaults to True. + is_point (bool): If `input` is neither an instance of + :obj:`BaseInstance3DBoxes` nor an instance of + :obj:`BasePoints`, whether or not it is point data. + Defaults to True. + + Returns: + (tuple | list | np.ndarray | torch.Tensor | + :obj:`BaseInstance3DBoxes` | :obj:`BasePoints`): + The converted box of the same type. + """ if isinstance(input, BaseInstance3DBoxes): - return Coord3DMode.convert_box(input, src, dst, rt_mat=rt_mat) + return Coord3DMode.convert_box( + input, src, dst, rt_mat=rt_mat, with_yaw=with_yaw) elif isinstance(input, BasePoints): return Coord3DMode.convert_point(input, src, dst, rt_mat=rt_mat) + elif isinstance(input, (tuple, list, np.ndarray, torch.Tensor)): + if is_point: + return Coord3DMode.convert_point( + input, src, dst, rt_mat=rt_mat) + else: + return Coord3DMode.convert_box( + input, src, dst, rt_mat=rt_mat, with_yaw=with_yaw) else: raise NotImplementedError @staticmethod - def convert_box(box, src, dst, rt_mat=None): + def convert_box(box, src, dst, rt_mat=None, with_yaw=True): """Convert boxes from `src` mode to `dst` mode. Args: box (tuple | list | np.ndarray | - torch.Tensor | BaseInstance3DBoxes): + torch.Tensor | :obj:`BaseInstance3DBoxes`): Can be a k-tuple, k-list or an Nxk array/tensor, where k = 7. - src (:obj:`CoordMode`): The src Box mode. - dst (:obj:`CoordMode`): The target Box mode. + src (:obj:`Box3DMode`): The src Box mode. + dst (:obj:`Box3DMode`): The target Box mode. rt_mat (np.ndarray | torch.Tensor): The rotation and translation matrix between different coordinates. Defaults to None. The conversion from `src` coordinates to `dst` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. + with_yaw (bool): If `box` is an instance of + :obj:`BaseInstance3DBoxes`, whether or not it has a yaw angle. + Defaults to True. Returns: - (tuple | list | np.ndarray | torch.Tensor | BaseInstance3DBoxes): \ + (tuple | list | np.ndarray | torch.Tensor | + :obj:`BaseInstance3DBoxes`): The converted box of the same type. """ - if src == dst: - return box - - is_numpy = isinstance(box, np.ndarray) - is_Instance3DBoxes = isinstance(box, BaseInstance3DBoxes) - single_box = isinstance(box, (list, tuple)) - if single_box: - assert len(box) >= 7, ( - 'CoordMode.convert takes either a k-tuple/list or ' - 'an Nxk array/tensor, where k >= 7') - arr = torch.tensor(box)[None, :] - else: - # avoid modifying the input box - if is_numpy: - arr = torch.from_numpy(np.asarray(box)).clone() - elif is_Instance3DBoxes: - arr = box.tensor.clone() - else: - arr = box.clone() - - # convert box from `src` mode to `dst` mode. - x_size, y_size, z_size = arr[..., 3:4], arr[..., 4:5], arr[..., 5:6] - if src == Coord3DMode.LIDAR and dst == Coord3DMode.CAM: - if rt_mat is None: - rt_mat = arr.new_tensor([[0, -1, 0], [0, 0, -1], [1, 0, 0]]) - xyz_size = torch.cat([y_size, z_size, x_size], dim=-1) - elif src == Coord3DMode.CAM and dst == Coord3DMode.LIDAR: - if rt_mat is None: - rt_mat = arr.new_tensor([[0, 0, 1], [-1, 0, 0], [0, -1, 0]]) - xyz_size = torch.cat([z_size, x_size, y_size], dim=-1) - elif src == Coord3DMode.DEPTH and dst == Coord3DMode.CAM: - if rt_mat is None: - rt_mat = arr.new_tensor([[1, 0, 0], [0, 0, 1], [0, -1, 0]]) - xyz_size = torch.cat([x_size, z_size, y_size], dim=-1) - elif src == Coord3DMode.CAM and dst == Coord3DMode.DEPTH: - if rt_mat is None: - rt_mat = arr.new_tensor([[1, 0, 0], [0, 0, -1], [0, 1, 0]]) - xyz_size = torch.cat([x_size, z_size, y_size], dim=-1) - elif src == Coord3DMode.LIDAR and dst == Coord3DMode.DEPTH: - if rt_mat is None: - rt_mat = arr.new_tensor([[0, -1, 0], [1, 0, 0], [0, 0, 1]]) - xyz_size = torch.cat([y_size, x_size, z_size], dim=-1) - elif src == Coord3DMode.DEPTH and dst == Coord3DMode.LIDAR: - if rt_mat is None: - rt_mat = arr.new_tensor([[0, 1, 0], [-1, 0, 0], [0, 0, 1]]) - xyz_size = torch.cat([y_size, x_size, z_size], dim=-1) - else: - raise NotImplementedError( - f'Conversion from Coord3DMode {src} to {dst} ' - 'is not supported yet') - - if not isinstance(rt_mat, torch.Tensor): - rt_mat = arr.new_tensor(rt_mat) - if rt_mat.size(1) == 4: - extended_xyz = torch.cat( - [arr[:, :3], arr.new_ones(arr.size(0), 1)], dim=-1) - xyz = extended_xyz @ rt_mat.t() - else: - xyz = arr[:, :3] @ rt_mat.t() - - remains = arr[..., 6:] - arr = torch.cat([xyz[:, :3], xyz_size, remains], dim=-1) - - # convert arr to the original type - original_type = type(box) - if single_box: - return original_type(arr.flatten().tolist()) - if is_numpy: - return arr.numpy() - elif is_Instance3DBoxes: - if dst == Coord3DMode.CAM: - target_type = CameraInstance3DBoxes - elif dst == Coord3DMode.LIDAR: - target_type = LiDARInstance3DBoxes - elif dst == Coord3DMode.DEPTH: - target_type = DepthInstance3DBoxes - else: - raise NotImplementedError( - f'Conversion to {dst} through {original_type}' - ' is not supported yet') - return target_type( - arr, box_dim=arr.size(-1), with_yaw=box.with_yaw) - else: - return arr + return Box3DMode.convert(box, src, dst, rt_mat=rt_mat) @staticmethod def convert_point(point, src, dst, rt_mat=None): @@ -183,7 +134,7 @@ def convert_point(point, src, dst, rt_mat=None): Args: point (tuple | list | np.ndarray | - torch.Tensor | BasePoints): + torch.Tensor | :obj:`BasePoints`): Can be a k-tuple, k-list or an Nxk array/tensor. src (:obj:`CoordMode`): The src Point mode. dst (:obj:`CoordMode`): The target Point mode. @@ -194,7 +145,7 @@ def convert_point(point, src, dst, rt_mat=None): to LiDAR. This requires a transformation matrix. Returns: - (tuple | list | np.ndarray | torch.Tensor | BasePoints): \ + (tuple | list | np.ndarray | torch.Tensor | :obj:`BasePoints`): The converted point of the same type. """ if src == dst: @@ -218,8 +169,6 @@ def convert_point(point, src, dst, rt_mat=None): arr = point.clone() # convert point from `src` mode to `dst` mode. - # TODO: LIDAR - # only implemented provided Rt matrix in cam-depth conversion if src == Coord3DMode.LIDAR and dst == Coord3DMode.CAM: if rt_mat is None: rt_mat = arr.new_tensor([[0, -1, 0], [0, 0, -1], [1, 0, 0]]) @@ -247,13 +196,13 @@ def convert_point(point, src, dst, rt_mat=None): rt_mat = arr.new_tensor(rt_mat) if rt_mat.size(1) == 4: extended_xyz = torch.cat( - [arr[:, :3], arr.new_ones(arr.size(0), 1)], dim=-1) + [arr[..., :3], arr.new_ones(arr.size(0), 1)], dim=-1) xyz = extended_xyz @ rt_mat.t() else: - xyz = arr[:, :3] @ rt_mat.t() + xyz = arr[..., :3] @ rt_mat.t() - remains = arr[:, 3:] - arr = torch.cat([xyz[:, :3], remains], dim=-1) + remains = arr[..., 3:] + arr = torch.cat([xyz[..., :3], remains], dim=-1) # convert arr to the original type original_type = type(point) diff --git a/mmdet3d/core/bbox/structures/depth_box3d.py b/mmdet3d/core/bbox/structures/depth_box3d.py index 73ed25e84d..d732f714ff 100644 --- a/mmdet3d/core/bbox/structures/depth_box3d.py +++ b/mmdet3d/core/bbox/structures/depth_box3d.py @@ -2,7 +2,6 @@ import torch from mmdet3d.core.points import BasePoints -from mmdet3d.ops import points_in_boxes_batch from .base_box3d import BaseInstance3DBoxes from .utils import limit_period, rotation_3d_in_axis @@ -37,6 +36,7 @@ class DepthInstance3DBoxes(BaseInstance3DBoxes): with_yaw (bool): If True, the value of yaw will be set to 0 as minmax boxes. """ + YAW_AXIS = 2 @property def gravity_center(self): @@ -66,7 +66,7 @@ def corners(self): / | / | (x0, y0, z1) + ----------- + + (x1, y1, z0) | / . | / - | / oriign | / + | / origin | / (x0, y0, z0) + ----------- + --------> right x (x1, y0, z0) """ @@ -84,7 +84,8 @@ def corners(self): corners = dims.view([-1, 1, 3]) * corners_norm.reshape([1, 8, 3]) # rotate around z axis - corners = rotation_3d_in_axis(corners, self.tensor[:, 6], axis=2) + corners = rotation_3d_in_axis( + corners, self.tensor[:, 6], axis=self.YAW_AXIS) corners += self.tensor[:, :3].view(-1, 1, 3) return corners @@ -116,8 +117,8 @@ def nearest_bev(self): return bev_boxes def rotate(self, angle, points=None): - """Rotate boxes with points (optional) with the given angle or \ - rotation matrix. + """Rotate boxes with points (optional) with the given angle or rotation + matrix. Args: angle (float | torch.Tensor | np.ndarray): @@ -126,30 +127,31 @@ def rotate(self, angle, points=None): Points to rotate. Defaults to None. Returns: - tuple or None: When ``points`` is None, the function returns \ - None, otherwise it returns the rotated points and the \ + tuple or None: When ``points`` is None, the function returns + None, otherwise it returns the rotated points and the rotation matrix ``rot_mat_T``. """ if not isinstance(angle, torch.Tensor): angle = self.tensor.new_tensor(angle) + assert angle.shape == torch.Size([3, 3]) or angle.numel() == 1, \ f'invalid rotation angle shape {angle.shape}' if angle.numel() == 1: - rot_sin = torch.sin(angle) - rot_cos = torch.cos(angle) - rot_mat_T = self.tensor.new_tensor([[rot_cos, -rot_sin, 0], - [rot_sin, rot_cos, 0], - [0, 0, 1]]).T + self.tensor[:, 0:3], rot_mat_T = rotation_3d_in_axis( + self.tensor[:, 0:3], + angle, + axis=self.YAW_AXIS, + return_mat=True) else: - rot_mat_T = angle.T + rot_mat_T = angle rot_sin = rot_mat_T[0, 1] rot_cos = rot_mat_T[0, 0] angle = np.arctan2(rot_sin, rot_cos) + self.tensor[:, 0:3] = self.tensor[:, 0:3] @ rot_mat_T - self.tensor[:, 0:3] = self.tensor[:, 0:3] @ rot_mat_T if self.with_yaw: - self.tensor[:, 6] -= angle + self.tensor[:, 6] += angle else: corners_rot = self.corners @ rot_mat_T new_x_size = corners_rot[..., 0].max( @@ -164,11 +166,10 @@ def rotate(self, angle, points=None): if isinstance(points, torch.Tensor): points[:, :3] = points[:, :3] @ rot_mat_T elif isinstance(points, np.ndarray): - rot_mat_T = rot_mat_T.numpy() + rot_mat_T = rot_mat_T.cpu().numpy() points[:, :3] = np.dot(points[:, :3], rot_mat_T) elif isinstance(points, BasePoints): - # anti-clockwise - points.rotate(angle) + points.rotate(rot_mat_T) else: raise ValueError return points, rot_mat_T @@ -220,7 +221,7 @@ def in_range_bev(self, box_range): polygon, we try to reduce the burdun for simpler cases. Returns: - torch.Tensor: Indicating whether each box is inside \ + torch.Tensor: Indicating whether each box is inside the reference range. """ in_range_flags = ((self.tensor[:, 0] > box_range[0]) @@ -241,41 +242,13 @@ def convert_to(self, dst, rt_mat=None): to LiDAR. This requires a transformation matrix. Returns: - :obj:`DepthInstance3DBoxes`: \ + :obj:`DepthInstance3DBoxes`: The converted box of the same type in the ``dst`` mode. """ from .box_3d_mode import Box3DMode return Box3DMode.convert( box=self, src=Box3DMode.DEPTH, dst=dst, rt_mat=rt_mat) - def points_in_boxes(self, points): - """Find points that are in boxes (CUDA). - - Args: - points (torch.Tensor): Points in shape [1, M, 3] or [M, 3], \ - 3 dimensions are [x, y, z] in LiDAR coordinate. - - Returns: - torch.Tensor: The index of boxes each point lies in with shape \ - of (B, M, T). - """ - from .box_3d_mode import Box3DMode - - # to lidar - points_lidar = points.clone() - points_lidar = points_lidar[..., [1, 0, 2]] - points_lidar[..., 1] *= -1 - if points.dim() == 2: - points_lidar = points_lidar.unsqueeze(0) - else: - assert points.dim() == 3 and points_lidar.shape[0] == 1 - - boxes_lidar = self.convert_to(Box3DMode.LIDAR).tensor - boxes_lidar = boxes_lidar.to(points.device).unsqueeze(0) - box_idxs_of_pts = points_in_boxes_batch(points_lidar, boxes_lidar) - - return box_idxs_of_pts.squeeze(0) - def enlarged_box(self, extra_width): """Enlarge the length, width and height boxes. @@ -330,13 +303,12 @@ def get_surface_line_center(self): -1, 3) surface_rot = rot_mat_T.repeat(6, 1, 1) - surface_3d = torch.matmul( - surface_3d.unsqueeze(-2), surface_rot.transpose(2, 1)).squeeze(-2) + surface_3d = torch.matmul(surface_3d.unsqueeze(-2), + surface_rot).squeeze(-2) surface_center = center.repeat(1, 6, 1).reshape(-1, 3) + surface_3d line_rot = rot_mat_T.repeat(12, 1, 1) - line_3d = torch.matmul( - line_3d.unsqueeze(-2), line_rot.transpose(2, 1)).squeeze(-2) + line_3d = torch.matmul(line_3d.unsqueeze(-2), line_rot).squeeze(-2) line_center = center.repeat(1, 12, 1).reshape(-1, 3) + line_3d return surface_center, line_center diff --git a/mmdet3d/core/bbox/structures/lidar_box3d.py b/mmdet3d/core/bbox/structures/lidar_box3d.py index 2150d61e2e..6adfa3ad0a 100644 --- a/mmdet3d/core/bbox/structures/lidar_box3d.py +++ b/mmdet3d/core/bbox/structures/lidar_box3d.py @@ -2,7 +2,6 @@ import torch from mmdet3d.core.points import BasePoints -from mmdet3d.ops.roiaware_pool3d import points_in_boxes_gpu from .base_box3d import BaseInstance3DBoxes from .utils import limit_period, rotation_3d_in_axis @@ -14,16 +13,16 @@ class LiDARInstance3DBoxes(BaseInstance3DBoxes): .. code-block:: none - up z x front (yaw=-0.5*pi) - ^ ^ - | / - | / - (yaw=-pi) left y <------ 0 -------- (yaw=0) + up z x front (yaw=0) + ^ ^ + | / + | / + (yaw=0.5*pi) left y <------ 0 The relative coordinate of bottom center in a LiDAR box is (0.5, 0.5, 0), and the yaw is around the z axis, thus the rotation axis=2. - The yaw is 0 at the negative direction of y axis, and decreases from - the negative direction of y to the positive direction of x. + The yaw is 0 at the positive direction of x axis, and increases from + the positive direction of x to the positive direction of y. A refactor is ongoing to make the three coordinate systems easier to understand and convert between each other. @@ -35,6 +34,7 @@ class LiDARInstance3DBoxes(BaseInstance3DBoxes): with_yaw (bool): If True, the value of yaw will be set to 0 as minmax boxes. """ + YAW_AXIS = 2 @property def gravity_center(self): @@ -64,7 +64,7 @@ def corners(self): / | / | (x0, y0, z1) + ----------- + + (x1, y1, z0) | / . | / - | / oriign | / + | / origin | / left y<-------- + ----------- + (x0, y1, z0) (x0, y0, z0) """ @@ -82,7 +82,8 @@ def corners(self): corners = dims.view([-1, 1, 3]) * corners_norm.reshape([1, 8, 3]) # rotate around z axis - corners = rotation_3d_in_axis(corners, self.tensor[:, 6], axis=2) + corners = rotation_3d_in_axis( + corners, self.tensor[:, 6], axis=self.YAW_AXIS) corners += self.tensor[:, :3].view(-1, 1, 3) return corners @@ -114,8 +115,8 @@ def nearest_bev(self): return bev_boxes def rotate(self, angle, points=None): - """Rotate boxes with points (optional) with the given angle or \ - rotation matrix. + """Rotate boxes with points (optional) with the given angle or rotation + matrix. Args: angles (float | torch.Tensor | np.ndarray): @@ -124,28 +125,29 @@ def rotate(self, angle, points=None): Points to rotate. Defaults to None. Returns: - tuple or None: When ``points`` is None, the function returns \ - None, otherwise it returns the rotated points and the \ + tuple or None: When ``points`` is None, the function returns + None, otherwise it returns the rotated points and the rotation matrix ``rot_mat_T``. """ if not isinstance(angle, torch.Tensor): angle = self.tensor.new_tensor(angle) + assert angle.shape == torch.Size([3, 3]) or angle.numel() == 1, \ f'invalid rotation angle shape {angle.shape}' if angle.numel() == 1: - rot_sin = torch.sin(angle) - rot_cos = torch.cos(angle) - rot_mat_T = self.tensor.new_tensor([[rot_cos, -rot_sin, 0], - [rot_sin, rot_cos, 0], - [0, 0, 1]]) + self.tensor[:, 0:3], rot_mat_T = rotation_3d_in_axis( + self.tensor[:, 0:3], + angle, + axis=self.YAW_AXIS, + return_mat=True) else: rot_mat_T = angle - rot_sin = rot_mat_T[1, 0] + rot_sin = rot_mat_T[0, 1] rot_cos = rot_mat_T[0, 0] angle = np.arctan2(rot_sin, rot_cos) + self.tensor[:, 0:3] = self.tensor[:, 0:3] @ rot_mat_T - self.tensor[:, :3] = self.tensor[:, :3] @ rot_mat_T self.tensor[:, 6] += angle if self.tensor.shape[1] == 9: @@ -156,11 +158,10 @@ def rotate(self, angle, points=None): if isinstance(points, torch.Tensor): points[:, :3] = points[:, :3] @ rot_mat_T elif isinstance(points, np.ndarray): - rot_mat_T = rot_mat_T.numpy() + rot_mat_T = rot_mat_T.cpu().numpy() points[:, :3] = np.dot(points[:, :3], rot_mat_T) elif isinstance(points, BasePoints): - # clockwise - points.rotate(-angle) + points.rotate(rot_mat_T) else: raise ValueError return points, rot_mat_T @@ -182,11 +183,11 @@ def flip(self, bev_direction='horizontal', points=None): if bev_direction == 'horizontal': self.tensor[:, 1::7] = -self.tensor[:, 1::7] if self.with_yaw: - self.tensor[:, 6] = -self.tensor[:, 6] + np.pi + self.tensor[:, 6] = -self.tensor[:, 6] elif bev_direction == 'vertical': self.tensor[:, 0::7] = -self.tensor[:, 0::7] if self.with_yaw: - self.tensor[:, 6] = -self.tensor[:, 6] + self.tensor[:, 6] = -self.tensor[:, 6] + np.pi if points is not None: assert isinstance(points, (torch.Tensor, np.ndarray, BasePoints)) @@ -232,7 +233,7 @@ def convert_to(self, dst, rt_mat=None): to LiDAR. This requires a transformation matrix. Returns: - :obj:`BaseInstance3DBoxes`: \ + :obj:`BaseInstance3DBoxes`: The converted box of the same type in the ``dst`` mode. """ from .box_3d_mode import Box3DMode @@ -253,17 +254,3 @@ def enlarged_box(self, extra_width): # bottom center z minus extra_width enlarged_boxes[:, 2] -= extra_width return self.new_box(enlarged_boxes) - - def points_in_boxes(self, points): - """Find the box which the points are in. - - Args: - points (torch.Tensor): Points in shape (N, 3). - - Returns: - torch.Tensor: The index of box where each point are in. - """ - box_idx = points_in_boxes_gpu( - points.unsqueeze(0), - self.tensor.unsqueeze(0).to(points.device)).squeeze(0) - return box_idx diff --git a/mmdet3d/core/bbox/structures/utils.py b/mmdet3d/core/bbox/structures/utils.py index fe63f9de11..ba37bc8842 100644 --- a/mmdet3d/core/bbox/structures/utils.py +++ b/mmdet3d/core/bbox/structures/utils.py @@ -2,84 +2,138 @@ import torch from logging import warning +from mmdet3d.core.utils import array_converter + +@array_converter(apply_to=('val', )) def limit_period(val, offset=0.5, period=np.pi): """Limit the value into a period for periodic function. Args: - val (torch.Tensor): The value to be converted. - offset (float, optional): Offset to set the value range. \ + val (torch.Tensor | np.ndarray): The value to be converted. + offset (float, optional): Offset to set the value range. Defaults to 0.5. period ([type], optional): Period of the value. Defaults to np.pi. Returns: - torch.Tensor: Value in the range of \ + (torch.Tensor | np.ndarray): Value in the range of [-offset * period, (1-offset) * period] """ - return val - torch.floor(val / period + offset) * period + limited_val = val - torch.floor(val / period + offset) * period + return limited_val -def rotation_3d_in_axis(points, angles, axis=0): +@array_converter(apply_to=('points', 'angles')) +def rotation_3d_in_axis(points, + angles, + axis=0, + return_mat=False, + clockwise=False): """Rotate points by angles according to axis. Args: - points (torch.Tensor): Points of shape (N, M, 3). - angles (torch.Tensor): Vector of angles in shape (N,) + points (np.ndarray | torch.Tensor | list | tuple ): + Points of shape (N, M, 3). + angles (np.ndarray | torch.Tensor | list | tuple | float): + Vector of angles in shape (N,) axis (int, optional): The axis to be rotated. Defaults to 0. + return_mat: Whether or not return the rotation matrix (transposed). + Defaults to False. + clockwise: Whether the rotation is clockwise. Defaults to False. Raises: - ValueError: when the axis is not in range [0, 1, 2], it will \ + ValueError: when the axis is not in range [0, 1, 2], it will raise value error. Returns: - torch.Tensor: Rotated points in shape (N, M, 3) + (torch.Tensor | np.ndarray): Rotated points in shape (N, M, 3). """ + batch_free = len(points.shape) == 2 + if batch_free: + points = points[None] + + if isinstance(angles, float) or len(angles.shape) == 0: + angles = torch.full(points.shape[:1], angles) + + assert len(points.shape) == 3 and len(angles.shape) == 1 \ + and points.shape[0] == angles.shape[0], f'Incorrect shape of points ' \ + f'angles: {points.shape}, {angles.shape}' + + assert points.shape[-1] in [2, 3], \ + f'Points size should be 2 or 3 instead of {points.shape[-1]}' + rot_sin = torch.sin(angles) rot_cos = torch.cos(angles) ones = torch.ones_like(rot_cos) zeros = torch.zeros_like(rot_cos) - if axis == 1: - rot_mat_T = torch.stack([ - torch.stack([rot_cos, zeros, -rot_sin]), - torch.stack([zeros, ones, zeros]), - torch.stack([rot_sin, zeros, rot_cos]) - ]) - elif axis == 2 or axis == -1: - rot_mat_T = torch.stack([ - torch.stack([rot_cos, -rot_sin, zeros]), - torch.stack([rot_sin, rot_cos, zeros]), - torch.stack([zeros, zeros, ones]) - ]) - elif axis == 0: + + if points.shape[-1] == 3: + if axis == 1 or axis == -2: + rot_mat_T = torch.stack([ + torch.stack([rot_cos, zeros, rot_sin]), + torch.stack([zeros, ones, zeros]), + torch.stack([-rot_sin, zeros, rot_cos]) + ]) + elif axis == 2 or axis == -1: + rot_mat_T = torch.stack([ + torch.stack([rot_cos, rot_sin, zeros]), + torch.stack([-rot_sin, rot_cos, zeros]), + torch.stack([zeros, zeros, ones]) + ]) + elif axis == 0 or axis == -3: + rot_mat_T = torch.stack([ + torch.stack([ones, zeros, zeros]), + torch.stack([zeros, rot_cos, rot_sin]), + torch.stack([zeros, -rot_sin, rot_cos]) + ]) + else: + raise ValueError(f'axis should in range ' + f'[-3, -2, -1, 0, 1, 2], got {axis}') + else: rot_mat_T = torch.stack([ - torch.stack([zeros, rot_cos, -rot_sin]), - torch.stack([zeros, rot_sin, rot_cos]), - torch.stack([ones, zeros, zeros]) + torch.stack([rot_cos, rot_sin]), + torch.stack([-rot_sin, rot_cos]) ]) + + if clockwise: + rot_mat_T = rot_mat_T.transpose(0, 1) + + if points.shape[0] == 0: + points_new = points else: - raise ValueError(f'axis should in range [0, 1, 2], got {axis}') + points_new = torch.einsum('aij,jka->aik', points, rot_mat_T) + + if batch_free: + points_new = points_new.squeeze(0) - return torch.einsum('aij,jka->aik', (points, rot_mat_T)) + if return_mat: + rot_mat_T = torch.einsum('jka->ajk', rot_mat_T) + if batch_free: + rot_mat_T = rot_mat_T.squeeze(0) + return points_new, rot_mat_T + else: + return points_new +@array_converter(apply_to=('boxes_xywhr', )) def xywhr2xyxyr(boxes_xywhr): """Convert a rotated boxes in XYWHR format to XYXYR format. Args: - boxes_xywhr (torch.Tensor): Rotated boxes in XYWHR format. + boxes_xywhr (torch.Tensor | np.ndarray): Rotated boxes in XYWHR format. Returns: - torch.Tensor: Converted boxes in XYXYR format. + (torch.Tensor | np.ndarray): Converted boxes in XYXYR format. """ boxes = torch.zeros_like(boxes_xywhr) - half_w = boxes_xywhr[:, 2] / 2 - half_h = boxes_xywhr[:, 3] / 2 - - boxes[:, 0] = boxes_xywhr[:, 0] - half_w - boxes[:, 1] = boxes_xywhr[:, 1] - half_h - boxes[:, 2] = boxes_xywhr[:, 0] + half_w - boxes[:, 3] = boxes_xywhr[:, 1] + half_h - boxes[:, 4] = boxes_xywhr[:, 4] + half_w = boxes_xywhr[..., 2] / 2 + half_h = boxes_xywhr[..., 3] / 2 + + boxes[..., 0] = boxes_xywhr[..., 0] - half_w + boxes[..., 1] = boxes_xywhr[..., 1] - half_h + boxes[..., 2] = boxes_xywhr[..., 0] + half_w + boxes[..., 3] = boxes_xywhr[..., 1] + half_h + boxes[..., 4] = boxes_xywhr[..., 4] return boxes @@ -90,6 +144,10 @@ def get_box_type(box_type): box_type (str): The type of box structure. The valid value are "LiDAR", "Camera", or "Depth". + Raises: + ValueError: A ValueError is raised when `box_type` + does not belong to the three valid types. + Returns: tuple: Box type and box mode. """ @@ -112,21 +170,24 @@ def get_box_type(box_type): return box_type_3d, box_mode_3d +@array_converter(apply_to=('points_3d', 'proj_mat')) def points_cam2img(points_3d, proj_mat, with_depth=False): - """Project points from camera coordicates to image coordinates. + """Project points in camera coordinates to image coordinates. Args: - points_3d (torch.Tensor): Points in shape (N, 3). - proj_mat (torch.Tensor): Transformation matrix between coordinates. + points_3d (torch.Tensor | np.ndarray): Points in shape (N, 3) + proj_mat (torch.Tensor | np.ndarray): + Transformation matrix between coordinates. with_depth (bool, optional): Whether to keep depth in the output. Defaults to False. Returns: - torch.Tensor: Points in image coordinates with shape [N, 2]. + (torch.Tensor | np.ndarray): Points in image coordinates, + with shape [N, 2] if `with_depth=False`, else [N, 3]. """ - points_num = list(points_3d.shape)[:-1] + points_shape = list(points_3d.shape) + points_shape[-1] = 1 - points_shape = np.concatenate([points_num, [1]], axis=0).tolist() assert len(proj_mat.shape) == 2, 'The dimension of the projection'\ f' matrix should be 2 instead of {len(proj_mat.shape)}.' d1, d2 = proj_mat.shape[:2] @@ -139,14 +200,15 @@ def points_cam2img(points_3d, proj_mat, with_depth=False): proj_mat_expanded[:d1, :d2] = proj_mat proj_mat = proj_mat_expanded - # previous implementation use new_zeros, new_one yeilds better results - points_4 = torch.cat( - [points_3d, points_3d.new_ones(*points_shape)], dim=-1) - point_2d = torch.matmul(points_4, proj_mat.t()) + # previous implementation use new_zeros, new_one yields better results + points_4 = torch.cat([points_3d, points_3d.new_ones(points_shape)], dim=-1) + + point_2d = points_4 @ proj_mat.T point_2d_res = point_2d[..., :2] / point_2d[..., 2:3] if with_depth: - return torch.cat([point_2d_res, point_2d[..., 2:3]], dim=-1) + point_2d_res = torch.cat([point_2d_res, point_2d[..., 2:3]], dim=-1) + return point_2d_res @@ -161,9 +223,9 @@ def mono_cam_box2vis(cam_box): After applying this function, we can project and draw it on 2D images. Args: - cam_box (:obj:`CameraInstance3DBoxes`): 3D bbox in camera coordinate \ - system before conversion. Could be gt bbox loaded from dataset or \ - network prediction output. + cam_box (:obj:`CameraInstance3DBoxes`): 3D bbox in camera coordinate + system before conversion. Could be gt bbox loaded from dataset + or network prediction output. Returns: :obj:`CameraInstance3DBoxes`: Box after conversion. diff --git a/mmdet3d/core/bbox/transforms.py b/mmdet3d/core/bbox/transforms.py index 686618e228..e5f5778b6a 100644 --- a/mmdet3d/core/bbox/transforms.py +++ b/mmdet3d/core/bbox/transforms.py @@ -31,7 +31,7 @@ def bbox3d2roi(bbox_list): corresponding to a batch of images. Returns: - torch.Tensor: Region of interests in shape (n, c), where \ + torch.Tensor: Region of interests in shape (n, c), where the channels are in order of [batch_ind, x, y ...]. """ rois_list = [] @@ -53,7 +53,7 @@ def bbox3d2result(bboxes, scores, labels, attrs=None): bboxes (torch.Tensor): Bounding boxes with shape of (n, 5). labels (torch.Tensor): Labels with shape of (n, ). scores (torch.Tensor): Scores with shape of (n, ). - attrs (torch.Tensor, optional): Attributes with shape of (n, ). \ + attrs (torch.Tensor, optional): Attributes with shape of (n, ). Defaults to None. Returns: diff --git a/mmdet3d/core/points/base_points.py b/mmdet3d/core/points/base_points.py index 95b754cb76..16fbe406ef 100644 --- a/mmdet3d/core/points/base_points.py +++ b/mmdet3d/core/points/base_points.py @@ -3,6 +3,8 @@ import warnings from abc import abstractmethod +from ..bbox.structures.utils import rotation_3d_in_axis + class BasePoints(object): """Base class for Points. @@ -140,7 +142,7 @@ def rotate(self, rotation, axis=None): """Rotate points with the given rotation matrix or angle. Args: - rotation (float, np.ndarray, torch.Tensor): Rotation matrix + rotation (float | np.ndarray | torch.Tensor): Rotation matrix or angle. axis (int): Axis to rotate at. Defaults to None. """ @@ -153,28 +155,14 @@ def rotate(self, rotation, axis=None): axis = self.rotation_axis if rotation.numel() == 1: - rot_sin = torch.sin(rotation) - rot_cos = torch.cos(rotation) - if axis == 1: - rot_mat_T = rotation.new_tensor([[rot_cos, 0, -rot_sin], - [0, 1, 0], - [rot_sin, 0, rot_cos]]) - elif axis == 2 or axis == -1: - rot_mat_T = rotation.new_tensor([[rot_cos, -rot_sin, 0], - [rot_sin, rot_cos, 0], - [0, 0, 1]]) - elif axis == 0: - rot_mat_T = rotation.new_tensor([[0, rot_cos, -rot_sin], - [0, rot_sin, rot_cos], - [1, 0, 0]]) - else: - raise ValueError('axis should in range') - rot_mat_T = rot_mat_T.T - elif rotation.numel() == 9: - rot_mat_T = rotation + rotated_points, rot_mat_T = rotation_3d_in_axis( + self.tensor[:, :3][None], rotation, axis=axis, return_mat=True) + self.tensor[:, :3] = rotated_points.squeeze(0) + rot_mat_T = rot_mat_T.squeeze(0) else: - raise NotImplementedError - self.tensor[:, :3] = self.tensor[:, :3] @ rot_mat_T + # rotation.numel() == 9 + self.tensor[:, :3] = self.tensor[:, :3] @ rotation + rot_mat_T = rotation return rot_mat_T diff --git a/mmdet3d/core/utils/__init__.py b/mmdet3d/core/utils/__init__.py index ad936667bd..7aa874543c 100644 --- a/mmdet3d/core/utils/__init__.py +++ b/mmdet3d/core/utils/__init__.py @@ -1,3 +1,7 @@ +from .array_converter import ArrayConverter, array_converter from .gaussian import draw_heatmap_gaussian, gaussian_2d, gaussian_radius -__all__ = ['gaussian_2d', 'gaussian_radius', 'draw_heatmap_gaussian'] +__all__ = [ + 'gaussian_2d', 'gaussian_radius', 'draw_heatmap_gaussian', + 'ArrayConverter', 'array_converter' +] diff --git a/mmdet3d/core/utils/array_converter.py b/mmdet3d/core/utils/array_converter.py new file mode 100644 index 0000000000..eeb3699973 --- /dev/null +++ b/mmdet3d/core/utils/array_converter.py @@ -0,0 +1,320 @@ +import functools +import numpy as np +import torch +from inspect import getfullargspec + + +def array_converter(to_torch=True, + apply_to=tuple(), + template_arg_name_=None, + recover=True): + """Wrapper function for data-type agnostic processing. + + First converts input arrays to PyTorch tensors or NumPy ndarrays + for middle calculation, then convert output to original data-type. + + Args: + to_torch (Bool): Whether convert to PyTorch tensors + for middle calculation. Defaults to True. + apply_to (tuple[str]): The arguments to which we apply data-type + conversion. Defaults to an empty tuple. + template_arg_name_ (str): Argument serving as the template ( + return arrays should have the same dtype and device + as the template). Defaults to None. If None, we will use the + first argument in `apply_to` as the template argument. + recover (Bool): Whether or not recover the wrapped function outputs + to the `template_arg_name_` type. Defaults to True. + + Raises: + ValueError: When template_arg_name_ is not among all args, or + when apply_to contains an arg which is not among all args, + a ValueError will be raised. When the template argument or + an argument to convert is a list or tuple, and cannot be + converted to a NumPy array, a ValueError will be raised. + TypeError: When the type of the template argument or + an argument to convert does not belong to the above range, + or the contents of such an list-or-tuple-type argument + do not share the same data type, a TypeError is raised. + + Returns: + (function): wrapped function. + + Example: + >>> import torch + >>> import numpy as np + >>> + >>> # Use torch addition for a + b, + >>> # and convert return values to the type of a + >>> @array_converter(apply_to=('a', 'b')) + >>> def simple_add(a, b): + >>> return a + b + >>> + >>> a = np.array([1.1]) + >>> b = np.array([2.2]) + >>> simple_add(a, b) + >>> + >>> # Use numpy addition for a + b, + >>> # and convert return values to the type of b + >>> @array_converter(to_torch=False, apply_to=('a', 'b'), + >>> template_arg_name_='b') + >>> def simple_add(a, b): + >>> return a + b + >>> + >>> simple_add() + >>> + >>> # Use torch funcs for floor(a) if flag=True else ceil(a), + >>> # and return the torch tensor + >>> @array_converter(apply_to=('a',), recover=False) + >>> def floor_or_ceil(a, flag=True): + >>> return torch.floor(a) if flag else torch.ceil(a) + >>> + >>> floor_or_ceil(a, flag=False) + """ + + def array_converter_wrapper(func): + """Outer wrapper for the function.""" + + @functools.wraps(func) + def new_func(*args, **kwargs): + """Inner wrapper for the arguments.""" + if len(apply_to) == 0: + return func(*args, **kwargs) + + func_name = func.__name__ + + arg_spec = getfullargspec(func) + + arg_names = arg_spec.args + arg_num = len(arg_names) + default_arg_values = arg_spec.defaults + if default_arg_values is None: + default_arg_values = [] + no_default_arg_num = len(arg_names) - len(default_arg_values) + + kwonly_arg_names = arg_spec.kwonlyargs + kwonly_default_arg_values = arg_spec.kwonlydefaults + if kwonly_default_arg_values is None: + kwonly_default_arg_values = {} + + all_arg_names = arg_names + kwonly_arg_names + + # in case there are args in the form of *args + if len(args) > arg_num: + named_args = args[:arg_num] + nameless_args = args[arg_num:] + else: + named_args = args + nameless_args = [] + + # template argument data type is used for all array-like arguments + if template_arg_name_ is None: + template_arg_name = apply_to[0] + else: + template_arg_name = template_arg_name_ + + if template_arg_name not in all_arg_names: + raise ValueError(f'{template_arg_name} is not among the ' + f'argument list of function {func_name}') + + # inspect apply_to + for arg_to_apply in apply_to: + if arg_to_apply not in all_arg_names: + raise ValueError(f'{arg_to_apply} is not ' + f'an argument of {func_name}') + + new_args = [] + new_kwargs = {} + + converter = ArrayConverter() + target_type = torch.Tensor if to_torch else np.ndarray + + # non-keyword arguments + for i, arg_value in enumerate(named_args): + if arg_names[i] in apply_to: + new_args.append( + converter.convert( + input_array=arg_value, target_type=target_type)) + else: + new_args.append(arg_value) + + if arg_names[i] == template_arg_name: + template_arg_value = arg_value + + kwonly_default_arg_values.update(kwargs) + kwargs = kwonly_default_arg_values + + # keyword arguments and non-keyword arguments using default value + for i in range(len(named_args), len(all_arg_names)): + arg_name = all_arg_names[i] + if arg_name in kwargs: + if arg_name in apply_to: + new_kwargs[arg_name] = converter.convert( + input_array=kwargs[arg_name], + target_type=target_type) + else: + new_kwargs[arg_name] = kwargs[arg_name] + else: + default_value = default_arg_values[i - no_default_arg_num] + if arg_name in apply_to: + new_kwargs[arg_name] = converter.convert( + input_array=default_value, target_type=target_type) + else: + new_kwargs[arg_name] = default_value + if arg_name == template_arg_name: + template_arg_value = kwargs[arg_name] + + # add nameless args provided by *args (if exists) + new_args += nameless_args + + return_values = func(*new_args, **new_kwargs) + converter.set_template(template_arg_value) + + def recursive_recover(input_data): + if isinstance(input_data, (tuple, list)): + new_data = [] + for item in input_data: + new_data.append(recursive_recover(item)) + return tuple(new_data) if isinstance(input_data, + tuple) else new_data + elif isinstance(input_data, dict): + new_data = {} + for k, v in input_data.items(): + new_data[k] = recursive_recover(v) + return new_data + elif isinstance(input_data, (torch.Tensor, np.ndarray)): + return converter.recover(input_data) + else: + return input_data + + if recover: + return recursive_recover(return_values) + else: + return return_values + + return new_func + + return array_converter_wrapper + + +class ArrayConverter: + + SUPPORTED_NON_ARRAY_TYPES = (int, float, np.int8, np.int16, np.int32, + np.int64, np.uint8, np.uint16, np.uint32, + np.uint64, np.float16, np.float32, np.float64) + + def __init__(self, template_array=None): + if template_array is not None: + self.set_template(template_array) + + def set_template(self, array): + """Set template array. + + Args: + array (tuple | list | int | float | np.ndarray | torch.Tensor): + Template array. + + Raises: + ValueError: If input is list or tuple and cannot be converted to + to a NumPy array, a ValueError is raised. + TypeError: If input type does not belong to the above range, + or the contents of a list or tuple do not share the + same data type, a TypeError is raised. + """ + self.array_type = type(array) + self.is_num = False + self.device = 'cpu' + + if isinstance(array, np.ndarray): + self.dtype = array.dtype + elif isinstance(array, torch.Tensor): + self.dtype = array.dtype + self.device = array.device + elif isinstance(array, (list, tuple)): + try: + array = np.array(array) + if array.dtype not in self.SUPPORTED_NON_ARRAY_TYPES: + raise TypeError + self.dtype = array.dtype + except (ValueError, TypeError): + print(f'The following list cannot be converted to' + f' a numpy array of supported dtype:\n{array}') + raise + elif isinstance(array, self.SUPPORTED_NON_ARRAY_TYPES): + self.array_type = np.ndarray + self.is_num = True + self.dtype = np.dtype(type(array)) + else: + raise TypeError(f'Template type {self.array_type}' + f' is not supported.') + + def convert(self, input_array, target_type=None, target_array=None): + """Convert input array to target data type. + + Args: + input_array (tuple | list | np.ndarray | + torch.Tensor | int | float ): + Input array. Defaults to None. + target_type ( | ): + Type to which input array is converted. Defaults to None. + target_array (np.ndarray | torch.Tensor): + Template array to which input array is converted. + Defaults to None. + + Raises: + ValueError: If input is list or tuple and cannot be converted to + to a NumPy array, a ValueError is raised. + TypeError: If input type does not belong to the above range, + or the contents of a list or tuple do not share the + same data type, a TypeError is raised. + """ + if isinstance(input_array, (list, tuple)): + try: + input_array = np.array(input_array) + if input_array.dtype not in self.SUPPORTED_NON_ARRAY_TYPES: + raise TypeError + except (ValueError, TypeError): + print(f'The input cannot be converted to' + f' a single-type numpy array:\n{input_array}') + raise + elif isinstance(input_array, self.SUPPORTED_NON_ARRAY_TYPES): + input_array = np.array(input_array) + array_type = type(input_array) + assert target_type is not None or target_array is not None, \ + 'must specify a target' + if target_type is not None: + assert target_type in (np.ndarray, torch.Tensor), \ + 'invalid target type' + if target_type == array_type: + return input_array + elif target_type == np.ndarray: + # default dtype is float32 + converted_array = input_array.cpu().numpy().astype(np.float32) + else: + # default dtype is float32, device is 'cpu' + converted_array = torch.tensor( + input_array, dtype=torch.float32) + else: + assert isinstance(target_array, (np.ndarray, torch.Tensor)), \ + 'invalid target array type' + if isinstance(target_array, array_type): + return input_array + elif isinstance(target_array, np.ndarray): + converted_array = input_array.cpu().numpy().astype( + target_array.dtype) + else: + converted_array = target_array.new_tensor(input_array) + return converted_array + + def recover(self, input_array): + assert isinstance(input_array, (np.ndarray, torch.Tensor)), \ + 'invalid input array type' + if isinstance(input_array, self.array_type): + return input_array + elif isinstance(input_array, torch.Tensor): + converted_array = input_array.cpu().numpy().astype(self.dtype) + else: + converted_array = torch.tensor( + input_array, dtype=self.dtype, device=self.device) + if self.is_num: + converted_array = converted_array.item() + return converted_array diff --git a/mmdet3d/core/visualizer/show_result.py b/mmdet3d/core/visualizer/show_result.py index 1792bfb0f6..20e46ca7d4 100644 --- a/mmdet3d/core/visualizer/show_result.py +++ b/mmdet3d/core/visualizer/show_result.py @@ -110,16 +110,14 @@ def show_result(points, if gt_bboxes is not None: # bottom center to gravity center gt_bboxes[..., 2] += gt_bboxes[..., 5] / 2 - # the positive direction for yaw in meshlab is clockwise - gt_bboxes[:, 6] *= -1 + _write_oriented_bbox(gt_bboxes, osp.join(result_path, f'{filename}_gt.obj')) if pred_bboxes is not None: # bottom center to gravity center pred_bboxes[..., 2] += pred_bboxes[..., 5] / 2 - # the positive direction for yaw in meshlab is clockwise - pred_bboxes[:, 6] *= -1 + _write_oriented_bbox(pred_bboxes, osp.join(result_path, f'{filename}_pred.obj')) diff --git a/mmdet3d/datasets/kitti_dataset.py b/mmdet3d/datasets/kitti_dataset.py index b6fd26a725..23dc207f89 100644 --- a/mmdet3d/datasets/kitti_dataset.py +++ b/mmdet3d/datasets/kitti_dataset.py @@ -616,8 +616,6 @@ def convert_valid_bboxes(self, box_dict, info): scores = box_dict['scores_3d'] labels = box_dict['labels_3d'] sample_idx = info['image']['image_idx'] - # TODO: remove the hack of yaw - box_preds.tensor[:, -1] = box_preds.tensor[:, -1] - np.pi box_preds.limit_yaw(offset=0.5, period=np.pi * 2) if len(box_preds) == 0: diff --git a/mmdet3d/datasets/lyft_dataset.py b/mmdet3d/datasets/lyft_dataset.py index a6d55632c2..d701fc6a98 100644 --- a/mmdet3d/datasets/lyft_dataset.py +++ b/mmdet3d/datasets/lyft_dataset.py @@ -516,16 +516,16 @@ def output_to_lyft_box(detection): box_gravity_center = box3d.gravity_center.numpy() box_dims = box3d.dims.numpy() box_yaw = box3d.yaw.numpy() - # TODO: check whether this is necessary - # with dir_offset & dir_limit in the head - box_yaw = -box_yaw - np.pi / 2 + + # our LiDAR coordinate system -> Lyft box coordinate system + lyft_box_dims = box_dims[:, [1, 0, 2]] box_list = [] for i in range(len(box3d)): quat = Quaternion(axis=[0, 0, 1], radians=box_yaw[i]) box = LyftBox( box_gravity_center[i], - box_dims[i], + lyft_box_dims[i], quat, label=labels[i], score=scores[i]) diff --git a/mmdet3d/datasets/nuscenes_dataset.py b/mmdet3d/datasets/nuscenes_dataset.py index a7d4b06375..4d07a576f2 100644 --- a/mmdet3d/datasets/nuscenes_dataset.py +++ b/mmdet3d/datasets/nuscenes_dataset.py @@ -587,9 +587,9 @@ def output_to_nusc_box(detection): box_gravity_center = box3d.gravity_center.numpy() box_dims = box3d.dims.numpy() box_yaw = box3d.yaw.numpy() - # TODO: check whether this is necessary - # with dir_offset & dir_limit in the head - box_yaw = -box_yaw - np.pi / 2 + + # our LiDAR coordinate system -> nuScenes box coordinate system + nus_box_dims = box_dims[:, [1, 0, 2]] box_list = [] for i in range(len(box3d)): @@ -601,7 +601,7 @@ def output_to_nusc_box(detection): # velo_val * np.cos(velo_ori), velo_val * np.sin(velo_ori), 0.0) box = NuScenesBox( box_gravity_center[i], - box_dims[i], + nus_box_dims[i], quat, label=labels[i], score=scores[i], diff --git a/mmdet3d/datasets/pipelines/data_augment_utils.py b/mmdet3d/datasets/pipelines/data_augment_utils.py index e0bc7165a1..2cb3925048 100644 --- a/mmdet3d/datasets/pipelines/data_augment_utils.py +++ b/mmdet3d/datasets/pipelines/data_augment_utils.py @@ -20,8 +20,8 @@ def _rotation_box2d_jit_(corners, angle, rot_mat_T): rot_sin = np.sin(angle) rot_cos = np.cos(angle) rot_mat_T[0, 0] = rot_cos - rot_mat_T[0, 1] = -rot_sin - rot_mat_T[1, 0] = rot_sin + rot_mat_T[0, 1] = rot_sin + rot_mat_T[1, 0] = -rot_sin rot_mat_T[1, 1] = rot_cos corners[:] = corners @ rot_mat_T @@ -210,8 +210,8 @@ def noise_per_box_v2_(boxes, valid_mask, loc_noises, rot_noises, rot_sin = np.sin(current_box[0, -1]) rot_cos = np.cos(current_box[0, -1]) rot_mat_T[0, 0] = rot_cos - rot_mat_T[0, 1] = -rot_sin - rot_mat_T[1, 0] = rot_sin + rot_mat_T[0, 1] = rot_sin + rot_mat_T[1, 0] = -rot_sin rot_mat_T[1, 1] = rot_cos current_corners[:] = current_box[ 0, 2:4] * corners_norm @ rot_mat_T + current_box[0, :2] @@ -263,18 +263,18 @@ def _rotation_matrix_3d_(rot_mat_T, angle, axis): rot_mat_T[:] = np.eye(3) if axis == 1: rot_mat_T[0, 0] = rot_cos - rot_mat_T[0, 2] = -rot_sin - rot_mat_T[2, 0] = rot_sin + rot_mat_T[0, 2] = rot_sin + rot_mat_T[2, 0] = -rot_sin rot_mat_T[2, 2] = rot_cos elif axis == 2 or axis == -1: rot_mat_T[0, 0] = rot_cos - rot_mat_T[0, 1] = -rot_sin - rot_mat_T[1, 0] = rot_sin + rot_mat_T[0, 1] = rot_sin + rot_mat_T[1, 0] = -rot_sin rot_mat_T[1, 1] = rot_cos elif axis == 0: rot_mat_T[1, 1] = rot_cos - rot_mat_T[1, 2] = -rot_sin - rot_mat_T[2, 1] = rot_sin + rot_mat_T[1, 2] = rot_sin + rot_mat_T[2, 1] = -rot_sin rot_mat_T[2, 2] = rot_cos diff --git a/mmdet3d/datasets/pipelines/formating.py b/mmdet3d/datasets/pipelines/formating.py index a36962cb84..e7eb7e9d2d 100644 --- a/mmdet3d/datasets/pipelines/formating.py +++ b/mmdet3d/datasets/pipelines/formating.py @@ -136,7 +136,8 @@ def __init__(self, 'scale_factor', 'flip', 'pcd_horizontal_flip', 'pcd_vertical_flip', 'box_mode_3d', 'box_type_3d', 'img_norm_cfg', 'pcd_trans', 'sample_idx', - 'pcd_scale_factor', 'pcd_rotation', 'pts_filename', + 'pcd_scale_factor', 'pcd_rotation', + 'pcd_rotation_angle', 'pts_filename', 'transformation_3d_flow')): self.keys = keys self.meta_keys = meta_keys diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index 29d9f4c3c2..bb82cd3351 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -393,7 +393,7 @@ def __call__(self, input_dict): gt_bboxes_3d = input_dict['gt_bboxes_3d'] points = input_dict['points'] - # TODO: check this inplace function + # TODO: this is inplace operation numpy_box = gt_bboxes_3d.tensor.numpy() numpy_points = points.tensor.numpy() @@ -588,6 +588,7 @@ def _rot_bbox_points(self, input_dict): if len(input_dict['bbox3d_fields']) == 0: rot_mat_T = input_dict['points'].rotate(noise_rotation) input_dict['pcd_rotation'] = rot_mat_T + input_dict['pcd_rotation_angle'] = noise_rotation return # rotate points with bboxes @@ -597,6 +598,7 @@ def _rot_bbox_points(self, input_dict): noise_rotation, input_dict['points']) input_dict['points'] = points input_dict['pcd_rotation'] = rot_mat_T + input_dict['pcd_rotation_angle'] = noise_rotation def _scale_bbox_points(self, input_dict): """Private function to scale bounding boxes and points. diff --git a/mmdet3d/datasets/waymo_dataset.py b/mmdet3d/datasets/waymo_dataset.py index 105f39c9ba..27eb3af720 100644 --- a/mmdet3d/datasets/waymo_dataset.py +++ b/mmdet3d/datasets/waymo_dataset.py @@ -493,7 +493,6 @@ def convert_valid_bboxes(self, box_dict, info): scores = box_dict['scores_3d'] labels = box_dict['labels_3d'] sample_idx = info['image']['image_idx'] - # TODO: remove the hack of yaw box_preds.limit_yaw(offset=0.5, period=np.pi * 2) if len(box_preds) == 0: diff --git a/mmdet3d/models/dense_heads/anchor3d_head.py b/mmdet3d/models/dense_heads/anchor3d_head.py index 59c79129a2..d5be4735c7 100644 --- a/mmdet3d/models/dense_heads/anchor3d_head.py +++ b/mmdet3d/models/dense_heads/anchor3d_head.py @@ -50,15 +50,15 @@ def __init__(self, type='Anchor3DRangeGenerator', range=[0, -39.68, -1.78, 69.12, 39.68, -1.78], strides=[2], - sizes=[[1.6, 3.9, 1.56]], + sizes=[[3.9, 1.6, 1.56]], rotations=[0, 1.57], custom_values=[], reshape_out=False), assigner_per_size=False, assign_per_class=False, diff_rad_by_sin=True, - dir_offset=0, - dir_limit_offset=1, + dir_offset=-np.pi / 2, + dir_limit_offset=0, bbox_coder=dict(type='DeltaXYZWLHRBBoxCoder'), loss_cls=dict( type='CrossEntropyLoss', diff --git a/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py b/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py index 357e370381..65c4e66e69 100644 --- a/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py +++ b/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py @@ -78,6 +78,7 @@ def __init__( use_direction_classifier=True, diff_rad_by_sin=True, dir_offset=0, + dir_limit_offset=0, loss_cls=dict( type='FocalLoss', use_sigmoid=True, diff --git a/mmdet3d/models/dense_heads/fcos_mono3d_head.py b/mmdet3d/models/dense_heads/fcos_mono3d_head.py index 36b15101d3..c7faf21f85 100644 --- a/mmdet3d/models/dense_heads/fcos_mono3d_head.py +++ b/mmdet3d/models/dense_heads/fcos_mono3d_head.py @@ -216,6 +216,7 @@ def add_sin_difference(boxes1, boxes2): @staticmethod def get_direction_target(reg_targets, dir_offset=0, + dir_limit_offset=0, num_bins=2, one_hot=True): """Encode direction to 0 ~ num_bins-1. @@ -230,7 +231,8 @@ def get_direction_target(reg_targets, torch.Tensor: Encoded direction targets. """ rot_gt = reg_targets[..., 6] - offset_rot = limit_period(rot_gt - dir_offset, 0, 2 * np.pi) + offset_rot = limit_period(rot_gt - dir_offset, dir_limit_offset, + 2 * np.pi) dir_cls_targets = torch.floor(offset_rot / (2 * np.pi / num_bins)).long() dir_cls_targets = torch.clamp(dir_cls_targets, min=0, max=num_bins - 1) @@ -376,7 +378,10 @@ def loss(self, if self.use_direction_classifier: pos_dir_cls_targets = self.get_direction_target( - pos_bbox_targets_3d, self.dir_offset, one_hot=False) + pos_bbox_targets_3d, + self.dir_offset, + self.dir_limit_offset, + one_hot=False) if self.diff_rad_by_sin: pos_bbox_preds, pos_bbox_targets_3d = self.add_sin_difference( diff --git a/mmdet3d/models/dense_heads/free_anchor3d_head.py b/mmdet3d/models/dense_heads/free_anchor3d_head.py index 76cad122ba..925220ec1c 100644 --- a/mmdet3d/models/dense_heads/free_anchor3d_head.py +++ b/mmdet3d/models/dense_heads/free_anchor3d_head.py @@ -194,6 +194,7 @@ def loss(self, matched_anchors, matched_object_targets, self.dir_offset, + self.dir_limit_offset, one_hot=False) loss_dir = self.loss_dir( dir_cls_preds_[matched].transpose(-2, -1), diff --git a/mmdet3d/models/dense_heads/groupfree3d_head.py b/mmdet3d/models/dense_heads/groupfree3d_head.py index c006e696b5..fda0f92745 100644 --- a/mmdet3d/models/dense_heads/groupfree3d_head.py +++ b/mmdet3d/models/dense_heads/groupfree3d_head.py @@ -950,7 +950,7 @@ def multiclass_nms_single(self, obj_scores, sem_scores, bbox, points, box_dim=bbox.shape[-1], with_yaw=self.bbox_coder.with_rot, origin=(0.5, 0.5, 0.5)) - box_indices = bbox.points_in_boxes(points) + box_indices = bbox.points_in_boxes_batch(points) corner3d = bbox.corners minmax_box3d = corner3d.new(torch.Size((corner3d.shape[0], 6))) diff --git a/mmdet3d/models/dense_heads/parta2_rpn_head.py b/mmdet3d/models/dense_heads/parta2_rpn_head.py index 24492aec60..418a161f51 100644 --- a/mmdet3d/models/dense_heads/parta2_rpn_head.py +++ b/mmdet3d/models/dense_heads/parta2_rpn_head.py @@ -59,15 +59,15 @@ def __init__(self, type='Anchor3DRangeGenerator', range=[0, -39.68, -1.78, 69.12, 39.68, -1.78], strides=[2], - sizes=[[1.6, 3.9, 1.56]], + sizes=[[3.9, 1.6, 1.56]], rotations=[0, 1.57], custom_values=[], reshape_out=False), assigner_per_size=False, assign_per_class=False, diff_rad_by_sin=True, - dir_offset=0, - dir_limit_offset=1, + dir_offset=-np.pi / 2, + dir_limit_offset=0, bbox_coder=dict(type='DeltaXYZWLHRBBoxCoder'), loss_cls=dict( type='CrossEntropyLoss', diff --git a/mmdet3d/models/dense_heads/ssd_3d_head.py b/mmdet3d/models/dense_heads/ssd_3d_head.py index aea2879391..c5feee107f 100644 --- a/mmdet3d/models/dense_heads/ssd_3d_head.py +++ b/mmdet3d/models/dense_heads/ssd_3d_head.py @@ -1,4 +1,3 @@ -import numpy as np import torch from mmcv.ops.nms import batched_nms from mmcv.runner import force_fp32 @@ -463,9 +462,7 @@ def get_bboxes(self, points, bbox_preds, input_metas, rescale=False): bbox_selected, score_selected, labels = self.multiclass_nms_single( obj_scores[b], sem_scores[b], bbox3d[b], points[b, ..., :3], input_metas[b]) - # fix the wrong direction - # To do: remove this ops - bbox_selected[..., 6] += np.pi + bbox = input_metas[b]['box_type_3d']( bbox_selected.clone(), box_dim=bbox_selected.shape[-1], @@ -488,23 +485,14 @@ def multiclass_nms_single(self, obj_scores, sem_scores, bbox, points, Returns: tuple[torch.Tensor]: Bounding boxes, scores and labels. """ - num_bbox = bbox.shape[0] bbox = input_meta['box_type_3d']( bbox.clone(), box_dim=bbox.shape[-1], with_yaw=self.bbox_coder.with_rot, origin=(0.5, 0.5, 0.5)) - if isinstance(bbox, LiDARInstance3DBoxes): - box_idx = bbox.points_in_boxes(points) - box_indices = box_idx.new_zeros([num_bbox + 1]) - box_idx[box_idx == -1] = num_bbox - box_indices.scatter_add_(0, box_idx.long(), - box_idx.new_ones(box_idx.shape)) - box_indices = box_indices[:-1] - nonempty_box_mask = box_indices >= 0 - elif isinstance(bbox, DepthInstance3DBoxes): - box_indices = bbox.points_in_boxes(points) + if isinstance(bbox, (LiDARInstance3DBoxes, DepthInstance3DBoxes)): + box_indices = bbox.points_in_boxes_batch(points) nonempty_box_mask = box_indices.T.sum(1) >= 0 else: raise NotImplementedError('Unsupported bbox type!') @@ -559,18 +547,8 @@ def _assign_targets_by_points_inside(self, bboxes_3d, points): tuple[torch.Tensor]: Flags indicating whether each point is inside bbox and the index of box where each point are in. """ - # TODO: align points_in_boxes function in each box_structures - num_bbox = bboxes_3d.tensor.shape[0] - if isinstance(bboxes_3d, LiDARInstance3DBoxes): - assignment = bboxes_3d.points_in_boxes(points).long() - points_mask = assignment.new_zeros( - [assignment.shape[0], num_bbox + 1]) - assignment[assignment == -1] = num_bbox - points_mask.scatter_(1, assignment.unsqueeze(1), 1) - points_mask = points_mask[:, :-1] - assignment[assignment == num_bbox] = num_bbox - 1 - elif isinstance(bboxes_3d, DepthInstance3DBoxes): - points_mask = bboxes_3d.points_in_boxes(points) + if isinstance(bboxes_3d, (LiDARInstance3DBoxes, DepthInstance3DBoxes)): + points_mask = bboxes_3d.points_in_boxes_batch(points) assignment = points_mask.argmax(dim=-1) else: raise NotImplementedError('Unsupported bbox type!') diff --git a/mmdet3d/models/dense_heads/train_mixins.py b/mmdet3d/models/dense_heads/train_mixins.py index f785a9dc06..21ce738477 100644 --- a/mmdet3d/models/dense_heads/train_mixins.py +++ b/mmdet3d/models/dense_heads/train_mixins.py @@ -292,6 +292,7 @@ def anchor_target_single_assigner(self, sampling_result.pos_bboxes, pos_bbox_targets, self.dir_offset, + self.dir_limit_offset, one_hot=False) bbox_targets[pos_inds, :] = pos_bbox_targets bbox_weights[pos_inds, :] = 1.0 @@ -317,6 +318,7 @@ def anchor_target_single_assigner(self, def get_direction_target(anchors, reg_targets, dir_offset=0, + dir_limit_offset=0, num_bins=2, one_hot=True): """Encode direction to 0 ~ num_bins-1. @@ -332,7 +334,7 @@ def get_direction_target(anchors, torch.Tensor: Encoded direction targets. """ rot_gt = reg_targets[..., 6] + anchors[..., 6] - offset_rot = limit_period(rot_gt - dir_offset, 0, 2 * np.pi) + offset_rot = limit_period(rot_gt - dir_offset, dir_limit_offset, 2 * np.pi) dir_cls_targets = torch.floor(offset_rot / (2 * np.pi / num_bins)).long() dir_cls_targets = torch.clamp(dir_cls_targets, min=0, max=num_bins - 1) if one_hot: diff --git a/mmdet3d/models/dense_heads/vote_head.py b/mmdet3d/models/dense_heads/vote_head.py index e03fe44413..13371b1bb3 100644 --- a/mmdet3d/models/dense_heads/vote_head.py +++ b/mmdet3d/models/dense_heads/vote_head.py @@ -470,7 +470,7 @@ def get_targets_single(self, vote_target_masks = points.new_zeros([num_points], dtype=torch.long) vote_target_idx = points.new_zeros([num_points], dtype=torch.long) - box_indices_all = gt_bboxes_3d.points_in_boxes(points) + box_indices_all = gt_bboxes_3d.points_in_boxes_batch(points) for i in range(gt_labels_3d.shape[0]): box_indices = box_indices_all[:, i] indices = torch.nonzero( @@ -620,7 +620,7 @@ def multiclass_nms_single(self, obj_scores, sem_scores, bbox, points, box_dim=bbox.shape[-1], with_yaw=self.bbox_coder.with_rot, origin=(0.5, 0.5, 0.5)) - box_indices = bbox.points_in_boxes(points) + box_indices = bbox.points_in_boxes_batch(points) corner3d = bbox.corners minmax_box3d = corner3d.new(torch.Size((corner3d.shape[0], 6))) diff --git a/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py b/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py index fc4bfc3b01..75d40af536 100644 --- a/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py +++ b/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py @@ -501,7 +501,7 @@ def multiclass_nms_single(self, obj_scores, sem_scores, bbox, points, box_dim=bbox.shape[-1], with_yaw=self.bbox_coder.with_rot, origin=(0.5, 0.5, 0.5)) - box_indices = bbox.points_in_boxes(points) + box_indices = bbox.points_in_boxes_batch(points) corner3d = bbox.corners minmax_box3d = corner3d.new(torch.Size((corner3d.shape[0], 6))) diff --git a/mmdet3d/models/roi_heads/bbox_heads/parta2_bbox_head.py b/mmdet3d/models/roi_heads/bbox_heads/parta2_bbox_head.py index 79d9cce95c..2c2a6c3067 100644 --- a/mmdet3d/models/roi_heads/bbox_heads/parta2_bbox_head.py +++ b/mmdet3d/models/roi_heads/bbox_heads/parta2_bbox_head.py @@ -343,7 +343,7 @@ def loss(self, cls_score, bbox_pred, rois, labels, bbox_targets, pred_boxes3d[..., 0:3] = rotation_3d_in_axis( pred_boxes3d[..., 0:3].unsqueeze(1), - (pos_rois_rotation + np.pi / 2), + pos_rois_rotation, axis=2).squeeze(1) pred_boxes3d[:, 0:3] += roi_xyz @@ -435,8 +435,7 @@ def _get_target_single(self, pos_bboxes, pos_gt_bboxes, ious, cfg): pos_gt_bboxes_ct[..., 0:3] -= roi_center pos_gt_bboxes_ct[..., 6] -= roi_ry pos_gt_bboxes_ct[..., 0:3] = rotation_3d_in_axis( - pos_gt_bboxes_ct[..., 0:3].unsqueeze(1), - -(roi_ry + np.pi / 2), + pos_gt_bboxes_ct[..., 0:3].unsqueeze(1), -roi_ry, axis=2).squeeze(1) # flip orientation if rois have opposite orientation @@ -529,8 +528,7 @@ def get_bboxes(self, local_roi_boxes[..., 0:3] = 0 rcnn_boxes3d = self.bbox_coder.decode(local_roi_boxes, bbox_pred) rcnn_boxes3d[..., 0:3] = rotation_3d_in_axis( - rcnn_boxes3d[..., 0:3].unsqueeze(1), (roi_ry + np.pi / 2), - axis=2).squeeze(1) + rcnn_boxes3d[..., 0:3].unsqueeze(1), roi_ry, axis=2).squeeze(1) rcnn_boxes3d[:, 0:3] += roi_xyz # post processing diff --git a/mmdet3d/models/roi_heads/mask_heads/primitive_head.py b/mmdet3d/models/roi_heads/mask_heads/primitive_head.py index 147219a28d..4ccc7ebbeb 100644 --- a/mmdet3d/models/roi_heads/mask_heads/primitive_head.py +++ b/mmdet3d/models/roi_heads/mask_heads/primitive_head.py @@ -354,7 +354,7 @@ def get_targets_single(self, # Generate pts_semantic_mask and pts_instance_mask when they are None if pts_semantic_mask is None or pts_instance_mask is None: - points2box_mask = gt_bboxes_3d.points_in_boxes(points) + points2box_mask = gt_bboxes_3d.points_in_boxes_batch(points) assignment = points2box_mask.argmax(1) background_mask = points2box_mask.max(1)[0] == 0 diff --git a/mmdet3d/ops/iou3d/src/iou3d_kernel.cu b/mmdet3d/ops/iou3d/src/iou3d_kernel.cu index 861aea3c5a..a993384780 100644 --- a/mmdet3d/ops/iou3d/src/iou3d_kernel.cu +++ b/mmdet3d/ops/iou3d/src/iou3d_kernel.cu @@ -61,9 +61,9 @@ __device__ inline int check_in_box2d(const float *box, const Point &p) { angle_sin = sin(-box[4]); // rotate the point in the opposite direction of box float rot_x = - (p.x - center_x) * angle_cos + (p.y - center_y) * angle_sin + center_x; + (p.x - center_x) * angle_cos - (p.y - center_y) * angle_sin + center_x; float rot_y = - -(p.x - center_x) * angle_sin + (p.y - center_y) * angle_cos + center_y; + (p.x - center_x) * angle_sin + (p.y - center_y) * angle_cos + center_y; #ifdef DEBUG printf("box: (%.3f, %.3f, %.3f, %.3f, %.3f)\n", box[0], box[1], box[2], box[3], box[4]); @@ -112,9 +112,9 @@ __device__ inline void rotate_around_center(const Point ¢er, const float angle_cos, const float angle_sin, Point &p) { float new_x = - (p.x - center.x) * angle_cos + (p.y - center.y) * angle_sin + center.x; + (p.x - center.x) * angle_cos - (p.y - center.y) * angle_sin + center.x; float new_y = - -(p.x - center.x) * angle_sin + (p.y - center.y) * angle_cos + center.y; + (p.x - center.x) * angle_sin + (p.y - center.y) * angle_cos + center.y; p.set(new_x, new_y); } diff --git a/mmdet3d/ops/roiaware_pool3d/points_in_boxes.py b/mmdet3d/ops/roiaware_pool3d/points_in_boxes.py index f576fedcc5..14e16b9926 100644 --- a/mmdet3d/ops/roiaware_pool3d/points_in_boxes.py +++ b/mmdet3d/ops/roiaware_pool3d/points_in_boxes.py @@ -7,17 +7,17 @@ def points_in_boxes_gpu(points, boxes): """Find points that are in boxes (CUDA) Args: - points (torch.Tensor): [B, M, 3], [x, y, z] in LiDAR coordinate + points (torch.Tensor): [B, M, 3], [x, y, z] in LiDAR/DEPTH coordinate boxes (torch.Tensor): [B, T, 7], - num_valid_boxes <= T, [x, y, z, w, l, h, ry] in LiDAR coordinate, - (x, y, z) is the bottom center + num_valid_boxes <= T, [x, y, z, dx, dy, dz, rz] in + LiDAR/DEPTH coordinate, (x, y, z) is the bottom center Returns: box_idxs_of_pts (torch.Tensor): (B, M), default background = -1 """ - assert boxes.shape[0] == points.shape[0], \ + assert points.shape[0] == boxes.shape[0], \ f'Points and boxes should have the same batch size, ' \ - f'got {boxes.shape[0]} and {boxes.shape[0]}' + f'got {points.shape[0]} and {boxes.shape[0]}' assert boxes.shape[2] == 7, \ f'boxes dimension should be 7, ' \ f'got unexpected shape {boxes.shape[2]}' @@ -53,31 +53,35 @@ def points_in_boxes_gpu(points, boxes): def points_in_boxes_cpu(points, boxes): """Find points that are in boxes (CPU) - Note: - Currently, the output of this function is different from that of - points_in_boxes_gpu. - Args: - points (torch.Tensor): [npoints, 3] - boxes (torch.Tensor): [N, 7], in LiDAR coordinate, - (x, y, z) is the bottom center + points (torch.Tensor): [B, M, 3], [x, y, z] in + LiDAR/DEPTH coordinate + boxes (torch.Tensor): [B, T, 7], + num_valid_boxes <= T, [x, y, z, dx, dy, dz, rz], + (x, y, z) is the bottom center. Returns: - point_indices (torch.Tensor): (N, npoints) + box_idxs_of_pts (torch.Tensor): (B, M, T), default background = 0 """ - # TODO: Refactor this function as a CPU version of points_in_boxes_gpu - assert boxes.shape[1] == 7, \ + assert points.shape[0] == boxes.shape[0], \ + f'Points and boxes should have the same batch size, ' \ + f'got {points.shape[0]} and {boxes.shape[0]}' + assert boxes.shape[2] == 7, \ f'boxes dimension should be 7, ' \ f'got unexpected shape {boxes.shape[2]}' - assert points.shape[1] == 3, \ + assert points.shape[2] == 3, \ f'points dimension should be 3, ' \ f'got unexpected shape {points.shape[2]}' + batch_size, num_points, _ = points.shape + num_boxes = boxes.shape[1] - point_indices = points.new_zeros((boxes.shape[0], points.shape[0]), + point_indices = points.new_zeros((batch_size, num_boxes, num_points), dtype=torch.int) - roiaware_pool3d_ext.points_in_boxes_cpu(boxes.float().contiguous(), - points.float().contiguous(), - point_indices) + for b in range(batch_size): + roiaware_pool3d_ext.points_in_boxes_cpu(boxes[b].float().contiguous(), + points[b].float().contiguous(), + point_indices[b]) + point_indices = point_indices.transpose(1, 2) return point_indices @@ -86,9 +90,9 @@ def points_in_boxes_batch(points, boxes): """Find points that are in boxes (CUDA) Args: - points (torch.Tensor): [B, M, 3], [x, y, z] in LiDAR coordinate + points (torch.Tensor): [B, M, 3], [x, y, z] in LiDAR/DEPTH coordinate boxes (torch.Tensor): [B, T, 7], - num_valid_boxes <= T, [x, y, z, w, l, h, ry] in LiDAR coordinate, + num_valid_boxes <= T, [x, y, z, dx, dy, dz, rz], (x, y, z) is the bottom center. Returns: diff --git a/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cpu.cpp b/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cpu.cpp index a26ffb62bb..7e5956b67e 100644 --- a/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cpu.cpp +++ b/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cpu.cpp @@ -15,9 +15,7 @@ inline void lidar_to_local_coords_cpu(float shift_x, float shift_y, float rz, float &local_x, float &local_y) { - // should rotate pi/2 + alpha to translate LiDAR to local - float rot_angle = rz + M_PI / 2; - float cosa = cos(rot_angle), sina = sin(rot_angle); + float cosa = cos(-rz), sina = sin(-rz); local_x = shift_x * cosa + shift_y * (-sina); local_y = shift_x * sina + shift_y * cosa; } @@ -29,13 +27,13 @@ inline int check_pt_in_box3d_cpu(const float *pt, const float *box3d, // bottom center float x = pt[0], y = pt[1], z = pt[2]; float cx = box3d[0], cy = box3d[1], cz = box3d[2]; - float w = box3d[3], l = box3d[4], h = box3d[5], rz = box3d[6]; - cz += h / 2.0; // shift to the center since cz in box3d is the bottom center + float dx = box3d[3], dy = box3d[4], dz = box3d[5], rz = box3d[6]; + cz += dz / 2.0; // shift to the center since cz in box3d is the bottom center - if (fabsf(z - cz) > h / 2.0) return 0; + if (fabsf(z - cz) > dz / 2.0) return 0; lidar_to_local_coords_cpu(x - cx, y - cy, rz, local_x, local_y); - float in_flag = (local_x > -l / 2.0) & (local_x < l / 2.0) & - (local_y > -w / 2.0) & (local_y < w / 2.0); + float in_flag = (local_x > -dx / 2.0) & (local_x < dx / 2.0) & + (local_y > -dy / 2.0) & (local_y < dy / 2.0); return in_flag; } diff --git a/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cuda.cu b/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cuda.cu index 896b316e69..4fed2002f1 100644 --- a/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cuda.cu +++ b/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cuda.cu @@ -24,9 +24,7 @@ __device__ inline void lidar_to_local_coords(float shift_x, float shift_y, float rz, float &local_x, float &local_y) { - // should rotate pi/2 + alpha to translate LiDAR to local - float rot_angle = rz + M_PI / 2; - float cosa = cos(rot_angle), sina = sin(rot_angle); + float cosa = cos(-rz), sina = sin(-rz); local_x = shift_x * cosa + shift_y * (-sina); local_y = shift_x * sina + shift_y * cosa; } @@ -38,13 +36,13 @@ __device__ inline int check_pt_in_box3d(const float *pt, const float *box3d, // bottom center float x = pt[0], y = pt[1], z = pt[2]; float cx = box3d[0], cy = box3d[1], cz = box3d[2]; - float w = box3d[3], l = box3d[4], h = box3d[5], rz = box3d[6]; - cz += h / 2.0; // shift to the center since cz in box3d is the bottom center + float dx = box3d[3], dy = box3d[4], dz = box3d[5], rz = box3d[6]; + cz += dz / 2.0; // shift to the center since cz in box3d is the bottom center - if (fabsf(z - cz) > h / 2.0) return 0; + if (fabsf(z - cz) > dz / 2.0) return 0; lidar_to_local_coords(x - cx, y - cy, rz, local_x, local_y); - float in_flag = (local_x > -l / 2.0) & (local_x < l / 2.0) & - (local_y > -w / 2.0) & (local_y < w / 2.0); + float in_flag = (local_x > -dx / 2.0) & (local_x < dx / 2.0) & + (local_y > -dy / 2.0) & (local_y < dy / 2.0); return in_flag; } @@ -52,7 +50,7 @@ __global__ void points_in_boxes_kernel(int batch_size, int boxes_num, int pts_num, const float *boxes, const float *pts, int *box_idx_of_points) { - // params boxes: (B, N, 7) [x, y, z, w, l, h, rz] in LiDAR coordinate, z is + // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is // the bottom center, each box DO NOT overlaps params pts: (B, npoints, 3) [x, // y, z] in LiDAR coordinate params boxes_idx_of_points: (B, npoints), default // -1 @@ -80,7 +78,7 @@ __global__ void points_in_boxes_batch_kernel(int batch_size, int boxes_num, int pts_num, const float *boxes, const float *pts, int *box_idx_of_points) { - // params boxes: (B, N, 7) [x, y, z, w, l, h, rz] in LiDAR coordinate, z is + // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is // the bottom center, each box DO NOT overlaps params pts: (B, npoints, 3) [x, // y, z] in LiDAR coordinate params boxes_idx_of_points: (B, npoints), default // -1 @@ -107,7 +105,7 @@ __global__ void points_in_boxes_batch_kernel(int batch_size, int boxes_num, void points_in_boxes_launcher(int batch_size, int boxes_num, int pts_num, const float *boxes, const float *pts, int *box_idx_of_points) { - // params boxes: (B, N, 7) [x, y, z, w, l, h, rz] in LiDAR coordinate, z is + // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is // the bottom center, each box DO NOT overlaps params pts: (B, npoints, 3) [x, // y, z] in LiDAR coordinate params boxes_idx_of_points: (B, npoints), default // -1 @@ -132,7 +130,7 @@ void points_in_boxes_launcher(int batch_size, int boxes_num, int pts_num, void points_in_boxes_batch_launcher(int batch_size, int boxes_num, int pts_num, const float *boxes, const float *pts, int *box_idx_of_points) { - // params boxes: (B, N, 7) [x, y, z, w, l, h, rz] in LiDAR coordinate, z is + // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is // the bottom center, each box params pts: (B, npoints, 3) [x, y, z] in // LiDAR coordinate params boxes_idx_of_points: (B, npoints), default -1 cudaError_t err; @@ -155,7 +153,7 @@ void points_in_boxes_batch_launcher(int batch_size, int boxes_num, int pts_num, int points_in_boxes_gpu(at::Tensor boxes_tensor, at::Tensor pts_tensor, at::Tensor box_idx_of_points_tensor) { - // params boxes: (B, N, 7) [x, y, z, w, l, h, rz] in LiDAR coordinate, z is + // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is // the bottom center, each box DO NOT overlaps params pts: (B, npoints, 3) [x, // y, z] in LiDAR coordinate params boxes_idx_of_points: (B, npoints), default // -1 @@ -180,7 +178,7 @@ int points_in_boxes_gpu(at::Tensor boxes_tensor, at::Tensor pts_tensor, int points_in_boxes_batch(at::Tensor boxes_tensor, at::Tensor pts_tensor, at::Tensor box_idx_of_points_tensor) { - // params boxes: (B, N, 7) [x, y, z, w, l, h, rz] in LiDAR coordinate, z is + // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is // the bottom center. params pts: (B, npoints, 3) [x, y, z] in LiDAR // coordinate params boxes_idx_of_points: (B, npoints), default -1 diff --git a/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d_kernel.cu b/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d_kernel.cu index 312b35dcbf..c1c948e96a 100644 --- a/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d_kernel.cu +++ b/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d_kernel.cu @@ -17,9 +17,7 @@ __device__ inline void lidar_to_local_coords(float shift_x, float shift_y, float rz, float &local_x, float &local_y) { - // should rotate pi/2 + alpha to translate LiDAR to local - float rot_angle = rz + M_PI / 2; - float cosa = cos(rot_angle), sina = sin(rot_angle); + float cosa = cos(-rz), sina = sin(-rz); local_x = shift_x * cosa + shift_y * (-sina); local_y = shift_x * sina + shift_y * cosa; } @@ -27,17 +25,17 @@ __device__ inline void lidar_to_local_coords(float shift_x, float shift_y, __device__ inline int check_pt_in_box3d(const float *pt, const float *box3d, float &local_x, float &local_y) { // param pt: (x, y, z) - // param box3d: (cx, cy, cz, w, l, h, rz) in LiDAR coordinate, cz in the + // param box3d: (cx, cy, cz, dx, dy, dz, rz) in LiDAR coordinate, cz in the // bottom center float x = pt[0], y = pt[1], z = pt[2]; float cx = box3d[0], cy = box3d[1], cz = box3d[2]; - float w = box3d[3], l = box3d[4], h = box3d[5], rz = box3d[6]; - cz += h / 2.0; // shift to the center since cz in box3d is the bottom center + float dx = box3d[3], dy = box3d[4], dz = box3d[5], rz = box3d[6]; + cz += dz / 2.0; // shift to the center since cz in box3d is the bottom center - if (fabsf(z - cz) > h / 2.0) return 0; + if (fabsf(z - cz) > dz / 2.0) return 0; lidar_to_local_coords(x - cx, y - cy, rz, local_x, local_y); - float in_flag = (local_x > -l / 2.0) & (local_x < l / 2.0) & - (local_y > -w / 2.0) & (local_y < w / 2.0); + float in_flag = (local_x > -dx / 2.0) & (local_x < dx / 2.0) & + (local_y > -dy / 2.0) & (local_y < dy / 2.0); return in_flag; } @@ -45,9 +43,9 @@ __global__ void generate_pts_mask_for_box3d(int boxes_num, int pts_num, int out_x, int out_y, int out_z, const float *rois, const float *pts, int *pts_mask) { - // params rois: (N, 7) [x, y, z, w, l, h, rz] in LiDAR coordinate + // params rois: (N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate // params pts: (npoints, 3) [x, y, z] - // params pts_mask: (N, npoints): -1 means point doesnot in this box, + // params pts_mask: (N, npoints): -1 means point does not in this box, // otherwise: encode (x_idxs, y_idxs, z_idxs) by binary bit int pt_idx = blockIdx.x * blockDim.x + threadIdx.x; int box_idx = blockIdx.y; @@ -63,14 +61,14 @@ __global__ void generate_pts_mask_for_box3d(int boxes_num, int pts_num, pts_mask[0] = -1; if (cur_in_flag > 0) { float local_z = pts[2] - rois[2]; - float w = rois[3], l = rois[4], h = rois[5]; + float dx = rois[3], dy = rois[4], dz = rois[5]; - float x_res = l / out_x; - float y_res = w / out_y; - float z_res = h / out_z; + float x_res = dx / out_x; + float y_res = dy / out_y; + float z_res = dz / out_z; - unsigned int x_idx = int((local_x + l / 2) / x_res); - unsigned int y_idx = int((local_y + w / 2) / y_res); + unsigned int x_idx = int((local_x + dx / 2) / x_res); + unsigned int y_idx = int((local_y + dy / 2) / y_res); unsigned int z_idx = int(local_z / z_res); x_idx = min(max(x_idx, 0), out_x - 1); @@ -231,7 +229,7 @@ void roiaware_pool3d_launcher(int boxes_num, int pts_num, int channels, const float *pts_feature, int *argmax, int *pts_idx_of_voxels, float *pooled_features, int pool_method) { - // params rois: (N, 7) [x, y, z, w, l, h, rz] in LiDAR coordinate + // params rois: (N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate // params pts: (npoints, 3) [x, y, z] in LiDAR coordinate // params pts_feature: (npoints, C) // params argmax: (N, out_x, out_y, out_z, C) diff --git a/tests/data/kitti/kitti_dbinfos_train.pkl b/tests/data/kitti/kitti_dbinfos_train.pkl index baa56c1a25e94befe70bcf99f231d6527bffa067..d9be817a13e4660fdfd72318934385436294e76a 100644 GIT binary patch delta 519 zcmYk2-)a*<6vlVANt0|^O>L}NwOaeftzg^O)K(2Gg7m_HTnvZ{rCoNDm|bW#`H~H^ z7lH^$nj)5xtM(N{LDcKO>p}BUHZ1t>TJ=PI6=k>i&bU>pr>9)fyvA|8iyIu% zkyyhHT-+R_)d%@oW{Tvznq93}UilEWnG)?&4We>}$}|l)^~1HYb>NllChiP8b>YR` zlHaa1j&z?|P5Vg4EK@qTM`Zbb2Xm4;&#}Yg98EYG*90dj*`ZF2un@*L}bZF~8fB0887;ITAMIxYhn d60yngq>HB<&&(t>tIc}5LGGZyl(yYk{sZ0!qBH;i delta 428 zcmZvY%S!@b6ve+8AKBPDdz(G2(OyO_B@1cOM=l0}*~BoDsRKHCbOthr+NiWC#8v$f zt=bE&+xJIAP(+JX^*Pe6ySVVXaL+lMcURrTUppFn%t;C58hZggAvV542c8;wB?u?* zUZPKworDlW&7Yw^%Vl}l!SHxHf`QIc9fJ~vMC!=8lgT6%sgrt43~RQ~$Vd&N5`4Bo zQ8lv|YsR>6bX@Lb8hT#MoGE(h921ntZg$C-Hcm41GSmv^J1Hfnr_>Urnh#v?YWmc$ z^2Lf^uv1B`2$-Ry3V(vrFk44J!kkDs+Ec|m%gHl|bQBBSn2QpY{$d{fu(1~DDT zg_8wlmD1B`{nW~t6^M3E9e+KFP}e^!VeNnabuH0pm|&xZO$l4Fhe4)Pu!^h=L@2S; G^5!qBjfLs} diff --git a/tests/data/lyft/lyft_infos.pkl b/tests/data/lyft/lyft_infos.pkl index 5fa2a41d135fa161668bd32ea9a9e8e2478c5c1c..84295d13414eca595c2ba41695a326f99ad515ba 100644 GIT binary patch delta 190 zcmZp%{BE%!gPEzbV{;C3peW;)$@?TUbPvrr+@f%3Ls5pX!{N!kNdgg;hxgeZo~y!k zsPWKL=fivL9S+T_aR7(3-<54{bcW^U(guf|ANchh`m`34<&SF^BpOHyz%pad?ir z1C#clcCo`V5BG68a39{!QFM65p($W(DUwS1ho>J}Twu#}c>3Yx_8x~9R*4*1=YDvl sy&}lcLlY0LJ-o|aOVjMoJfN07p~DT6H%lf!+$$=j$p~RNNSiYP05}3&i~s-t diff --git a/tests/data/lyft/lyft_infos_val.pkl b/tests/data/lyft/lyft_infos_val.pkl index 4ee7a7767dba1ec3e98f95fecaf4d7ae02594f2a..30e61d7606db37aefeda18091feafa59fda56019 100644 GIT binary patch delta 7150 zcmbtZd0bW3wZ0bx900+AfGC3C5EB#!2#WDsCqzLp#|bqA0nrNzx(^2oQF0MM88m`N z5eEbj2gCuK1@X3PrEQv~Y0@;!(6mW9ro-!do!)Ev`d;6+&&A92{oa2Mf39=R-fOSD z*0@_u-roVGE9W0TP+q#wwRFagXhjHJ+E-4 zh&U)BRWb39h*CxL)D&yVpp-!e#iX3V;)30!!?x`ot)cN^PN4R&(b+1m=94m*9nuGI<`_9_i;0~EmD@#)fwF5MJMeFwenpbv|r)c zrR3zMNz@$7_t=RE#PfDo6Qhdhx|oseNDqMmMY)AVyR=z5^R{MbnYxx$SS0PtTr-iz>c_bg^ za=NOBI5}y&Dqhe^1|-^p z>?7@??4#{tGQ@o2oxzdT7meQ!PIN4gnkcHUx!thP{;Y&@J5-4u77|3Kxf^x@q_zk_ z7dFrha?YJw>=l&|tMD6p`Q~V9Mx#QtQ0+EwK4tQA?KXctU68d*3CWCpbrIj21>i3D zh&|lOXOhS{L*a|t*y(VAxJcTwSQSfju{7K6hABxZw@eku{V*+e!8BRIv@DgX=^@pP zvRSB{Z}6@4e3I@0D6Z!kzF{YW9xGyn6r7@pR9&o;K&|Qr%Ca>}&ziBb)cC@Cfi=yT z;FB7#+7qIej8dOSt7@F_Np!4{TC7#YIyXe?T@dYt8tdrxdfrdRxJ5?WwX{tyt;6R% z7+V|652+bSRz;C>Hnsb5Pm1hPQt3EX4%f^PIg_Z8>k_!NMB#^t1ysRpd^Db%aa_j@ zv-uh~mN>{s75hS#g5M0d4br4^RczG7CJ(GOOD$hkMMghXuMC6~mHSZLHo7!tJg{mF z;#R63#q~0#n&>bcn^($}ipZ2=x2PgZ7ugc2oF_;bpLoAu^!Wx@w;J#GrUvAC!nDnp zJS5V(-AEpi=&(x#@>G%UhG~ZjrXs*Zja(aw&}rdAz)7J;E4Y!)@Kx#~=b$`#Ol@?O zx?rlxXR#t1*x(z~r))+%8Eff)Ig0Q4Q9IWzT2K2GuGq-87p~l~SJ|~Mlkc!I$Uz4H z`cb|_4Z$TYq;^V^3RF?3i(MW_?UsrPRcQT46}ga#l2H|x&o`-geUuEUFgiMJ9k6Q;e!2SX#R`;5N~ zO>~sW*2WCFganAm_A&iI&3jB69lsZ!e>2 z2UjTcz$^Au>M?og2=(sF8v%&R73$eYr>PpF+IK2+FefTDkT27IEV{B%hd*6#p|fAw zQK5Ef`2=g5;da;-SybMIIqc=!nG8^+|}sR2hlVX8OEheui)jJDy4j$={{Llwu} zFr9G0qyeUQKfW-5uF*l6m^!iiJF-iVb*ivZ?r&2n5y@ zKL3hqP+L-D6cTzDLTQN)T_9sSJ=&s_76Z4-`Dk_k@twQ*BHyC(F1Q+{T_;u1q>ED? zaGjQVo>9fwez=+k;G**Rg~;BQ=+yHzOJFje;g(YFmS}bHv7j;Zn2#&soD|!lidJ2m zmvFTWL{s1#Wb)XAxlc0r1>*z1)PQzR#4Z{!{*l&8Mw)-3MEf-o-p`ne=QV*8BlUif3`kJr_*FOVy-j{$i^23=>sezjmaa)SMql&w_xF-?2 zKY*CAJ79$Mfl(fi8t~8)rXJ(dfJkev@sEH+$0MmipDG@^VRE`)+GVc4cxtD+FHe_& z)s`GM6q)G`!@B3t{9r6!jYOJ4YUj3%lYQwSA6ED=x5c17I3D2c+cEfoS zHVxx;s#T`Fh|s*F@;#FA>FAJ)jk+vuhXs}H4FKUBp>y7<@wt52k+pQ_?V{aAhG!YV>y zm72yEQWnM8EU!Y?HSE=cxZIL7b_+JvS{X|}mLh+miqCcNg+%JhCrDw_i#Og58e#pZ z@o7+Mz|TBk`nfS@WTf>M#`=+oj$g{jf2E4A+%Wyx1yeDm=juehiisP5%?$;1bu6QE z+(}osft+ItvR*{)-Y>VU9dtArk#>)+B7<(wU$bCGLrtT^rSdVV<5Od(P6^}#fygEo zsSc&WT?jh@Y#jCZ66WbxAUzs)xN!Q7H0rmi_?<5P#RI3`OI80>75~hoFBAVU zHGL?)7cX3ynWUzzS}XoqCjMr;?;UOo3m>0d*Dn z4zL5eE?qRmkTYcL>WM0NA$~hO+&5)8J9jJEV5vpKLCxD#&<9l4d_f0*)`mzH*f7*& z83yV?TpKPs1`7RHu0y2Prg5WUu`F3ahlGQg6ciqlp~_uq;YxC9esT=v^EZVBfa=-^ z(Cl=7(*p&Wd4;-`S6GynqXkM{*9k)llJ8JrB%Z1^3e*kcXxU||u?ZsVNbYnwZY?#~ zEVJ<~|CWUE2^6BWsKJid%oY$2Mx0NYQQY3m;$|`;Lqg_pm!gf4VxeiUIpZ^+x)uVO zU7S7w!`;D=sV-lZyjBZESD7}}=#Gdp#srV`)56eT_s^H@MqKc+S=u<+5R{>f$8+j3 z<_L$UDLx%RBhh<>LQBZooqbUamU2B@yZf@bG?#>q^{1!QKKT(d#l$jp&l=8S7G z;e0B8K90{vtdW7z|L0p4I%OD2%Yh(~T>VU8blsl373oh#NT`gEI4KYs#hY`z0IF*V zhAkx2cP`$`w0UMQoCpaUk|;U-^XD25LQ;m#mmi3O7Y)DAXzK!FW@wC6F*5OPGb%$D zI2Ou&n6L;>Ra*?|HereEGMy_AA(?!HDkkF$(TpVQl*JtzySb`_TNLEpI=)3XMV}iC)j=_AZOkruDy0#khX}7;5xiF<_a;*V%W42ayx!pdw zbZd6#W)Gj|PoNXn6fRGs*u?dzmKXtds(YXhjR$$K0m zh9MX#6T_l~i&M0XXe-k;nL)52ENsYTd?N@p8n1<=41HOCAP6#ycf+EsuNZ#{i?L=J zk>jGRTa2V}3mjRp9|mRPscJc(ZiBYUE;9(^*#Yi3QQJ^x^+!P|fIo}Kx#zma7|-2-@JLcF_zD* z<{s=B^k@Se*n`@@678V7e1@7z%#~jthhVxwQ`Rm}UE2-%bO;E^g&8%Is|eJMm@d1_ zIoAQP(ZQ3cc~g!@4CHWi0gil^aYE={`oT}}QRkYFf;ET|x%3lc3=~U=P^rY6Y!9fe zy$XsLko(w^7|2XrnzU9cMMs(Tni&Y;;bBAe;v0cbYAg&-8M;q?AP~xo6XDU;*NvX= z7^}nh3h(8H|AYmO{jwisRp6;=2SD9sRmv`Rs(+PkpcK710p$X+LDFj|o(_>Q9k(M+ zY6-$hZ%tl`LjLhWc^|Pj3T3X7TWwsshHm3DX z@)nm^?%AHqjXN`N=crZ5>4CvPQ?o;$x>gP95d}4p1x0j2yLU6 z;-uQdw-rl7(MrA&gZo0KB>`dSIL0TD*<>a-BFA99H%wtiL3OPj^yx5YkX)GZn8{^; zy74+LyUa;5;3Eh>L%lv+lkO1-cHFr1(Ln&w<_d*z;QU{_8yPp6RXeaS+&jSuDH*CY znvX2e(zyOb9Kt&lg*$WO*Uw?3a3%5O*M0K~ z;rtWSn?_ftw`?aqI#M@rFCC=^>*YBHIWP#pSO_om3ewSa99Rn_pP2sCKI02tR^5=$W zeu&=V*zeo%yuyc~9gK5?c3qCad^b#CH$io+3-oE1cS|lzdCTOw4eEyKj_figl`B7L zsuxP^`WjJuo&#MOq&k zd!xn=?~(oTT3YKhE<{CJ9~o~&Epqh9*8WoVv3!TRPCQlZYoJoy!7086wXe%AQ(d|3 z=Z22MS+R%iEV5AxcL%3q&EAi|we&G+j*@phn{h?#+&h|XEaUUs6J*25?D{?>?iGt| z1~*6hpt?K3eTu$<9aY)S(KT-y_XKV$EStUBHj)qIjzqe+o(OTRD=-iOa3|jNl)Ob- zu=MID@)@hm{mS>16&e<6w_k~uZ3rKkryirurCgJ?6^sFVU4PGN1jQCT~sTmNe9>C;1V!!6fQTK{O4vX#^x4(qoJlReKXu z*S-OoZEn}w^~}Pog4`nQn`nCCqHoD}xaciBRqflLri-eyw`GUxmz;I8weR2yKjvuf zm~7uQ*#_C=k2xuRaOe9NMyNmGo@Nq!FPduWsjy=M-4A5_=AFB#3f z13wyNx_)Zetruw@NZ$Uy`GI_gRv+T2YCi<+*XkqL;nFHELHihAQmao)wogqqmsaMl zOFN(Z#-#lST~IDO-6m6<^Z)lx`wY^gRX;YT{Ryb9eQxZU{LJhxWKX{hU&?pb@KclP zXQ2Hy{2Z_OQdy}`gdrk=jq(Uz{P)Swet~8w`j>dB+OI%$?JLmi;w;mL1-W{bypGj= sjm9$VH=yahu&E?hE5h$o+HYmUaM>_(dhE;?{DhTPsQnI&Ma9$q2c75z&;S4c delta 5158 zcmaKwdq7NE`^WcGx~Wu%N;FE+QKnK#v_>g0Q?Vlv(tV23TJ0i?>7qouF6(eyPB`Rt zILR&7>_RW%AiOxorQ|MDayfGQtuLcM0srLVIV4K%y(rErP=M^sKau9ODEoCnM1q_K(tG7$(UFI!G9aUKVPWIGlaJC!7!D4MD-I0VZRUAE_N4`US?I0*tMg`F5@-E6x=O|; zjej@SdeCNIb^gF5_@%`O>`B=6^?p+T82kH>4xL|QVC#GRug~j(_&Wnitv-Bj0nnL) zgK!_cexqDuwz57FrV#!lBN-27p{p|ELg*$Nur#nYbJX|PjcvMOh7kuH`~-QX;=bkO z>fZqz!a-47mK?g{JY#=zFE5dYmyeIPkGES~N;>qwkMyKwo-zxWrL4QGhpgvxI24cU zA{2Pxm@a+_ZyIVC3w<==_{s(lJa6OP78~1Iz=0-vxP5Xt@9UeT!?*{Wjz>}o!lx{c&{o-UAv=KDkNEZ65 z&;~)EVjCkZK3>vY%;dg#8+rQ(&s@9sMutxVDCXcO$`(M`@Ok}if07 z8wdr#IKseBF@^>RVc}Q}&^RT~q(hN4YZ_X>tMI84Ce>8(bylBBiwhli&oYA!Nlhh? z{(P=0BL4URi!b>!W`j?DY57f8y?L)o^`5V8W%Kz9dj~a8APEPhT0o&RUKk6*)j;Ew zK)L!QNjrzyiJ7eyjjOs}VMcz6C>m=D3#V$>qLpmx>a5GRoNoc()VN!lURLqCDW|ustWfamz4NPY zyr}}woAJZ~;(u>aRu>SN2hL1s9~C-l0hqHe=G}LT0{-gL)mGV*Z5jusYq8Cs@nTpw zQ_U8uDwbDjOKGG7DVFWZK9d@F=9P8Ti+yzf#&Iy7vLz6+UF zCy0N=$=NC0%5%YqDI1H=l+6Z3@eU&MXLqu=c43Fq4BEg~=i?pxZD~Cy!yi9;!NJ<8vlbv$@)6^Kf5N zp&$o~P5l(}X{ZG(T&O{ltG-kwkNWK^Y5{w5U&Lj%)$)g%Yq$NAlg3Lo?e7x$widv9 z&mYbp{%yfRhp(-9V6^^roP25~$jREcE44MBpPljK?Qaxq5eF4oXt^|A9t-o;Xa%Y> zI3oSv(+z{gjCsi2hMYE@X}xaI^Yl#s7jv+Xq7^C8$o)d(2BD50U2vv7q5?O#eXQAB?K@ag1;^{>?17=4kqtSIw6+&7<}wqxzkrWqH( znse>BH>S0K$6n1gVQEW&%bvN8MKA|E{b%Fntb*nIwTJzKOeoq)4zAKdTTSDwVIfwd z{j5Y2|9axa_pKr^v;O|IUWY&O%&kxEYwGU+xR!(KDB3R+4evKM6RgLl&821=w1GC_ z-WEc^COq82Pw^`aP|Cv18lWwzW9VIWWSSePMToi4>BE=#pURHeycDkF>zbxG863Y1 zzBRdS{fqcrFV0GHt6Bt>4evMA`0+B}GP3O*`=OAp*eox-{*3@_<=}5xK-*}%?JO)) z1MN@(*$+R~wXn@Y%!p(jmvX-IjQ7+0+WCJ2xRZmsD9~;t&@M|e!5&;`DK-0D8)z>! z?=BSV!yeuJ6o1eFNr$7fdcu))I5RG@3g-6st zl`0@<_#;VxmzbGXyKmAV-D)P-X6CCpQlX<9JVt?z6Ci(cPhXM8P#<5>P}LXf1U}G1 zYIaha?G*0XQz)pyu08z}r)eynh1D9i8YSBWzn8VEYu%*|)sRlfmV((5IE^(+DAXUz{jDai)!>sB;eOI^vk-ReLuDBBqhp4DPI zN8{D9@ZT!7vGBZ-ZRWnB2Xp1a#7u~t*}6G~)r`)URrd~)VqM_i#kj1*C_VDz5FeYC z3G3wWl5><1=?0DpjR|JM!zRMZa`-3CvkL8UMP@3yjBi+l=wFptOo!L7u$NF!j{|$T z2(IIFa(x4@>NQ*eZ_+IO!@^q{>f1`{_AZMbh8;Hm7iFqqj!8-9P)d4c zN<3^-KhHd*8#b})U0vj8B_<|_<3=gy!V*p=LNt^Kk8+#r23U9R; zqyt~$E423px7!d^TWC_PENs&x^;W5BN1Mw4hvH^X-F;lkZ()rfc7%6-i_3ZZXYu`{ zF6-pH$D<_Q7mqXfgTrsFs32fKC>BkS~D-w!MqFXmx82j6L_`ku!7 zz`~DeRsT_`T0OpJ_*;FrG_JN z2S2or=+w|aZavWWS*M0-zwIBgkc8S?wYJ#Vts7PHJBJwjx{tMifIRKWkq&Nf^1=aj zwuZW-FEdRQ;6yu#ksj?>Oh;YtaXUwWK7L^5Dlot%eH{gc*uAg6!if6&EjR z)mb7F>Y`7SF@NSemECXVpS-=1QJdWWDodVaR~(H7!NIU%f8-qgr<6%UK2Dg$w`aD0 zeN1!>*k0;`vm5w#Fa3sYBo#Lw}Tn#)GYkWaZcEg6%eKFTr_*xMS~(O9Obws@z##HcUrkTUec9rlg_d)!axDiGpOp`)Nb zMnZpu1ND=f9ZAcg0mN!@cA_p?hO#ew@v=>!pwZ(_@GBy7&({fdjWG}`KenZRN>(!8 z9vqvsCJypdCz|qU8FKdiT0;M(*tMD8Eb$5j&aa;Sk|Rwy<1za@GwzJuH|PE9VDTLKuzBX`$l*YL&2;Z+<7P8o_5wzTi+clG{>qri{HG8^@N2AD&910*-Ej&H{fiw|EIHP z3bC3Dr&1RcUHmru+Q`;s0GboMBZoHdz;D_jgVc0>?aA=nC*`yGJ<_S4S00N5dpDky zwh_@^dmH9VAKCzJr+hb)l66b1ciALf06dB&J920mvD%_fr*TLMGf2y#7-D~jK9f3> zqKhOi^klLFhh+HMAfX@y+q>9xO{II%R|04j4smf5 z%*JUh0g5y_s(KTF(&;rxXbx#vltC;_NKax=gdla1gc=jA`|R&H93bA(W)+ot)V_L6 zsY7dC^lg6i>xIB;1|we3qkPQ64h!*+`;Pi~Yrlfrip-*)`&v|Ym?x#nJ;#eJ&n8-a ze(hJib=>NQ9S2$!e89{n*ZSWTXcnsaAHl8SQ_;o--Kq`$9FD$SkyKYlym(o@wO@5^ z?^5V4*Q# zt6ZzUowJ*;duHQ*C91FxC-^mlulVH)s4RclvFL<(H9~}yLz%?JDR0-wTugdwT4E;R zs0cr_%A(gqt8CJ;XfClztqRdR>QKF8n&gRcNKg5m3C&kdT%eq&E2Hn3BqkC)6Nw>* z7LviZrGX-RV6eMFPC6V~M2D0&F7om!W_C(KEQ-nCGLeFet5e9O*Cd5J(y}O@*dHkr zP=_jo6d$yh^i(MnDkl~xC#q8Tf3LdG65>&iMC|Dcu>bWz#bhRjmJ-{UB#Kohxr~gf zlPsavB+2EZWzh;^evJ+8e%A9nB=|ERH;O-`JcjZuQN;>+P3}$Az{U6LiictUn diff --git a/tests/data/lyft/sample_results.pkl b/tests/data/lyft/sample_results.pkl index c6509dfc4af3e591b8c777019698f9938265f8c8..132baa4848040b4aa588616759ab9f42545f54f0 100644 GIT binary patch literal 1764 zcmds2ZBHCk6y8@B2C!KBV$o`?EiSg*vb(TABFvPKnr17?YU-qhPG)Du&X{3`Gjo>~ zBXO6w1p2`mEG8{P8;G%`1|w|>4TMe3`D5PMrN)}p_*Kt`xpVJ*=Da-5xpzvTF$hdU zP;Lwa0u#(r@_FYvgr*@JuPyPJ;VEeyBHH-4Zm_hT&>YuD_^^b>+@i*cuHhwYOIKZm zFQ#?jqBVRjccI`hwV)a4;UV9J0MX7eG)zM*F2{H%j#PAO92&E+TAXVZGnFaz8Z^zx z4Sb&&1=ZI(|T+$5?Km8N6`?pPa*QF5ykwjnWP+OHul%Q#wVfp45i(xPjGw85n1I#`k%- zx4Upo;;vW1RKi8t?GfqM!7=eO+Mso@4S%LNtftU0joJVX~IxFZd`eYPevE zG_#+67v;;ih1JGETINny1YQCzQJJVTPo4zs? zIF=1o;H`PnC;vEuw{vi!M!l`(4ZBAD$*0th{@19#$Sk!yr{4qP0tsG|Cw%m%Sqptw2m^Ziy9TBx^$47#T4s&zdR2^JyxS zr5N`Eg9kokDaST-pNP$@Detn2J4&y)jO(sJv&BN_LWV@|B(zl^sc1%a95Sz^Md=3dUs>Q_0 zb~&`EaV;MD@Z3X?RO3M)VzwA8Ul^p;1LU3J;{5gHSBR9X6rc6w(%aLouIWMSauo%vhC*1QpI!*3j!#ro{8=*7iL zXuUUCV2eLA`cEy@b#e!t$=9&kw}^HJ-T3xo*TV6GpYC{X|IFpY_q{lA{|bI-kzvLC z{E&dad#DgLAkxwp2H27W(L77A>s) z1s(nI3*0_pDXES}lvGnJN-Crgnm;s-I^XnBb<0~QnD0aO#Y_3yo5%786>BiqcP2kKHJpca zRd}id-68ra^r(?4Jblax=|00@eF>gzvUT(_b((CSYq2f<7ufC+9sTfb?a!qDJ=&)D uPiR|!A3h2^P5WaG|EQt;eTEHH*jR#1E&R`i_&2xkZ{e`D1TO?kPyG%6ebhhz diff --git a/tests/data/sunrgbd/sunrgbd_infos.pkl b/tests/data/sunrgbd/sunrgbd_infos.pkl index 8f98f2fb19281c8ba25abff71626731cade6cd51..c637abb9e45c80ab269baa6e801fd310cb5534a6 100644 GIT binary patch delta 95 zcmV-l0HFV)5vdWdQU)g@0000QCd08$VIjl2!oL{9vBQZ@2 BA_)Kh diff --git a/tests/data/waymo/kitti_format/waymo_dbinfos_train.pkl b/tests/data/waymo/kitti_format/waymo_dbinfos_train.pkl index 1816e6a94fa29cd98cf1225e74ac2c2f29922045..a88b8703e46c4797298581ac2ddb65fc8e7d1877 100644 GIT binary patch delta 58 zcmV-A0LB031n2~?qya7}0000K!mh)=LBq7euolCBL&LQUK*OPW!-PN-!mbxU!;Qm? QOhCiE!t& NP~_15$(I= np.pi) + angle += 2 * np.pi * (angle < -np.pi) + scale = output['img_metas']._data['pcd_scale_factor'] + expected_tensor = torch.tensor( - [[20.6514, -8.8250, -1.0816, 1.5893, 3.0637, 1.5414, -1.9216], - [7.9374, 4.9457, -1.2008, 2.1829, 12.2357, 3.5041, 1.6629], - [20.8115, -2.0273, -1.8893, 2.2212, 14.1026, 3.4850, 2.6513], - [32.3850, -5.2135, -1.1321, 2.2212, 9.6124, 3.4562, 2.6498], - [43.7022, -7.8316, -0.5090, 2.2403, 12.2836, 3.4754, 2.0146], - [25.3300, -9.6670, -1.0855, 1.4074, 2.1350, 1.4170, -0.7141], - [16.5414, -29.0583, -0.9768, 1.4936, 3.3318, 1.3404, -0.7153], - [24.6548, -18.9226, -1.3567, 1.6659, 3.6094, 1.4170, 1.3970], - [45.8403, 1.8183, -1.1626, 1.6180, 3.9254, 1.3499, -0.6886], - [30.6288, -8.4497, -1.4881, 1.6659, 3.9158, 1.4265, -0.7241], - [32.3316, -22.4611, -1.3131, 1.5223, 4.0977, 1.3882, 2.4186], - [22.4492, 3.2944, -2.1674, 1.5510, 3.4084, 1.6372, 0.3928], - [37.3824, 5.0472, -0.6579, 1.5797, 3.3988, 1.7233, -1.4862], - [8.9259, -1.2578, -1.6081, 1.5223, 3.0350, 1.3308, -1.7212]]) + [[20.6514, -8.8250, -1.0816, 1.5893, 3.0637, 1.5414], + [7.9374, 4.9457, -1.2008, 2.1829, 12.2357, 3.5041], + [20.8115, -2.0273, -1.8893, 2.2212, 14.1026, 3.4850], + [32.3850, -5.2135, -1.1321, 2.2212, 9.6124, 3.4562], + [43.7022, -7.8316, -0.5090, 2.2403, 12.2836, 3.4754], + [25.3300, -9.6670, -1.0855, 1.4074, 2.1350, 1.4170], + [16.5414, -29.0583, -0.9768, 1.4936, 3.3318, 1.3404], + [24.6548, -18.9226, -1.3567, 1.6659, 3.6094, 1.4170], + [45.8403, 1.8183, -1.1626, 1.6180, 3.9254, 1.3499], + [30.6288, -8.4497, -1.4881, 1.6659, 3.9158, 1.4265], + [32.3316, -22.4611, -1.3131, 1.5223, 4.0977, 1.3882], + [22.4492, 3.2944, -2.1674, 1.5510, 3.4084, 1.6372], + [37.3824, 5.0472, -0.6579, 1.5797, 3.3988, 1.7233], + [8.9259, -1.2578, -1.6081, 1.5223, 3.0350, 1.3308]]) + + expected_tensor[:, :3] = (( + (origin_center + noise_trans) * torch.tensor([1, -1, 1])) + @ rotation_matrix) * scale + + expected_tensor = torch.cat([expected_tensor, angle.unsqueeze(-1)], dim=-1) assert torch.allclose( output['gt_bboxes_3d']._data.tensor, expected_tensor, atol=1e-3) @@ -207,6 +244,11 @@ def test_outdoor_velocity_aug_pipeline(): bbox3d_fields=[], img_fields=[]) + origin_center = gt_bboxes_3d.tensor[:, :3].clone() + origin_angle = gt_bboxes_3d.tensor[:, 6].clone( + ) # TODO: ObjectNoise modifies tensor!! + origin_velo = gt_bboxes_3d.tensor[:, 7:9].clone() + output = pipeline(results) expected_tensor = torch.tensor( @@ -246,5 +288,21 @@ def test_outdoor_velocity_aug_pipeline(): -4.4522e+00, -2.9166e+01, -7.8938e-01, 2.2841e+00, 3.8348e+00, 1.5925e+00, 1.4721e+00, -7.8371e-03, -8.1931e-03 ]]) + # coord sys refactor (manually go through pipeline) + rotation_angle = output['img_metas']._data['pcd_rotation_angle'] + rotation_matrix = output['img_metas']._data['pcd_rotation'] + expected_tensor[:, :3] = ((origin_center @ rotation_matrix) * + output['img_metas']._data['pcd_scale_factor'] * + torch.tensor([1, -1, 1]))[[ + 0, 1, 2, 3, 4, 5, 6, 7, 9 + ]] + angle = -origin_angle - rotation_angle + angle -= 2 * np.pi * (angle >= np.pi) + angle += 2 * np.pi * (angle < -np.pi) + expected_tensor[:, 6:7] = angle.unsqueeze(-1)[[0, 1, 2, 3, 4, 5, 6, 7, 9]] + expected_tensor[:, + 7:9] = ((origin_velo @ rotation_matrix[:2, :2]) * + output['img_metas']._data['pcd_scale_factor'] * + torch.tensor([1, -1]))[[0, 1, 2, 3, 4, 5, 6, 7, 9]] assert torch.allclose( output['gt_bboxes_3d']._data.tensor, expected_tensor, atol=1e-3) diff --git a/tests/test_models/test_common_modules/test_roiaware_pool3d.py b/tests/test_models/test_common_modules/test_roiaware_pool3d.py index 5b40decf86..c005be6a1f 100644 --- a/tests/test_models/test_common_modules/test_roiaware_pool3d.py +++ b/tests/test_models/test_common_modules/test_roiaware_pool3d.py @@ -1,3 +1,4 @@ +import numpy as np import pytest import torch @@ -15,8 +16,8 @@ def test_RoIAwarePool3d(): roiaware_pool3d_avg = RoIAwarePool3d( out_size=4, max_pts_per_voxel=128, mode='avg') rois = torch.tensor( - [[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 0.3], - [-10.0, 23.0, 16.0, 10, 20, 20, 0.5]], + [[1.0, 2.0, 3.0, 5.0, 4.0, 6.0, -0.3 - np.pi / 2], + [-10.0, 23.0, 16.0, 20.0, 10.0, 20.0, -0.5 - np.pi / 2]], dtype=torch.float32).cuda( ) # boxes (m, 7) with bottom center in lidar coordinate pts = torch.tensor( @@ -63,6 +64,17 @@ def test_points_in_boxes_gpu(): assert point_indices.shape == torch.Size([2, 8]) assert (point_indices == expected_point_indices).all() + boxes = torch.tensor([[[0.0, 0.0, 0.0, 1.0, 20.0, 1.0, 0.523598]]], + dtype=torch.float32).cuda() # 30 degrees + pts = torch.tensor( + [[[4, 6.928, 0], [6.928, 4, 0], [4, -6.928, 0], [6.928, -4, 0], + [-4, 6.928, 0], [-6.928, 4, 0], [-4, -6.928, 0], [-6.928, -4, 0]]], + dtype=torch.float32).cuda() + point_indices = points_in_boxes_gpu(points=pts, boxes=boxes) + expected_point_indices = torch.tensor([[-1, -1, 0, -1, 0, -1, -1, -1]], + dtype=torch.int32).cuda() + assert (point_indices == expected_point_indices).all() + if torch.cuda.device_count() > 1: pts = pts.to('cuda:1') boxes = boxes.to('cuda:1') @@ -74,23 +86,35 @@ def test_points_in_boxes_gpu(): def test_points_in_boxes_cpu(): boxes = torch.tensor( - [[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 0.3], - [-10.0, 23.0, 16.0, 10, 20, 20, 0.5]], + [[[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 0.3], + [-10.0, 23.0, 16.0, 10, 20, 20, 0.5]]], dtype=torch.float32 ) # boxes (m, 7) with bottom center in lidar coordinate pts = torch.tensor( - [[1, 2, 3.3], [1.2, 2.5, 3.0], [0.8, 2.1, 3.5], [1.6, 2.6, 3.6], - [0.8, 1.2, 3.9], [-9.2, 21.0, 18.2], [3.8, 7.9, 6.3], - [4.7, 3.5, -12.2], [3.8, 7.6, -2], [-10.6, -12.9, -20], [-16, -18, 9], - [-21.3, -52, -5], [0, 0, 0], [6, 7, 8], [-2, -3, -4]], + [[[1, 2, 3.3], [1.2, 2.5, 3.0], [0.8, 2.1, 3.5], [1.6, 2.6, 3.6], + [0.8, 1.2, 3.9], [-9.2, 21.0, 18.2], [3.8, 7.9, 6.3], + [4.7, 3.5, -12.2], [3.8, 7.6, -2], [-10.6, -12.9, -20], [ + -16, -18, 9 + ], [-21.3, -52, -5], [0, 0, 0], [6, 7, 8], [-2, -3, -4]]], dtype=torch.float32) # points (n, 3) in lidar coordinate point_indices = points_in_boxes_cpu(points=pts, boxes=boxes) expected_point_indices = torch.tensor( - [[1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]], + [[[1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 0], [0, 0], + [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]], dtype=torch.int32) - assert point_indices.shape == torch.Size([2, 15]) + assert point_indices.shape == torch.Size([1, 15, 2]) + assert (point_indices == expected_point_indices).all() + + boxes = torch.tensor([[[0.0, 0.0, 0.0, 1.0, 20.0, 1.0, 0.523598]]], + dtype=torch.float32) # 30 degrees + pts = torch.tensor( + [[[4, 6.928, 0], [6.928, 4, 0], [4, -6.928, 0], [6.928, -4, 0], + [-4, 6.928, 0], [-6.928, 4, 0], [-4, -6.928, 0], [-6.928, -4, 0]]], + dtype=torch.float32) + point_indices = points_in_boxes_cpu(points=pts, boxes=boxes) + expected_point_indices = torch.tensor( + [[[0], [0], [1], [0], [1], [0], [0], [0]]], dtype=torch.int32) assert (point_indices == expected_point_indices).all() diff --git a/tests/test_models/test_heads/test_parta2_bbox_head.py b/tests/test_models/test_heads/test_parta2_bbox_head.py index 7ba5e7a5d8..27174d30ea 100644 --- a/tests/test_models/test_heads/test_parta2_bbox_head.py +++ b/tests/test_models/test_heads/test_parta2_bbox_head.py @@ -75,7 +75,7 @@ def test_loss(): 2.0579e-02, 1.5005e-04, 3.5252e-05, 0.0000e+00, 2.0433e-05, 1.5422e-05 ]) expected_loss_bbox = torch.as_tensor(0.0622) - expected_loss_corner = torch.Tensor([0.1379]) + expected_loss_corner = torch.Tensor([0.1374]) assert torch.allclose(loss['loss_cls'], expected_loss_cls, 1e-3) assert torch.allclose(loss['loss_bbox'], expected_loss_bbox, 1e-3) @@ -200,7 +200,7 @@ def test_get_targets(): ]) expected_bbox_targets = torch.Tensor( - [[0.0805, 0.0130, 0.0047, 0.0542, -0.2252, 0.0299, -0.1495]]) + [[-0.0632, 0.0516, 0.0047, 0.0542, -0.2252, 0.0299, -0.1495]]) expected_pos_gt_bboxes = torch.Tensor( [[7.8417, -0.1405, -1.9652, 1.6122, 3.2838, 1.5331, -2.0835]]) @@ -344,12 +344,11 @@ def test_get_bboxes(): selected_bboxes, selected_scores, selected_label_preds = result_list[0] expected_selected_bboxes = torch.Tensor( - [[56.2170, 25.9074, -1.3610, 1.6025, 3.6730, 1.5128, -0.1179], - [54.6521, 28.8846, -1.9145, 1.6362, 4.0573, 1.5599, -1.7335], - [31.6179, -5.6004, -1.2470, 1.6458, 4.1622, 1.5632, -1.5734]]).cuda() + [[56.0888, 25.6445, -1.3610, 1.6025, 3.6730, 1.5128, -0.1179], + [54.4606, 29.2412, -1.9145, 1.6362, 4.0573, 1.5599, -1.7335], + [31.8887, -5.8574, -1.2470, 1.6458, 4.1622, 1.5632, -1.5734]]).cuda() expected_selected_scores = torch.Tensor([-2.2061, -2.1121, -0.1761]).cuda() expected_selected_label_preds = torch.Tensor([2., 2., 2.]).cuda() - assert torch.allclose(selected_bboxes.tensor, expected_selected_bboxes, 1e-3) assert torch.allclose(selected_scores, expected_selected_scores, 1e-3) @@ -386,43 +385,43 @@ def test_multi_class_nms(): box_preds = torch.Tensor( [[ 5.6217e+01, 2.5908e+01, -1.3611e+00, 1.6025e+00, 3.6730e+00, - 1.5129e+00, -1.1786e-01 + 1.5129e+00, 1.1786e-01 ], [ 5.4653e+01, 2.8885e+01, -1.9145e+00, 1.6362e+00, 4.0574e+00, - 1.5599e+00, -1.7335e+00 + 1.5599e+00, 1.7335e+00 ], [ 5.5809e+01, 2.5686e+01, -1.4457e+00, 1.5939e+00, 3.8270e+00, - 1.4997e+00, -2.9191e+00 + 1.4997e+00, 2.9191e+00 ], [ 5.6107e+01, 2.6082e+01, -1.3557e+00, 1.5782e+00, 3.7444e+00, - 1.5266e+00, 1.7707e-01 + 1.5266e+00, -1.7707e-01 ], [ 3.1618e+01, -5.6004e+00, -1.2470e+00, 1.6459e+00, 4.1622e+00, - 1.5632e+00, -1.5734e+00 + 1.5632e+00, 1.5734e+00 ], [ 3.1605e+01, -5.6342e+00, -1.2467e+00, 1.6474e+00, 4.1519e+00, - 1.5481e+00, -1.6313e+00 + 1.5481e+00, 1.6313e+00 ], [ 5.6211e+01, 2.7294e+01, -1.5350e+00, 1.5422e+00, 3.7733e+00, - 1.5140e+00, 9.5846e-02 + 1.5140e+00, -9.5846e-02 ], [ 5.5907e+01, 2.7155e+01, -1.4712e+00, 1.5416e+00, 3.7611e+00, - 1.5142e+00, -5.2059e-02 + 1.5142e+00, 5.2059e-02 ], [ 5.4000e+01, 3.0585e+01, -1.6874e+00, 1.6495e+00, 4.0376e+00, - 1.5554e+00, -1.7900e+00 + 1.5554e+00, 1.7900e+00 ], [ 5.6007e+01, 2.6300e+01, -1.3945e+00, 1.5716e+00, 3.7064e+00, - 1.4715e+00, -2.9639e+00 + 1.4715e+00, 2.9639e+00 ]]).cuda() input_meta = dict( diff --git a/tests/test_models/test_heads/test_roi_extractors.py b/tests/test_models/test_heads/test_roi_extractors.py index 703cae3ba6..1316aa3594 100644 --- a/tests/test_models/test_heads/test_roi_extractors.py +++ b/tests/test_models/test_heads/test_roi_extractors.py @@ -1,3 +1,4 @@ +import numpy as np import pytest import torch @@ -20,8 +21,8 @@ def test_single_roiaware_extractor(): dtype=torch.float32).cuda() coordinate = feats.clone() batch_inds = torch.zeros(feats.shape[0]).cuda() - rois = torch.tensor([[0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 0.3], - [0, -10.0, 23.0, 16.0, 10, 20, 20, 0.5]], + rois = torch.tensor([[0, 1.0, 2.0, 3.0, 5.0, 4.0, 6.0, -0.3 - np.pi / 2], + [0, -10.0, 23.0, 16.0, 20, 10, 20, -0.5 - np.pi / 2]], dtype=torch.float32).cuda() # test forward pooled_feats = self(feats, coordinate, batch_inds, rois) diff --git a/tests/test_models/test_heads/test_semantic_heads.py b/tests/test_models/test_heads/test_semantic_heads.py index b7ac069009..ac0e13f4a8 100644 --- a/tests/test_models/test_heads/test_semantic_heads.py +++ b/tests/test_models/test_heads/test_semantic_heads.py @@ -52,11 +52,11 @@ def test_PointwiseSemanticHead(): gt_bboxes = [ LiDARInstance3DBoxes( torch.tensor( - [[6.4118, -3.4305, -1.7291, 1.7033, 3.4693, 1.6197, -0.9091]], + [[6.4118, -3.4305, -1.7291, 1.7033, 3.4693, 1.6197, 0.9091]], dtype=torch.float32).cuda()), LiDARInstance3DBoxes( torch.tensor( - [[16.9107, 9.7925, -1.9201, 1.6097, 3.2786, 1.5307, -2.4056]], + [[16.9107, 9.7925, -1.9201, 1.6097, 3.2786, 1.5307, 2.4056]], dtype=torch.float32).cuda()) ] # batch size is 2 in the unit test diff --git a/tests/test_utils/test_anchors.py b/tests/test_utils/test_anchors.py index eb6172d6d7..e052e5ed86 100644 --- a/tests/test_utils/test_anchors.py +++ b/tests/test_utils/test_anchors.py @@ -21,7 +21,7 @@ def test_anchor_3d_range_generator(): [0, -39.68, -0.6, 70.4, 39.68, -0.6], [0, -39.68, -1.78, 70.4, 39.68, -1.78], ], - sizes=[[0.6, 0.8, 1.73], [0.6, 1.76, 1.73], [1.6, 3.9, 1.56]], + sizes=[[0.8, 0.6, 1.73], [1.76, 0.6, 1.73], [3.9, 1.6, 1.56]], rotations=[0, 1.57], reshape_out=False) @@ -31,8 +31,8 @@ def test_anchor_3d_range_generator(): '[[0, -39.68, -0.6, 70.4, 39.68, -0.6], ' \ '[0, -39.68, -0.6, 70.4, 39.68, -0.6], ' \ '[0, -39.68, -1.78, 70.4, 39.68, -1.78]],' \ - '\nscales=[1],\nsizes=[[0.6, 0.8, 1.73], ' \ - '[0.6, 1.76, 1.73], [1.6, 3.9, 1.56]],' \ + '\nscales=[1],\nsizes=[[0.8, 0.6, 1.73], ' \ + '[1.76, 0.6, 1.73], [3.9, 1.6, 1.56]],' \ '\nrotations=[0, 1.57],\nreshape_out=False,' \ '\nsize_per_range=True)' assert repr_str == expected_repr_str @@ -53,8 +53,8 @@ def test_aligned_anchor_generator(): ranges=[[-51.2, -51.2, -1.80, 51.2, 51.2, -1.80]], scales=[1, 2, 4], sizes=[ - [0.8660, 2.5981, 1.], # 1.5/sqrt(3) - [0.5774, 1.7321, 1.], # 1/sqrt(3) + [2.5981, 0.8660, 1.], # 1.5/sqrt(3) + [1.7321, 0.5774, 1.], # 1/sqrt(3) [1., 1., 1.], [0.4, 0.4, 1], ], @@ -70,7 +70,7 @@ def test_aligned_anchor_generator(): # check base anchors expected_grid_anchors = [ torch.tensor([[ - -51.0000, -51.0000, -1.8000, 0.8660, 2.5981, 1.0000, 0.0000, + -51.0000, -51.0000, -1.8000, 2.5981, 0.8660, 1.0000, 0.0000, 0.0000, 0.0000 ], [ @@ -90,20 +90,20 @@ def test_aligned_anchor_generator(): 0.0000, 0.0000, 0.0000 ], [ - -49.4000, -51.0000, -1.8000, 0.5774, 1.7321, 1.0000, + -49.4000, -51.0000, -1.8000, 1.7321, 0.5774, 1.0000, 1.5700, 0.0000, 0.0000 ], [ - -49.0000, -51.0000, -1.8000, 0.5774, 1.7321, 1.0000, + -49.0000, -51.0000, -1.8000, 1.7321, 0.5774, 1.0000, 0.0000, 0.0000, 0.0000 ], [ - -48.6000, -51.0000, -1.8000, 0.8660, 2.5981, 1.0000, + -48.6000, -51.0000, -1.8000, 2.5981, 0.8660, 1.0000, 1.5700, 0.0000, 0.0000 ]], device=device), torch.tensor([[ - -50.8000, -50.8000, -1.8000, 1.7320, 5.1962, 2.0000, 0.0000, + -50.8000, -50.8000, -1.8000, 5.1962, 1.7320, 2.0000, 0.0000, 0.0000, 0.0000 ], [ @@ -123,20 +123,20 @@ def test_aligned_anchor_generator(): 0.0000, 0.0000, 0.0000 ], [ - -47.6000, -50.8000, -1.8000, 1.1548, 3.4642, 2.0000, + -47.6000, -50.8000, -1.8000, 3.4642, 1.1548, 2.0000, 1.5700, 0.0000, 0.0000 ], [ - -46.8000, -50.8000, -1.8000, 1.1548, 3.4642, 2.0000, + -46.8000, -50.8000, -1.8000, 3.4642, 1.1548, 2.0000, 0.0000, 0.0000, 0.0000 ], [ - -46.0000, -50.8000, -1.8000, 1.7320, 5.1962, 2.0000, + -46.0000, -50.8000, -1.8000, 5.1962, 1.7320, 2.0000, 1.5700, 0.0000, 0.0000 ]], device=device), torch.tensor([[ - -50.4000, -50.4000, -1.8000, 3.4640, 10.3924, 4.0000, 0.0000, + -50.4000, -50.4000, -1.8000, 10.3924, 3.4640, 4.0000, 0.0000, 0.0000, 0.0000 ], [ @@ -156,15 +156,15 @@ def test_aligned_anchor_generator(): 0.0000, 0.0000, 0.0000 ], [ - -44.0000, -50.4000, -1.8000, 2.3096, 6.9284, 4.0000, + -44.0000, -50.4000, -1.8000, 6.9284, 2.3096, 4.0000, 1.5700, 0.0000, 0.0000 ], [ - -42.4000, -50.4000, -1.8000, 2.3096, 6.9284, 4.0000, + -42.4000, -50.4000, -1.8000, 6.9284, 2.3096, 4.0000, 0.0000, 0.0000, 0.0000 ], [ - -40.8000, -50.4000, -1.8000, 3.4640, 10.3924, 4.0000, + -40.8000, -50.4000, -1.8000, 10.3924, 3.4640, 4.0000, 1.5700, 0.0000, 0.0000 ]], device=device) @@ -193,7 +193,7 @@ def test_aligned_anchor_generator_per_cls(): type='AlignedAnchor3DRangeGeneratorPerCls', ranges=[[-100, -100, -1.80, 100, 100, -1.80], [-100, -100, -1.30, 100, 100, -1.30]], - sizes=[[0.63, 1.76, 1.44], [0.96, 2.35, 1.59]], + sizes=[[1.76, 0.63, 1.44], [2.35, 0.96, 1.59]], custom_values=[0, 0], rotations=[0, 1.57], reshape_out=False) @@ -204,20 +204,20 @@ def test_aligned_anchor_generator_per_cls(): # check base anchors expected_grid_anchors = [[ torch.tensor([[ - -99.0000, -99.0000, -1.8000, 0.6300, 1.7600, 1.4400, 0.0000, + -99.0000, -99.0000, -1.8000, 1.7600, 0.6300, 1.4400, 0.0000, 0.0000, 0.0000 ], [ - -99.0000, -99.0000, -1.8000, 0.6300, 1.7600, 1.4400, + -99.0000, -99.0000, -1.8000, 1.7600, 0.6300, 1.4400, 1.5700, 0.0000, 0.0000 ]], device=device), torch.tensor([[ - -98.0000, -98.0000, -1.3000, 0.9600, 2.3500, 1.5900, 0.0000, + -98.0000, -98.0000, -1.3000, 2.3500, 0.9600, 1.5900, 0.0000, 0.0000, 0.0000 ], [ - -98.0000, -98.0000, -1.3000, 0.9600, 2.3500, 1.5900, + -98.0000, -98.0000, -1.3000, 2.3500, 0.9600, 1.5900, 1.5700, 0.0000, 0.0000 ]], device=device) diff --git a/tests/test_utils/test_box3d.py b/tests/test_utils/test_box3d.py index 8bcf12b46b..810921c806 100644 --- a/tests/test_utils/test_box3d.py +++ b/tests/test_utils/test_box3d.py @@ -139,10 +139,15 @@ def test_lidar_boxes3d(): assert torch.allclose(expected_tensor, bottom_center_box.tensor) # Test init with numpy array - np_boxes = np.array( - [[1.7802081, 2.516249, -1.7501148, 1.75, 3.39, 1.65, 1.48], - [8.959413, 2.4567227, -1.6357126, 1.54, 4.01, 1.57, 1.62]], - dtype=np.float32) + np_boxes = np.array([[ + 1.7802081, 2.516249, -1.7501148, 1.75, 3.39, 1.65, + 1.48 - 0.13603681398218053 * 4 + ], + [ + 8.959413, 2.4567227, -1.6357126, 1.54, 4.01, 1.57, + 1.62 - 0.13603681398218053 * 4 + ]], + dtype=np.float32) boxes_1 = LiDARInstance3DBoxes(np_boxes) assert torch.allclose(boxes_1.tensor, torch.from_numpy(np_boxes)) @@ -156,15 +161,15 @@ def test_lidar_boxes3d(): th_boxes = torch.tensor( [[ 28.29669987, -0.5557558, -1.30332506, 1.47000003, 2.23000002, - 1.48000002, -1.57000005 + 1.48000002, -1.57000005 - 0.13603681398218053 * 4 ], [ 26.66901946, 21.82302134, -1.73605708, 1.55999994, 3.48000002, - 1.39999998, -1.69000006 + 1.39999998, -1.69000006 - 0.13603681398218053 * 4 ], [ 31.31977974, 8.16214412, -1.62177875, 1.74000001, 3.76999998, - 1.48000002, 2.78999996 + 1.48000002, 2.78999996 - 0.13603681398218053 * 4 ]], dtype=torch.float32) boxes_2 = LiDARInstance3DBoxes(th_boxes) @@ -175,12 +180,30 @@ def test_lidar_boxes3d(): boxes_1 = boxes_1.to(boxes_2.device) # test box concatenation - expected_tensor = torch.tensor( - [[1.7802081, 2.516249, -1.7501148, 1.75, 3.39, 1.65, 1.48], - [8.959413, 2.4567227, -1.6357126, 1.54, 4.01, 1.57, 1.62], - [28.2967, -0.5557558, -1.303325, 1.47, 2.23, 1.48, -1.57], - [26.66902, 21.82302, -1.736057, 1.56, 3.48, 1.4, -1.69], - [31.31978, 8.162144, -1.6217787, 1.74, 3.77, 1.48, 2.79]]) + expected_tensor = torch.tensor([[ + 1.7802081, 2.516249, -1.7501148, 1.75, 3.39, 1.65, + 1.48 - 0.13603681398218053 * 4 + ], + [ + 8.959413, 2.4567227, -1.6357126, 1.54, + 4.01, 1.57, + 1.62 - 0.13603681398218053 * 4 + ], + [ + 28.2967, -0.5557558, -1.303325, 1.47, + 2.23, 1.48, + -1.57 - 0.13603681398218053 * 4 + ], + [ + 26.66902, 21.82302, -1.736057, 1.56, + 3.48, 1.4, + -1.69 - 0.13603681398218053 * 4 + ], + [ + 31.31978, 8.162144, -1.6217787, 1.74, + 3.77, 1.48, + 2.79 - 0.13603681398218053 * 4 + ]]) boxes = LiDARInstance3DBoxes.cat([boxes_1, boxes_2]) assert torch.allclose(boxes.tensor, expected_tensor) # concatenate empty list @@ -195,11 +218,26 @@ def test_lidar_boxes3d(): [0.6533, -0.5520, -0.5265], [4.5870, 0.5358, -1.4741]]) expected_tensor = torch.tensor( - [[1.7802081, -2.516249, -1.7501148, 1.75, 3.39, 1.65, 1.6615927], - [8.959413, -2.4567227, -1.6357126, 1.54, 4.01, 1.57, 1.5215927], - [28.2967, 0.5557558, -1.303325, 1.47, 2.23, 1.48, 4.7115927], - [26.66902, -21.82302, -1.736057, 1.56, 3.48, 1.4, 4.8315926], - [31.31978, -8.162144, -1.6217787, 1.74, 3.77, 1.48, 0.35159278]]) + [[ + 1.7802081, -2.516249, -1.7501148, 1.75, 3.39, 1.65, + 1.6615927 - np.pi + 0.13603681398218053 * 4 + ], + [ + 8.959413, -2.4567227, -1.6357126, 1.54, 4.01, 1.57, + 1.5215927 - np.pi + 0.13603681398218053 * 4 + ], + [ + 28.2967, 0.5557558, -1.303325, 1.47, 2.23, 1.48, + 4.7115927 - np.pi + 0.13603681398218053 * 4 + ], + [ + 26.66902, -21.82302, -1.736057, 1.56, 3.48, 1.4, + 4.8315926 - np.pi + 0.13603681398218053 * 4 + ], + [ + 31.31978, -8.162144, -1.6217787, 1.74, 3.77, 1.48, + 0.35159278 - np.pi + 0.13603681398218053 * 4 + ]]) expected_points = torch.tensor([[1.2559, 0.6762, -1.4658], [4.7814, 0.8784, -1.3857], [6.7053, -0.2517, -0.9697], @@ -210,11 +248,26 @@ def test_lidar_boxes3d(): assert torch.allclose(points, expected_points, 1e-3) expected_tensor = torch.tensor( - [[-1.7802, -2.5162, -1.7501, 1.7500, 3.3900, 1.6500, -1.6616], - [-8.9594, -2.4567, -1.6357, 1.5400, 4.0100, 1.5700, -1.5216], - [-28.2967, 0.5558, -1.3033, 1.4700, 2.2300, 1.4800, -4.7116], - [-26.6690, -21.8230, -1.7361, 1.5600, 3.4800, 1.4000, -4.8316], - [-31.3198, -8.1621, -1.6218, 1.7400, 3.7700, 1.4800, -0.3516]]) + [[ + -1.7802, -2.5162, -1.7501, 1.7500, 3.3900, 1.6500, + -1.6616 + np.pi * 2 - 0.13603681398218053 * 4 + ], + [ + -8.9594, -2.4567, -1.6357, 1.5400, 4.0100, 1.5700, + -1.5216 + np.pi * 2 - 0.13603681398218053 * 4 + ], + [ + -28.2967, 0.5558, -1.3033, 1.4700, 2.2300, 1.4800, + -4.7116 + np.pi * 2 - 0.13603681398218053 * 4 + ], + [ + -26.6690, -21.8230, -1.7361, 1.5600, 3.4800, 1.4000, + -4.8316 + np.pi * 2 - 0.13603681398218053 * 4 + ], + [ + -31.3198, -8.1621, -1.6218, 1.7400, 3.7700, 1.4800, + -0.3516 + np.pi * 2 - 0.13603681398218053 * 4 + ]]) boxes_flip_vert = boxes.clone() points = boxes_flip_vert.flip('vertical', points) expected_points = torch.tensor([[-1.2559, 0.6762, -1.4658], @@ -228,12 +281,27 @@ def test_lidar_boxes3d(): # test box rotation # with input torch.Tensor points and angle expected_tensor = torch.tensor( - [[1.4225, -2.7344, -1.7501, 1.7500, 3.3900, 1.6500, 1.7976], - [8.5435, -3.6491, -1.6357, 1.5400, 4.0100, 1.5700, 1.6576], - [28.1106, -3.2869, -1.3033, 1.4700, 2.2300, 1.4800, 4.8476], - [23.4630, -25.2382, -1.7361, 1.5600, 3.4800, 1.4000, 4.9676], - [29.9235, -12.3342, -1.6218, 1.7400, 3.7700, 1.4800, 0.4876]]) - points, rot_mat_T = boxes.rotate(0.13603681398218053, points) + [[ + 1.4225, -2.7344, -1.7501, 1.7500, 3.3900, 1.6500, + 1.7976 - np.pi + 0.13603681398218053 * 2 + ], + [ + 8.5435, -3.6491, -1.6357, 1.5400, 4.0100, 1.5700, + 1.6576 - np.pi + 0.13603681398218053 * 2 + ], + [ + 28.1106, -3.2869, -1.3033, 1.4700, 2.2300, 1.4800, + 4.8476 - np.pi + 0.13603681398218053 * 2 + ], + [ + 23.4630, -25.2382, -1.7361, 1.5600, 3.4800, 1.4000, + 4.9676 - np.pi + 0.13603681398218053 * 2 + ], + [ + 29.9235, -12.3342, -1.6218, 1.7400, 3.7700, 1.4800, + 0.4876 - np.pi + 0.13603681398218053 * 2 + ]]) + points, rot_mat_T = boxes.rotate(-0.13603681398218053, points) expected_points = torch.tensor([[-1.1526, 0.8403, -1.4658], [-4.6181, 1.5187, -1.3857], [-6.6775, 0.6600, -0.9697], @@ -247,7 +315,7 @@ def test_lidar_boxes3d(): assert torch.allclose(rot_mat_T, expected_rot_mat_T, 1e-3) # with input torch.Tensor points and rotation matrix - points, rot_mat_T = boxes.rotate(-0.13603681398218053, points) # back + points, rot_mat_T = boxes.rotate(0.13603681398218053, points) # back rot_mat = np.array([[0.99076125, -0.13561762, 0.], [0.13561762, 0.99076125, 0.], [0., 0., 1.]]) points, rot_mat_T = boxes.rotate(rot_mat, points) @@ -261,7 +329,7 @@ def test_lidar_boxes3d(): [-6.5263, 1.5595, -0.9697], [-0.4809, 0.7073, -0.5265], [-4.5623, 0.7166, -1.4741]]) - points_np, rot_mat_T_np = boxes.rotate(0.13603681398218053, points_np) + points_np, rot_mat_T_np = boxes.rotate(-0.13603681398218053, points_np) expected_points_np = np.array([[-0.8844, 1.1191, -1.4658], [-4.0401, 2.7039, -1.3857], [-6.2545, 2.4302, -0.9697], @@ -275,7 +343,7 @@ def test_lidar_boxes3d(): assert np.allclose(rot_mat_T_np, expected_rot_mat_T_np, 1e-3) # with input LiDARPoints and rotation matrix - points_np, rot_mat_T_np = boxes.rotate(-0.13603681398218053, points_np) + points_np, rot_mat_T_np = boxes.rotate(0.13603681398218053, points_np) lidar_points = LiDARPoints(points_np) lidar_points, rot_mat_T_np = boxes.rotate(rot_mat, lidar_points) points_np = lidar_points.tensor.numpy() @@ -286,27 +354,27 @@ def test_lidar_boxes3d(): # test box scaling expected_tensor = torch.tensor([[ 1.0443488, -2.9183323, -1.7599131, 1.7597977, 3.4089797, 1.6592377, - 1.9336663 + 1.9336663 - np.pi ], [ 8.014273, -4.8007393, -1.6448704, 1.5486219, 4.0324507, 1.57879, - 1.7936664 + 1.7936664 - np.pi ], [ 27.558605, -7.1084175, -1.310622, 1.4782301, 2.242485, 1.488286, - 4.9836664 + 4.9836664 - np.pi ], [ 19.934517, -28.344835, -1.7457767, 1.5687338, 3.4994833, 1.4078381, - 5.1036663 + 5.1036663 - np.pi ], [ 28.130915, -16.369587, -1.6308585, 1.7497417, 3.791107, 1.488286, - 0.6236664 + 0.6236664 - np.pi ]]) boxes.scale(1.00559866335275) assert torch.allclose(boxes.tensor, expected_tensor) @@ -314,27 +382,27 @@ def test_lidar_boxes3d(): # test box translation expected_tensor = torch.tensor([[ 1.1281544, -3.0507944, -1.9169292, 1.7597977, 3.4089797, 1.6592377, - 1.9336663 + 1.9336663 - np.pi ], [ 8.098079, -4.9332013, -1.8018866, 1.5486219, 4.0324507, 1.57879, - 1.7936664 + 1.7936664 - np.pi ], [ 27.64241, -7.2408795, -1.4676381, 1.4782301, 2.242485, 1.488286, - 4.9836664 + 4.9836664 - np.pi ], [ 20.018322, -28.477297, -1.9027928, 1.5687338, 3.4994833, 1.4078381, - 5.1036663 + 5.1036663 - np.pi ], [ 28.21472, -16.502048, -1.7878747, 1.7497417, 3.791107, 1.488286, - 0.6236664 + 0.6236664 - np.pi ]]) boxes.translate([0.0838056, -0.13246193, -0.15701613]) assert torch.allclose(boxes.tensor, expected_tensor) @@ -355,17 +423,17 @@ def test_lidar_boxes3d(): index_boxes = boxes[2:5] expected_tensor = torch.tensor([[ 27.64241, -7.2408795, -1.4676381, 1.4782301, 2.242485, 1.488286, - 4.9836664 + 4.9836664 - np.pi ], [ 20.018322, -28.477297, -1.9027928, 1.5687338, 3.4994833, 1.4078381, - 5.1036663 + 5.1036663 - np.pi ], [ 28.21472, -16.502048, -1.7878747, 1.7497417, 3.791107, 1.488286, - 0.6236664 + 0.6236664 - np.pi ]]) assert len(index_boxes) == 3 assert torch.allclose(index_boxes.tensor, expected_tensor) @@ -373,7 +441,7 @@ def test_lidar_boxes3d(): index_boxes = boxes[2] expected_tensor = torch.tensor([[ 27.64241, -7.2408795, -1.4676381, 1.4782301, 2.242485, 1.488286, - 4.9836664 + 4.9836664 - np.pi ]]) assert len(index_boxes) == 1 assert torch.allclose(index_boxes.tensor, expected_tensor) @@ -381,12 +449,12 @@ def test_lidar_boxes3d(): index_boxes = boxes[[2, 4]] expected_tensor = torch.tensor([[ 27.64241, -7.2408795, -1.4676381, 1.4782301, 2.242485, 1.488286, - 4.9836664 + 4.9836664 - np.pi ], [ 28.21472, -16.502048, -1.7878747, 1.7497417, 3.791107, 1.488286, - 0.6236664 + 0.6236664 - np.pi ]]) assert len(index_boxes) == 2 assert torch.allclose(index_boxes.tensor, expected_tensor) @@ -407,13 +475,13 @@ def test_lidar_boxes3d(): assert (boxes.tensor[:, 6] >= -np.pi / 2).all() Box3DMode.convert(boxes, Box3DMode.LIDAR, Box3DMode.LIDAR) - expected_tesor = boxes.tensor.clone() - assert torch.allclose(expected_tesor, boxes.tensor) + expected_tensor = boxes.tensor.clone() + assert torch.allclose(expected_tensor, boxes.tensor) boxes.flip() boxes.flip() boxes.limit_yaw() - assert torch.allclose(expected_tesor, boxes.tensor) + assert torch.allclose(expected_tensor, boxes.tensor) # test nearest_bev expected_tensor = torch.tensor([[-0.5763, -3.9307, 2.8326, -2.1709], @@ -421,52 +489,50 @@ def test_lidar_boxes3d(): [26.5212, -7.9800, 28.7637, -6.5018], [18.2686, -29.2617, 21.7681, -27.6929], [27.3398, -18.3976, 29.0896, -14.6065]]) - # the pytorch print loses some precision assert torch.allclose( boxes.nearest_bev, expected_tensor, rtol=1e-4, atol=1e-7) - # obtained by the print of the original implementation - expected_tensor = torch.tensor([[[2.4093e+00, -4.4784e+00, -1.9169e+00], - [2.4093e+00, -4.4784e+00, -2.5769e-01], - [-7.7767e-01, -3.2684e+00, -2.5769e-01], - [-7.7767e-01, -3.2684e+00, -1.9169e+00], - [3.0340e+00, -2.8332e+00, -1.9169e+00], - [3.0340e+00, -2.8332e+00, -2.5769e-01], - [-1.5301e-01, -1.6232e+00, -2.5769e-01], - [-1.5301e-01, -1.6232e+00, -1.9169e+00]], - [[9.8933e+00, -6.1340e+00, -1.8019e+00], - [9.8933e+00, -6.1340e+00, -2.2310e-01], - [5.9606e+00, -5.2427e+00, -2.2310e-01], - [5.9606e+00, -5.2427e+00, -1.8019e+00], - [1.0236e+01, -4.6237e+00, -1.8019e+00], - [1.0236e+01, -4.6237e+00, -2.2310e-01], - [6.3029e+00, -3.7324e+00, -2.2310e-01], - [6.3029e+00, -3.7324e+00, -1.8019e+00]], - [[2.8525e+01, -8.2534e+00, -1.4676e+00], - [2.8525e+01, -8.2534e+00, 2.0648e-02], - [2.6364e+01, -7.6525e+00, 2.0648e-02], - [2.6364e+01, -7.6525e+00, -1.4676e+00], - [2.8921e+01, -6.8292e+00, -1.4676e+00], - [2.8921e+01, -6.8292e+00, 2.0648e-02], - [2.6760e+01, -6.2283e+00, 2.0648e-02], - [2.6760e+01, -6.2283e+00, -1.4676e+00]], - [[2.1337e+01, -2.9870e+01, -1.9028e+00], - [2.1337e+01, -2.9870e+01, -4.9495e-01], - [1.8102e+01, -2.8535e+01, -4.9495e-01], - [1.8102e+01, -2.8535e+01, -1.9028e+00], - [2.1935e+01, -2.8420e+01, -1.9028e+00], - [2.1935e+01, -2.8420e+01, -4.9495e-01], - [1.8700e+01, -2.7085e+01, -4.9495e-01], - [1.8700e+01, -2.7085e+01, -1.9028e+00]], - [[2.6398e+01, -1.7530e+01, -1.7879e+00], - [2.6398e+01, -1.7530e+01, -2.9959e-01], - [2.8612e+01, -1.4452e+01, -2.9959e-01], - [2.8612e+01, -1.4452e+01, -1.7879e+00], - [2.7818e+01, -1.8552e+01, -1.7879e+00], - [2.7818e+01, -1.8552e+01, -2.9959e-01], - [3.0032e+01, -1.5474e+01, -2.9959e-01], - [3.0032e+01, -1.5474e+01, -1.7879e+00]]]) - # the pytorch print loses some precision + expected_tensor = torch.tensor([[[-7.7767e-01, -2.8332e+00, -1.9169e+00], + [-7.7767e-01, -2.8332e+00, -2.5769e-01], + [2.4093e+00, -1.6232e+00, -2.5769e-01], + [2.4093e+00, -1.6232e+00, -1.9169e+00], + [-1.5301e-01, -4.4784e+00, -1.9169e+00], + [-1.5301e-01, -4.4784e+00, -2.5769e-01], + [3.0340e+00, -3.2684e+00, -2.5769e-01], + [3.0340e+00, -3.2684e+00, -1.9169e+00]], + [[5.9606e+00, -4.6237e+00, -1.8019e+00], + [5.9606e+00, -4.6237e+00, -2.2310e-01], + [9.8933e+00, -3.7324e+00, -2.2310e-01], + [9.8933e+00, -3.7324e+00, -1.8019e+00], + [6.3029e+00, -6.1340e+00, -1.8019e+00], + [6.3029e+00, -6.1340e+00, -2.2310e-01], + [1.0236e+01, -5.2427e+00, -2.2310e-01], + [1.0236e+01, -5.2427e+00, -1.8019e+00]], + [[2.6364e+01, -6.8292e+00, -1.4676e+00], + [2.6364e+01, -6.8292e+00, 2.0648e-02], + [2.8525e+01, -6.2283e+00, 2.0648e-02], + [2.8525e+01, -6.2283e+00, -1.4676e+00], + [2.6760e+01, -8.2534e+00, -1.4676e+00], + [2.6760e+01, -8.2534e+00, 2.0648e-02], + [2.8921e+01, -7.6525e+00, 2.0648e-02], + [2.8921e+01, -7.6525e+00, -1.4676e+00]], + [[1.8102e+01, -2.8420e+01, -1.9028e+00], + [1.8102e+01, -2.8420e+01, -4.9495e-01], + [2.1337e+01, -2.7085e+01, -4.9495e-01], + [2.1337e+01, -2.7085e+01, -1.9028e+00], + [1.8700e+01, -2.9870e+01, -1.9028e+00], + [1.8700e+01, -2.9870e+01, -4.9495e-01], + [2.1935e+01, -2.8535e+01, -4.9495e-01], + [2.1935e+01, -2.8535e+01, -1.9028e+00]], + [[2.8612e+01, -1.8552e+01, -1.7879e+00], + [2.8612e+01, -1.8552e+01, -2.9959e-01], + [2.6398e+01, -1.5474e+01, -2.9959e-01], + [2.6398e+01, -1.5474e+01, -1.7879e+00], + [3.0032e+01, -1.7530e+01, -1.7879e+00], + [3.0032e+01, -1.7530e+01, -2.9959e-01], + [2.7818e+01, -1.4452e+01, -2.9959e-01], + [2.7818e+01, -1.4452e+01, -1.7879e+00]]]) + assert torch.allclose(boxes.corners, expected_tensor, rtol=1e-4, atol=1e-7) # test new_box @@ -557,26 +623,27 @@ def test_boxes_conversion(): [0.000000e+00, 0.000000e+00, 0.000000e+00, 1.000000e+00]], dtype=torch.float32) + # coord sys refactor (reverse sign of yaw) expected_tensor = torch.tensor( [[ - 2.16902434e+01, -4.06038554e-02, -1.61906639e+00, 1.65999997e+00, - 3.20000005e+00, 1.61000001e+00, -1.53999996e+00 + 2.16902434e+01, -4.06038554e-02, -1.61906639e+00, 3.20000005e+00, + 1.65999997e+00, 1.61000001e+00, 1.53999996e+00 - np.pi / 2 ], [ - 7.05006905e+00, -6.57459601e+00, -1.60107949e+00, 2.27999997e+00, - 1.27799997e+01, 3.66000009e+00, 1.54999995e+00 + 7.05006905e+00, -6.57459601e+00, -1.60107949e+00, 1.27799997e+01, + 2.27999997e+00, 3.66000009e+00, -1.54999995e+00 - np.pi / 2 ], [ - 2.24698818e+01, -6.69203759e+00, -1.50118145e+00, 2.31999993e+00, - 1.47299995e+01, 3.64000010e+00, 1.59000003e+00 + 2.24698818e+01, -6.69203759e+00, -1.50118145e+00, 1.47299995e+01, + 2.31999993e+00, 3.64000010e+00, -1.59000003e+00 + 3 * np.pi / 2 ], [ - 3.48291965e+01, -7.09058388e+00, -1.36622983e+00, 2.31999993e+00, - 1.00400000e+01, 3.60999990e+00, 1.61000001e+00 + 3.48291965e+01, -7.09058388e+00, -1.36622983e+00, 1.00400000e+01, + 2.31999993e+00, 3.60999990e+00, -1.61000001e+00 + 3 * np.pi / 2 ], [ - 4.62394617e+01, -7.75838800e+00, -1.32405020e+00, 2.33999991e+00, - 1.28299999e+01, 3.63000011e+00, 1.63999999e+00 + 4.62394617e+01, -7.75838800e+00, -1.32405020e+00, 1.28299999e+01, + 2.33999991e+00, 3.63000011e+00, -1.63999999e+00 + 3 * np.pi / 2 ]], dtype=torch.float32) @@ -636,10 +703,15 @@ def test_boxes_conversion(): def test_camera_boxes3d(): # Test init with numpy array - np_boxes = np.array( - [[1.7802081, 2.516249, -1.7501148, 1.75, 3.39, 1.65, 1.48], - [8.959413, 2.4567227, -1.6357126, 1.54, 4.01, 1.57, 1.62]], - dtype=np.float32) + np_boxes = np.array([[ + 1.7802081, 2.516249, -1.7501148, 1.75, 3.39, 1.65, + 1.48 - 0.13603681398218053 * 4 - 2 * np.pi + ], + [ + 8.959413, 2.4567227, -1.6357126, 1.54, 4.01, 1.57, + 1.62 - 0.13603681398218053 * 4 - 2 * np.pi + ]], + dtype=np.float32) boxes_1 = Box3DMode.convert( LiDARInstance3DBoxes(np_boxes), Box3DMode.LIDAR, Box3DMode.CAM) @@ -653,15 +725,15 @@ def test_camera_boxes3d(): th_boxes = torch.tensor( [[ 28.29669987, -0.5557558, -1.30332506, 1.47000003, 2.23000002, - 1.48000002, -1.57000005 + 1.48000002, -1.57000005 - 0.13603681398218053 * 4 - 2 * np.pi ], [ 26.66901946, 21.82302134, -1.73605708, 1.55999994, 3.48000002, - 1.39999998, -1.69000006 + 1.39999998, -1.69000006 - 0.13603681398218053 * 4 - 2 * np.pi ], [ 31.31977974, 8.16214412, -1.62177875, 1.74000001, 3.76999998, - 1.48000002, 2.78999996 + 1.48000002, 2.78999996 - 0.13603681398218053 * 4 - 2 * np.pi ]], dtype=torch.float32) cam_th_boxes = Box3DMode.convert(th_boxes, Box3DMode.LIDAR, Box3DMode.CAM) @@ -674,13 +746,26 @@ def test_camera_boxes3d(): # test box concatenation expected_tensor = Box3DMode.convert( - torch.tensor( - [[1.7802081, 2.516249, -1.7501148, 1.75, 3.39, 1.65, 1.48], - [8.959413, 2.4567227, -1.6357126, 1.54, 4.01, 1.57, 1.62], - [28.2967, -0.5557558, -1.303325, 1.47, 2.23, 1.48, -1.57], - [26.66902, 21.82302, -1.736057, 1.56, 3.48, 1.4, -1.69], - [31.31978, 8.162144, -1.6217787, 1.74, 3.77, 1.48, 2.79]]), - Box3DMode.LIDAR, Box3DMode.CAM) + torch.tensor([[ + 1.7802081, 2.516249, -1.7501148, 1.75, 3.39, 1.65, + 1.48 - 0.13603681398218053 * 4 - 2 * np.pi + ], + [ + 8.959413, 2.4567227, -1.6357126, 1.54, 4.01, 1.57, + 1.62 - 0.13603681398218053 * 4 - 2 * np.pi + ], + [ + 28.2967, -0.5557558, -1.303325, 1.47, 2.23, 1.48, + -1.57 - 0.13603681398218053 * 4 - 2 * np.pi + ], + [ + 26.66902, 21.82302, -1.736057, 1.56, 3.48, 1.4, + -1.69 - 0.13603681398218053 * 4 - 2 * np.pi + ], + [ + 31.31978, 8.162144, -1.6217787, 1.74, 3.77, 1.48, + 2.79 - 0.13603681398218053 * 4 - 2 * np.pi + ]]), Box3DMode.LIDAR, Box3DMode.CAM) boxes = CameraInstance3DBoxes.cat([boxes_1, boxes_2]) assert torch.allclose(boxes.tensor, expected_tensor) @@ -689,28 +774,60 @@ def test_camera_boxes3d(): [-0.2517, 0.9697, 6.7053], [0.5520, 0.5265, 0.6533], [-0.5358, 1.4741, 4.5870]]) expected_tensor = Box3DMode.convert( - torch.tensor( - [[1.7802081, -2.516249, -1.7501148, 1.75, 3.39, 1.65, 1.6615927], - [8.959413, -2.4567227, -1.6357126, 1.54, 4.01, 1.57, 1.5215927], - [28.2967, 0.5557558, -1.303325, 1.47, 2.23, 1.48, 4.7115927], - [26.66902, -21.82302, -1.736057, 1.56, 3.48, 1.4, 4.8315926], - [31.31978, -8.162144, -1.6217787, 1.74, 3.77, 1.48, 0.35159278]]), - Box3DMode.LIDAR, Box3DMode.CAM) + torch.tensor([[ + 1.7802081, -2.516249, -1.7501148, 1.75, 3.39, 1.65, + 1.6615927 + 0.13603681398218053 * 4 - np.pi + ], + [ + 8.959413, -2.4567227, -1.6357126, 1.54, 4.01, 1.57, + 1.5215927 + 0.13603681398218053 * 4 - np.pi + ], + [ + 28.2967, 0.5557558, -1.303325, 1.47, 2.23, 1.48, + 4.7115927 + 0.13603681398218053 * 4 - np.pi + ], + [ + 26.66902, -21.82302, -1.736057, 1.56, 3.48, 1.4, + 4.8315926 + 0.13603681398218053 * 4 - np.pi + ], + [ + 31.31978, -8.162144, -1.6217787, 1.74, 3.77, 1.48, + 0.35159278 + 0.13603681398218053 * 4 - np.pi + ]]), Box3DMode.LIDAR, Box3DMode.CAM) points = boxes.flip('horizontal', points) expected_points = torch.tensor([[-0.6762, 1.4658, 1.2559], [-0.8784, 1.3857, 4.7814], [0.2517, 0.9697, 6.7053], [-0.5520, 0.5265, 0.6533], [0.5358, 1.4741, 4.5870]]) - assert torch.allclose(boxes.tensor, expected_tensor) + + yaw_normalized_tensor = boxes.tensor.clone() + yaw_normalized_tensor[:, -1:] = limit_period( + yaw_normalized_tensor[:, -1:], period=np.pi * 2) + assert torch.allclose(yaw_normalized_tensor, expected_tensor, 1e-3) assert torch.allclose(points, expected_points, 1e-3) expected_tensor = torch.tensor( - [[2.5162, 1.7501, -1.7802, 3.3900, 1.6500, 1.7500, -1.6616], - [2.4567, 1.6357, -8.9594, 4.0100, 1.5700, 1.5400, -1.5216], - [-0.5558, 1.3033, -28.2967, 2.2300, 1.4800, 1.4700, -4.7116], - [21.8230, 1.7361, -26.6690, 3.4800, 1.4000, 1.5600, -4.8316], - [8.1621, 1.6218, -31.3198, 3.7700, 1.4800, 1.7400, -0.3516]]) + [[ + 2.5162, 1.7501, -1.7802, 1.7500, 1.6500, 3.3900, + 1.6616 + 0.13603681398218053 * 4 - np.pi / 2 + ], + [ + 2.4567, 1.6357, -8.9594, 1.5400, 1.5700, 4.0100, + 1.5216 + 0.13603681398218053 * 4 - np.pi / 2 + ], + [ + -0.5558, 1.3033, -28.2967, 1.4700, 1.4800, 2.2300, + 4.7116 + 0.13603681398218053 * 4 - np.pi / 2 + ], + [ + 21.8230, 1.7361, -26.6690, 1.5600, 1.4000, 3.4800, + 4.8316 + 0.13603681398218053 * 4 - np.pi / 2 + ], + [ + 8.1621, 1.6218, -31.3198, 1.7400, 1.4800, 3.7700, + 0.3516 + 0.13603681398218053 * 4 - np.pi / 2 + ]]) boxes_flip_vert = boxes.clone() points = boxes_flip_vert.flip('vertical', points) expected_points = torch.tensor([[-0.6762, 1.4658, -1.2559], @@ -718,19 +835,38 @@ def test_camera_boxes3d(): [0.2517, 0.9697, -6.7053], [-0.5520, 0.5265, -0.6533], [0.5358, 1.4741, -4.5870]]) - assert torch.allclose(boxes_flip_vert.tensor, expected_tensor, 1e-4) + + yaw_normalized_tensor = boxes_flip_vert.tensor.clone() + yaw_normalized_tensor[:, -1:] = limit_period( + yaw_normalized_tensor[:, -1:], period=np.pi * 2) + expected_tensor[:, -1:] = limit_period( + expected_tensor[:, -1:], period=np.pi * 2) + assert torch.allclose(yaw_normalized_tensor, expected_tensor, 1e-4) assert torch.allclose(points, expected_points) # test box rotation # with input torch.Tensor points and angle expected_tensor = Box3DMode.convert( - torch.tensor( - [[1.4225, -2.7344, -1.7501, 1.7500, 3.3900, 1.6500, 1.7976], - [8.5435, -3.6491, -1.6357, 1.5400, 4.0100, 1.5700, 1.6576], - [28.1106, -3.2869, -1.3033, 1.4700, 2.2300, 1.4800, 4.8476], - [23.4630, -25.2382, -1.7361, 1.5600, 3.4800, 1.4000, 4.9676], - [29.9235, -12.3342, -1.6218, 1.7400, 3.7700, 1.4800, 0.4876]]), - Box3DMode.LIDAR, Box3DMode.CAM) + torch.tensor([[ + 1.4225, -2.7344, -1.7501, 1.7500, 3.3900, 1.6500, + 1.7976 + 0.13603681398218053 * 2 - np.pi + ], + [ + 8.5435, -3.6491, -1.6357, 1.5400, 4.0100, 1.5700, + 1.6576 + 0.13603681398218053 * 2 - np.pi + ], + [ + 28.1106, -3.2869, -1.3033, 1.4700, 2.2300, 1.4800, + 4.8476 + 0.13603681398218053 * 2 - np.pi + ], + [ + 23.4630, -25.2382, -1.7361, 1.5600, 3.4800, 1.4000, + 4.9676 + 0.13603681398218053 * 2 - np.pi + ], + [ + 29.9235, -12.3342, -1.6218, 1.7400, 3.7700, 1.4800, + 0.4876 + 0.13603681398218053 * 2 - np.pi + ]]), Box3DMode.LIDAR, Box3DMode.CAM) points, rot_mat_T = boxes.rotate(torch.tensor(0.13603681398218053), points) expected_points = torch.tensor([[-0.8403, 1.4658, -1.1526], [-1.5187, 1.3857, -4.6181], @@ -740,7 +876,12 @@ def test_camera_boxes3d(): expected_rot_mat_T = torch.tensor([[0.9908, 0.0000, -0.1356], [0.0000, 1.0000, 0.0000], [0.1356, 0.0000, 0.9908]]) - assert torch.allclose(boxes.tensor, expected_tensor, 1e-3) + yaw_normalized_tensor = boxes.tensor.clone() + yaw_normalized_tensor[:, -1:] = limit_period( + yaw_normalized_tensor[:, -1:], period=np.pi * 2) + expected_tensor[:, -1:] = limit_period( + expected_tensor[:, -1:], period=np.pi * 2) + assert torch.allclose(yaw_normalized_tensor, expected_tensor, 1e-3) assert torch.allclose(points, expected_points, 1e-3) assert torch.allclose(rot_mat_T, expected_rot_mat_T, 1e-3) @@ -750,7 +891,10 @@ def test_camera_boxes3d(): rot_mat = np.array([[0.99076125, 0., -0.13561762], [0., 1., 0.], [0.13561762, 0., 0.99076125]]) points, rot_mat_T = boxes.rotate(rot_mat, points) - assert torch.allclose(boxes.tensor, expected_tensor, 1e-3) + yaw_normalized_tensor = boxes.tensor.clone() + yaw_normalized_tensor[:, -1:] = limit_period( + yaw_normalized_tensor[:, -1:], period=np.pi * 2) + assert torch.allclose(yaw_normalized_tensor, expected_tensor, 1e-3) assert torch.allclose(points, expected_points, 1e-3) assert torch.allclose(rot_mat_T, expected_rot_mat_T, 1e-3) @@ -787,51 +931,61 @@ def test_camera_boxes3d(): expected_tensor = Box3DMode.convert( torch.tensor([[ 1.0443488, -2.9183323, -1.7599131, 1.7597977, 3.4089797, 1.6592377, - 1.9336663 + 1.9336663 - np.pi ], [ 8.014273, -4.8007393, -1.6448704, 1.5486219, - 4.0324507, 1.57879, 1.7936664 + 4.0324507, 1.57879, 1.7936664 - np.pi ], [ 27.558605, -7.1084175, -1.310622, 1.4782301, - 2.242485, 1.488286, 4.9836664 + 2.242485, 1.488286, 4.9836664 - np.pi ], [ 19.934517, -28.344835, -1.7457767, 1.5687338, - 3.4994833, 1.4078381, 5.1036663 + 3.4994833, 1.4078381, 5.1036663 - np.pi ], [ 28.130915, -16.369587, -1.6308585, 1.7497417, - 3.791107, 1.488286, 0.6236664 + 3.791107, 1.488286, 0.6236664 - np.pi ]]), Box3DMode.LIDAR, Box3DMode.CAM) boxes.scale(1.00559866335275) - assert torch.allclose(boxes.tensor, expected_tensor) + yaw_normalized_tensor = boxes.tensor.clone() + yaw_normalized_tensor[:, -1:] = limit_period( + yaw_normalized_tensor[:, -1:], period=np.pi * 2) + expected_tensor[:, -1:] = limit_period( + expected_tensor[:, -1:], period=np.pi * 2) + assert torch.allclose(yaw_normalized_tensor, expected_tensor) # test box translation expected_tensor = Box3DMode.convert( torch.tensor([[ 1.1281544, -3.0507944, -1.9169292, 1.7597977, 3.4089797, 1.6592377, - 1.9336663 + 1.9336663 - np.pi ], [ 8.098079, -4.9332013, -1.8018866, 1.5486219, - 4.0324507, 1.57879, 1.7936664 + 4.0324507, 1.57879, 1.7936664 - np.pi ], [ 27.64241, -7.2408795, -1.4676381, 1.4782301, - 2.242485, 1.488286, 4.9836664 + 2.242485, 1.488286, 4.9836664 - np.pi ], [ 20.018322, -28.477297, -1.9027928, 1.5687338, - 3.4994833, 1.4078381, 5.1036663 + 3.4994833, 1.4078381, 5.1036663 - np.pi ], [ 28.21472, -16.502048, -1.7878747, 1.7497417, - 3.791107, 1.488286, 0.6236664 + 3.791107, 1.488286, 0.6236664 - np.pi ]]), Box3DMode.LIDAR, Box3DMode.CAM) boxes.translate(torch.tensor([0.13246193, 0.15701613, 0.0838056])) - assert torch.allclose(boxes.tensor, expected_tensor) + yaw_normalized_tensor = boxes.tensor.clone() + yaw_normalized_tensor[:, -1:] = limit_period( + yaw_normalized_tensor[:, -1:], period=np.pi * 2) + expected_tensor[:, -1:] = limit_period( + expected_tensor[:, -1:], period=np.pi * 2) + assert torch.allclose(yaw_normalized_tensor, expected_tensor) # test bbox in_range_bev expected_tensor = torch.tensor([1, 1, 1, 1, 1], dtype=torch.bool) @@ -857,13 +1011,13 @@ def test_camera_boxes3d(): assert (boxes.tensor[:, 6] >= -np.pi / 2).all() Box3DMode.convert(boxes, Box3DMode.LIDAR, Box3DMode.LIDAR) - expected_tesor = boxes.tensor.clone() - assert torch.allclose(expected_tesor, boxes.tensor) + expected_tensor = boxes.tensor.clone() + assert torch.allclose(expected_tensor, boxes.tensor) boxes.flip() boxes.flip() boxes.limit_yaw() - assert torch.allclose(expected_tesor, boxes.tensor) + assert torch.allclose(expected_tensor, boxes.tensor) # test nearest_bev # BEV box in lidar coordinates (x, y) @@ -877,54 +1031,66 @@ def test_camera_boxes3d(): expected_tensor = lidar_expected_tensor.clone() expected_tensor[:, 0::2] = -lidar_expected_tensor[:, [3, 1]] expected_tensor[:, 1::2] = lidar_expected_tensor[:, 0::2] - # the pytorch print loses some precision assert torch.allclose( boxes.nearest_bev, expected_tensor, rtol=1e-4, atol=1e-7) - # obtained by the print of the original implementation - expected_tensor = torch.tensor([[[3.2684e+00, 2.5769e-01, -7.7767e-01], - [1.6232e+00, 2.5769e-01, -1.5301e-01], - [1.6232e+00, 1.9169e+00, -1.5301e-01], - [3.2684e+00, 1.9169e+00, -7.7767e-01], - [4.4784e+00, 2.5769e-01, 2.4093e+00], - [2.8332e+00, 2.5769e-01, 3.0340e+00], - [2.8332e+00, 1.9169e+00, 3.0340e+00], - [4.4784e+00, 1.9169e+00, 2.4093e+00]], - [[5.2427e+00, 2.2310e-01, 5.9606e+00], - [3.7324e+00, 2.2310e-01, 6.3029e+00], - [3.7324e+00, 1.8019e+00, 6.3029e+00], - [5.2427e+00, 1.8019e+00, 5.9606e+00], - [6.1340e+00, 2.2310e-01, 9.8933e+00], - [4.6237e+00, 2.2310e-01, 1.0236e+01], - [4.6237e+00, 1.8019e+00, 1.0236e+01], - [6.1340e+00, 1.8019e+00, 9.8933e+00]], - [[7.6525e+00, -2.0648e-02, 2.6364e+01], - [6.2283e+00, -2.0648e-02, 2.6760e+01], - [6.2283e+00, 1.4676e+00, 2.6760e+01], - [7.6525e+00, 1.4676e+00, 2.6364e+01], - [8.2534e+00, -2.0648e-02, 2.8525e+01], - [6.8292e+00, -2.0648e-02, 2.8921e+01], - [6.8292e+00, 1.4676e+00, 2.8921e+01], - [8.2534e+00, 1.4676e+00, 2.8525e+01]], - [[2.8535e+01, 4.9495e-01, 1.8102e+01], - [2.7085e+01, 4.9495e-01, 1.8700e+01], - [2.7085e+01, 1.9028e+00, 1.8700e+01], - [2.8535e+01, 1.9028e+00, 1.8102e+01], - [2.9870e+01, 4.9495e-01, 2.1337e+01], - [2.8420e+01, 4.9495e-01, 2.1935e+01], - [2.8420e+01, 1.9028e+00, 2.1935e+01], - [2.9870e+01, 1.9028e+00, 2.1337e+01]], - [[1.4452e+01, 2.9959e-01, 2.8612e+01], - [1.5474e+01, 2.9959e-01, 3.0032e+01], - [1.5474e+01, 1.7879e+00, 3.0032e+01], - [1.4452e+01, 1.7879e+00, 2.8612e+01], - [1.7530e+01, 2.9959e-01, 2.6398e+01], - [1.8552e+01, 2.9959e-01, 2.7818e+01], - [1.8552e+01, 1.7879e+00, 2.7818e+01], - [1.7530e+01, 1.7879e+00, 2.6398e+01]]]) - - # the pytorch print loses some precision - assert torch.allclose(boxes.corners, expected_tensor, rtol=1e-4, atol=1e-7) + expected_tensor = torch.tensor([[[2.8332e+00, 2.5769e-01, -7.7767e-01], + [1.6232e+00, 2.5769e-01, 2.4093e+00], + [1.6232e+00, 1.9169e+00, 2.4093e+00], + [2.8332e+00, 1.9169e+00, -7.7767e-01], + [4.4784e+00, 2.5769e-01, -1.5302e-01], + [3.2684e+00, 2.5769e-01, 3.0340e+00], + [3.2684e+00, 1.9169e+00, 3.0340e+00], + [4.4784e+00, 1.9169e+00, -1.5302e-01]], + [[4.6237e+00, 2.2310e-01, 5.9606e+00], + [3.7324e+00, 2.2310e-01, 9.8933e+00], + [3.7324e+00, 1.8019e+00, 9.8933e+00], + [4.6237e+00, 1.8019e+00, 5.9606e+00], + [6.1340e+00, 2.2310e-01, 6.3029e+00], + [5.2427e+00, 2.2310e-01, 1.0236e+01], + [5.2427e+00, 1.8019e+00, 1.0236e+01], + [6.1340e+00, 1.8019e+00, 6.3029e+00]], + [[6.8292e+00, -2.0648e-02, 2.6364e+01], + [6.2283e+00, -2.0648e-02, 2.8525e+01], + [6.2283e+00, 1.4676e+00, 2.8525e+01], + [6.8292e+00, 1.4676e+00, 2.6364e+01], + [8.2534e+00, -2.0648e-02, 2.6760e+01], + [7.6525e+00, -2.0648e-02, 2.8921e+01], + [7.6525e+00, 1.4676e+00, 2.8921e+01], + [8.2534e+00, 1.4676e+00, 2.6760e+01]], + [[2.8420e+01, 4.9495e-01, 1.8102e+01], + [2.7085e+01, 4.9495e-01, 2.1337e+01], + [2.7085e+01, 1.9028e+00, 2.1337e+01], + [2.8420e+01, 1.9028e+00, 1.8102e+01], + [2.9870e+01, 4.9495e-01, 1.8700e+01], + [2.8535e+01, 4.9495e-01, 2.1935e+01], + [2.8535e+01, 1.9028e+00, 2.1935e+01], + [2.9870e+01, 1.9028e+00, 1.8700e+01]], + [[1.4452e+01, 2.9959e-01, 2.7818e+01], + [1.7530e+01, 2.9959e-01, 3.0032e+01], + [1.7530e+01, 1.7879e+00, 3.0032e+01], + [1.4452e+01, 1.7879e+00, 2.7818e+01], + [1.5474e+01, 2.9959e-01, 2.6398e+01], + [1.8552e+01, 2.9959e-01, 2.8612e+01], + [1.8552e+01, 1.7879e+00, 2.8612e+01], + [1.5474e+01, 1.7879e+00, 2.6398e+01]]]) + + assert torch.allclose(boxes.corners, expected_tensor, rtol=1e-3, atol=1e-4) + + th_boxes = torch.tensor( + [[ + 28.29669987, -0.5557558, -1.30332506, 1.47000003, 2.23000002, + 1.48000002, -1.57000005 + ], + [ + 26.66901946, 21.82302134, -1.73605708, 1.55999994, 3.48000002, + 1.39999998, -1.69000006 + ], + [ + 31.31977974, 8.16214412, -1.62177875, 1.74000001, 3.76999998, + 1.48000002, 2.78999996 + ]], + dtype=torch.float32) # test init with a given origin boxes_origin_given = CameraInstance3DBoxes( @@ -947,17 +1113,17 @@ def test_boxes3d_overlaps(): # Test LiDAR boxes 3D overlaps boxes1_tensor = torch.tensor( - [[1.8, -2.5, -1.8, 1.75, 3.39, 1.65, 1.6615927], - [8.9, -2.5, -1.6, 1.54, 4.01, 1.57, 1.5215927], - [28.3, 0.5, -1.3, 1.47, 2.23, 1.48, 4.7115927], - [31.3, -8.2, -1.6, 1.74, 3.77, 1.48, 0.35]], + [[1.8, -2.5, -1.8, 1.75, 3.39, 1.65, -1.6615927], + [8.9, -2.5, -1.6, 1.54, 4.01, 1.57, -1.5215927], + [28.3, 0.5, -1.3, 1.47, 2.23, 1.48, -4.7115927], + [31.3, -8.2, -1.6, 1.74, 3.77, 1.48, -0.35]], device='cuda') boxes1 = LiDARInstance3DBoxes(boxes1_tensor) - boxes2_tensor = torch.tensor([[1.2, -3.0, -1.9, 1.8, 3.4, 1.7, 1.9], - [8.1, -2.9, -1.8, 1.5, 4.1, 1.6, 1.8], - [31.3, -8.2, -1.6, 1.74, 3.77, 1.48, 0.35], - [20.1, -28.5, -1.9, 1.6, 3.5, 1.4, 5.1]], + boxes2_tensor = torch.tensor([[1.2, -3.0, -1.9, 1.8, 3.4, 1.7, -1.9], + [8.1, -2.9, -1.8, 1.5, 4.1, 1.6, -1.8], + [31.3, -8.2, -1.6, 1.74, 3.77, 1.48, -0.35], + [20.1, -28.5, -1.9, 1.6, 3.5, 1.4, -5.1]], device='cuda') boxes2 = LiDARInstance3DBoxes(boxes2_tensor) @@ -1100,6 +1266,7 @@ def test_depth_boxes3d(): [-2.4016, -3.2521, 0.4426, 0.8234, 0.5325, 1.0099, -0.1215], [-2.5181, -2.5298, -0.4321, 0.8597, 0.6193, 1.0204, -0.0493], [-1.5434, -2.4951, -0.5570, 0.9385, 2.1404, 0.8954, -0.0585]]) + expected_tensor[:, -1:] -= 0.022998953275003075 * 2 points, rot_mat_T = boxes_rot.rotate(-0.022998953275003075, points) expected_points = torch.tensor([[-0.7049, -1.2400, -1.4658, 2.5359], [-0.9881, -4.7599, -1.3857, 0.7167], @@ -1114,10 +1281,13 @@ def test_depth_boxes3d(): assert torch.allclose(rot_mat_T, expected_rot_mat_T, 1e-3) # with input torch.Tensor points and rotation matrix - points, rot_mat_T = boxes.rotate(0.022998953275003075, points) # back + points, rot_mat_T = boxes.rotate(-0.022998953275003075, points) # back rot_mat = np.array([[0.99973554, 0.02299693, 0.], [-0.02299693, 0.99973554, 0.], [0., 0., 1.]]) points, rot_mat_T = boxes.rotate(rot_mat, points) + expected_rot_mat_T = torch.tensor([[0.99973554, 0.02299693, 0.0000], + [-0.02299693, 0.99973554, 0.0000], + [0.0000, 0.0000, 1.0000]]) assert torch.allclose(boxes_rot.tensor, expected_tensor, 1e-3) assert torch.allclose(points, expected_points, 1e-3) assert torch.allclose(rot_mat_T, expected_rot_mat_T, 1e-3) @@ -1134,27 +1304,64 @@ def test_depth_boxes3d(): [-0.0974, 6.7093, -0.9697, 0.5599], [0.5669, 0.6404, -0.5265, 1.0032], [-0.4302, 4.5981, -1.4741, 0.0556]]) - expected_rot_mat_T_np = np.array([[0.9997, -0.0230, 0.0000], - [0.0230, 0.9997, 0.0000], + expected_rot_mat_T_np = np.array([[0.99973554, -0.02299693, 0.0000], + [0.02299693, 0.99973554, 0.0000], [0.0000, 0.0000, 1.0000]]) expected_tensor = torch.tensor( [[-1.5434, -2.4951, -0.5570, 0.9385, 2.1404, 0.8954, -0.0585], [-2.4016, -3.2521, 0.4426, 0.8234, 0.5325, 1.0099, -0.1215], [-2.5181, -2.5298, -0.4321, 0.8597, 0.6193, 1.0204, -0.0493], [-1.5434, -2.4951, -0.5570, 0.9385, 2.1404, 0.8954, -0.0585]]) + expected_tensor[:, -1:] -= 0.022998953275003075 * 2 assert torch.allclose(boxes.tensor, expected_tensor, 1e-3) assert np.allclose(points_np, expected_points_np, 1e-3) assert np.allclose(rot_mat_T_np, expected_rot_mat_T_np, 1e-3) # with input DepthPoints and rotation matrix - points_np, rot_mat_T_np = boxes.rotate(0.022998953275003075, points_np) + points_np, rot_mat_T_np = boxes.rotate(-0.022998953275003075, points_np) depth_points = DepthPoints(points_np, points_dim=4) depth_points, rot_mat_T_np = boxes.rotate(rot_mat, depth_points) points_np = depth_points.tensor.numpy() + expected_rot_mat_T_np = expected_rot_mat_T_np.T assert torch.allclose(boxes.tensor, expected_tensor, 1e-3) assert np.allclose(points_np, expected_points_np, 1e-3) assert np.allclose(rot_mat_T_np, expected_rot_mat_T_np, 1e-3) + expected_tensor = torch.tensor([[[-2.1217, -3.5105, -0.5570], + [-2.1217, -3.5105, 0.3384], + [-1.8985, -1.3818, 0.3384], + [-1.8985, -1.3818, -0.5570], + [-1.1883, -3.6084, -0.5570], + [-1.1883, -3.6084, 0.3384], + [-0.9651, -1.4796, 0.3384], + [-0.9651, -1.4796, -0.5570]], + [[-2.8519, -3.4460, 0.4426], + [-2.8519, -3.4460, 1.4525], + [-2.7632, -2.9210, 1.4525], + [-2.7632, -2.9210, 0.4426], + [-2.0401, -3.5833, 0.4426], + [-2.0401, -3.5833, 1.4525], + [-1.9513, -3.0582, 1.4525], + [-1.9513, -3.0582, 0.4426]], + [[-2.9755, -2.7971, -0.4321], + [-2.9755, -2.7971, 0.5883], + [-2.9166, -2.1806, 0.5883], + [-2.9166, -2.1806, -0.4321], + [-2.1197, -2.8789, -0.4321], + [-2.1197, -2.8789, 0.5883], + [-2.0608, -2.2624, 0.5883], + [-2.0608, -2.2624, -0.4321]], + [[-2.1217, -3.5105, -0.5570], + [-2.1217, -3.5105, 0.3384], + [-1.8985, -1.3818, 0.3384], + [-1.8985, -1.3818, -0.5570], + [-1.1883, -3.6084, -0.5570], + [-1.1883, -3.6084, 0.3384], + [-0.9651, -1.4796, 0.3384], + [-0.9651, -1.4796, -0.5570]]]) + + assert torch.allclose(boxes.corners, expected_tensor, 1e-3) + th_boxes = torch.tensor( [[0.61211395, 0.8129094, 0.10563634, 1.497534, 0.16927195, 0.27956772], [1.430009, 0.49797538, 0.9382923, 0.07694054, 0.9312509, 1.8919173]], @@ -1197,11 +1404,11 @@ def test_depth_boxes3d(): [1.5112, -0.0352, 2.8302], [1.5112, 0.8986, 2.8302], [1.5112, 0.8986, 0.9383]]]) - torch.allclose(boxes.corners, expected_tensor) + assert torch.allclose(boxes.corners, expected_tensor, 1e-3) # test points in boxes if torch.cuda.is_available(): - box_idxs_of_pts = boxes.points_in_boxes(points.cuda()) + box_idxs_of_pts = boxes.points_in_boxes_batch(points.cuda()) expected_idxs_of_pts = torch.tensor( [[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]], device='cuda:0', @@ -1210,8 +1417,8 @@ def test_depth_boxes3d(): # test get_surface_line_center boxes = torch.tensor( - [[0.3294, 1.0359, 0.1171, 1.0822, 1.1247, 1.3721, 0.4916], - [-2.4630, -2.6324, -0.1616, 0.9202, 1.7896, 0.1992, 0.3185]]) + [[0.3294, 1.0359, 0.1171, 1.0822, 1.1247, 1.3721, -0.4916], + [-2.4630, -2.6324, -0.1616, 0.9202, 1.7896, 0.1992, -0.3185]]) boxes = DepthInstance3DBoxes( boxes, box_dim=boxes.shape[-1], with_yaw=True, origin=(0.5, 0.5, 0.5)) surface_center, line_center = boxes.get_surface_line_center() @@ -1259,22 +1466,97 @@ def test_depth_boxes3d(): def test_rotation_3d_in_axis(): + # # clockwise + # points = torch.tensor([[[-0.4599, -0.0471, 0.0000], + # [-0.4599, -0.0471, 1.8433], + # [-0.4599, 0.0471, 1.8433]], + # [[-0.2555, -0.2683, 0.0000], + # [-0.2555, -0.2683, 0.9072], + # [-0.2555, 0.2683, 0.9072]]]) + # rotated = rotation_3d_in_axis( + # points, torch.tensor([-np.pi / 10, np.pi / 10]), + # axis=0, clockwise=True) + # expected_rotated = torch.tensor([[[0.0000, -0.4228, -0.1869], + # [1.8433, -0.4228, -0.1869], + # [1.8433, -0.4519, -0.0973]], + # [[0.0000, -0.3259, -0.1762], + # [0.9072, -0.3259, -0.1762], + # [0.9072, -0.1601, 0.3341]]]) + # assert torch.allclose(rotated, expected_rotated, 1e-3) + + # anti-clockwise with return rotation mat points = torch.tensor([[[-0.4599, -0.0471, 0.0000], - [-0.4599, -0.0471, 1.8433], - [-0.4599, 0.0471, 1.8433]], - [[-0.2555, -0.2683, 0.0000], - [-0.2555, -0.2683, 0.9072], - [-0.2555, 0.2683, 0.9072]]]) - rotated = rotation_3d_in_axis( - points, torch.tensor([-np.pi / 10, np.pi / 10]), axis=0) - expected_rotated = torch.tensor([[[0.0000, -0.4228, -0.1869], - [1.8433, -0.4228, -0.1869], - [1.8433, -0.4519, -0.0973]], - [[0.0000, -0.3259, -0.1762], - [0.9072, -0.3259, -0.1762], - [0.9072, -0.1601, 0.3341]]]) + [-0.4599, -0.0471, 1.8433]]]) + rotated = rotation_3d_in_axis(points, torch.tensor([np.pi / 2]), axis=0) + expected_rotated = torch.tensor([[[-0.4599, 0.0000, -0.0471], + [-0.4599, -1.8433, -0.0471]]]) assert torch.allclose(rotated, expected_rotated, 1e-3) + points = torch.tensor([[[-0.4599, -0.0471, 0.0000], + [-0.4599, -0.0471, 1.8433]]]) + rotated, mat = rotation_3d_in_axis( + points, torch.tensor([np.pi / 2]), axis=0, return_mat=True) + expected_rotated = torch.tensor([[[-0.4599, 0.0000, -0.0471], + [-0.4599, -1.8433, -0.0471]]]) + expected_mat = torch.tensor([[[1, 0, 0], [0, 0, 1], [0, -1, 0]]]).float() + assert torch.allclose(rotated, expected_rotated, atol=1e-6) + assert torch.allclose(mat, expected_mat, atol=1e-6) + + points = torch.tensor([[[-0.4599, -0.0471, 0.0000], + [-0.4599, -0.0471, 1.8433]], + [[-0.2555, -0.2683, 0.0000], + [-0.2555, -0.2683, 0.9072]]]) + rotated = rotation_3d_in_axis(points, np.pi / 2, axis=0) + expected_rotated = torch.tensor([[[-0.4599, 0.0000, -0.0471], + [-0.4599, -1.8433, -0.0471]], + [[-0.2555, 0.0000, -0.2683], + [-0.2555, -0.9072, -0.2683]]]) + assert torch.allclose(rotated, expected_rotated, atol=1e-3) + + points = np.array([[[-0.4599, -0.0471, 0.0000], [-0.4599, -0.0471, + 1.8433]], + [[-0.2555, -0.2683, 0.0000], + [-0.2555, -0.2683, 0.9072]]]).astype(np.float32) + + rotated = rotation_3d_in_axis(points, np.pi / 2, axis=0) + expected_rotated = np.array([[[-0.4599, 0.0000, -0.0471], + [-0.4599, -1.8433, -0.0471]], + [[-0.2555, 0.0000, -0.2683], + [-0.2555, -0.9072, -0.2683]]]) + assert np.allclose(rotated, expected_rotated, atol=1e-3) + + points = torch.tensor([[[-0.4599, -0.0471, 0.0000], + [-0.4599, -0.0471, 1.8433]], + [[-0.2555, -0.2683, 0.0000], + [-0.2555, -0.2683, 0.9072]]]) + angles = [np.pi / 2, -np.pi / 2] + rotated = rotation_3d_in_axis(points, angles, axis=0) + expected_rotated = np.array([[[-0.4599, 0.0000, -0.0471], + [-0.4599, -1.8433, -0.0471]], + [[-0.2555, 0.0000, 0.2683], + [-0.2555, 0.9072, 0.2683]]]) + assert np.allclose(rotated, expected_rotated, atol=1e-3) + + points = torch.tensor([[[-0.0471, 0.0000], [-0.0471, 1.8433]], + [[-0.2683, 0.0000], [-0.2683, 0.9072]]]) + angles = [np.pi / 2, -np.pi / 2] + rotated = rotation_3d_in_axis(points, angles) + expected_rotated = np.array([[[0.0000, -0.0471], [-1.8433, -0.0471]], + [[0.0000, 0.2683], [0.9072, 0.2683]]]) + assert np.allclose(rotated, expected_rotated, atol=1e-3) + + +def test_rotation_2d(): + angles = np.array([3.14]) + corners = np.array([[[-0.235, -0.49], [-0.235, 0.49], [0.235, 0.49], + [0.235, -0.49]]]) + corners_rotated = rotation_3d_in_axis(corners, angles) + expected_corners = np.array([[[0.2357801, 0.48962511], + [0.2342193, -0.49037365], + [-0.2357801, -0.48962511], + [-0.2342193, 0.49037365]]]) + assert np.allclose(corners_rotated, expected_corners) + def test_limit_period(): torch.manual_seed(0) @@ -1284,6 +1566,11 @@ def test_limit_period(): [0.3074]]) assert torch.allclose(result, expected_result, 1e-3) + val = val.numpy() + result = limit_period(val) + expected_result = expected_result.numpy() + assert np.allclose(result, expected_result, 1e-3) + def test_xywhr2xyxyr(): torch.manual_seed(0) @@ -1323,3 +1610,14 @@ def test_points_cam2img(): [0.6994, 0.7782], [0.5623, 0.6303], [0.4359, 0.6532]]) assert torch.allclose(point_2d_res, expected_point_2d_res, 1e-3) + + points = points.numpy() + proj_mat = proj_mat.numpy() + point_2d_res = points_cam2img(points, proj_mat) + expected_point_2d_res = expected_point_2d_res.numpy() + assert np.allclose(point_2d_res, expected_point_2d_res, 1e-3) + + points = torch.from_numpy(points) + point_2d_res = points_cam2img(points, proj_mat) + expected_point_2d_res = torch.from_numpy(expected_point_2d_res) + assert torch.allclose(point_2d_res, expected_point_2d_res, 1e-3) diff --git a/tests/test_utils/test_box_np_ops.py b/tests/test_utils/test_box_np_ops.py index 9825d547e5..a6beab0b27 100644 --- a/tests/test_utils/test_box_np_ops.py +++ b/tests/test_utils/test_box_np_ops.py @@ -19,7 +19,7 @@ def test_camera_to_lidar(): def test_box_camera_to_lidar(): from mmdet3d.core.bbox.box_np_ops import box_camera_to_lidar - box = np.array([[1.84, 1.47, 8.41, 1.2, 1.89, 0.48, 0.01]]) + box = np.array([[1.84, 1.47, 8.41, 1.2, 1.89, 0.48, -0.01]]) rect = np.array([[0.9999128, 0.01009263, -0.00851193, 0.], [-0.01012729, 0.9999406, -0.00403767, 0.], [0.00847068, 0.00412352, 0.9999556, 0.], [0., 0., 0., @@ -29,8 +29,9 @@ def test_box_camera_to_lidar(): [0.9999753, 0.00693114, -0.0011439, -0.3321029], [0., 0., 0., 1.]]) box_lidar = box_camera_to_lidar(box, rect, Trv2c) - expected_box = np.array( - [[8.73138192, -1.85591746, -1.59969933, 0.48, 1.2, 1.89, 0.01]]) + expected_box = np.array([[ + 8.73138192, -1.85591746, -1.59969933, 1.2, 0.48, 1.89, 0.01 - np.pi / 2 + ]]) assert np.allclose(box_lidar, expected_box) @@ -47,22 +48,17 @@ def test_center_to_corner_box2d(): from mmdet3d.core.bbox.box_np_ops import center_to_corner_box2d center = np.array([[9.348705, -3.6271024]]) dims = np.array([[0.47, 0.98]]) - angles = np.array([-3.14]) + angles = np.array([3.14]) corner = center_to_corner_box2d(center, dims, angles) expected_corner = np.array([[[9.584485, -3.1374772], [9.582925, -4.117476], [9.112926, -4.1167274], [9.114486, -3.1367288]]]) assert np.allclose(corner, expected_corner) - -def test_rotation_2d(): - from mmdet3d.core.bbox.box_np_ops import rotation_2d - angles = np.array([-3.14]) - corners = np.array([[[-0.235, -0.49], [-0.235, 0.49], [0.235, 0.49], - [0.235, -0.49]]]) - corners_rotated = rotation_2d(corners, angles) - expected_corners = np.array([[[0.2357801, 0.48962511], - [0.2342193, -0.49037365], - [-0.2357801, -0.48962511], - [-0.2342193, 0.49037365]]]) - assert np.allclose(corners_rotated, expected_corners) + center = np.array([[-0.0, 0.0]]) + dims = np.array([[4.0, 8.0]]) + angles = np.array([-0.785398]) # -45 degrees + corner = center_to_corner_box2d(center, dims, angles) + expected_corner = np.array([[[-4.24264, -1.41421], [1.41421, 4.24264], + [4.24264, 1.41421], [-1.41421, -4.24264]]]) + assert np.allclose(corner, expected_corner) diff --git a/tests/test_utils/test_coord_3d_mode.py b/tests/test_utils/test_coord_3d_mode.py index ea1f2e3a8f..123c84103a 100644 --- a/tests/test_utils/test_coord_3d_mode.py +++ b/tests/test_utils/test_coord_3d_mode.py @@ -2,7 +2,8 @@ import torch from mmdet3d.core.bbox import (CameraInstance3DBoxes, Coord3DMode, - DepthInstance3DBoxes, LiDARInstance3DBoxes) + DepthInstance3DBoxes, LiDARInstance3DBoxes, + limit_period) from mmdet3d.core.points import CameraPoints, DepthPoints, LiDARPoints @@ -241,22 +242,31 @@ def test_boxes_conversion(): convert_lidar_boxes = Coord3DMode.convert(cam_boxes, Coord3DMode.CAM, Coord3DMode.LIDAR) - expected_tensor = torch.tensor( - [[-1.7501, -1.7802, -2.5162, 1.6500, 1.7500, 3.3900, 1.4800], - [-1.6357, -8.9594, -2.4567, 1.5700, 1.5400, 4.0100, 1.6200], - [-1.3033, -28.2967, 0.5558, 1.4800, 1.4700, 2.2300, -1.5700], - [-1.7361, -26.6690, -21.8230, 1.4000, 1.5600, 3.4800, -1.6900], - [-1.6218, -31.3198, -8.1621, 1.4800, 1.7400, 3.7700, 2.7900]]) + expected_tensor = torch.tensor([[ + -1.7501, -1.7802, -2.5162, 1.7500, 1.6500, 3.3900, -1.4800 - np.pi / 2 + ], [ + -1.6357, -8.9594, -2.4567, 1.5400, 1.5700, 4.0100, -1.6200 - np.pi / 2 + ], [-1.3033, -28.2967, 0.5558, 1.4700, 1.4800, 2.2300, 1.5700 - np.pi / 2], + [ + -1.7361, -26.6690, -21.8230, 1.5600, + 1.4000, 3.4800, 1.6900 - np.pi / 2 + ], + [ + -1.6218, -31.3198, -8.1621, 1.7400, + 1.4800, 3.7700, -2.7900 - np.pi / 2 + ]]) + expected_tensor[:, -1:] = limit_period( + expected_tensor[:, -1:], period=np.pi * 2) assert torch.allclose(expected_tensor, convert_lidar_boxes.tensor, 1e-3) convert_depth_boxes = Coord3DMode.convert(cam_boxes, Coord3DMode.CAM, Coord3DMode.DEPTH) expected_tensor = torch.tensor( - [[1.7802, 1.7501, 2.5162, 1.7500, 1.6500, 3.3900, 1.4800], - [8.9594, 1.6357, 2.4567, 1.5400, 1.5700, 4.0100, 1.6200], - [28.2967, 1.3033, -0.5558, 1.4700, 1.4800, 2.2300, -1.5700], - [26.6690, 1.7361, 21.8230, 1.5600, 1.4000, 3.4800, -1.6900], - [31.3198, 1.6218, 8.1621, 1.7400, 1.4800, 3.7700, 2.7900]]) + [[1.7802, 1.7501, 2.5162, 1.7500, 1.6500, 3.3900, -1.4800], + [8.9594, 1.6357, 2.4567, 1.5400, 1.5700, 4.0100, -1.6200], + [28.2967, 1.3033, -0.5558, 1.4700, 1.4800, 2.2300, 1.5700], + [26.6690, 1.7361, 21.8230, 1.5600, 1.4000, 3.4800, 1.6900], + [31.3198, 1.6218, 8.1621, 1.7400, 1.4800, 3.7700, -2.7900]]) assert torch.allclose(expected_tensor, convert_depth_boxes.tensor, 1e-3) # test LIDAR to CAM and DEPTH @@ -268,22 +278,42 @@ def test_boxes_conversion(): [31.31978, 8.162144, -1.6217787, 1.74, 3.77, 1.48, 2.79]]) convert_cam_boxes = Coord3DMode.convert(lidar_boxes, Coord3DMode.LIDAR, Coord3DMode.CAM) - expected_tensor = torch.tensor( - [[-2.5162, 1.7501, 1.7802, 3.3900, 1.6500, 1.7500, 1.4800], - [-2.4567, 1.6357, 8.9594, 4.0100, 1.5700, 1.5400, 1.6200], - [0.5558, 1.3033, 28.2967, 2.2300, 1.4800, 1.4700, -1.5700], - [-21.8230, 1.7361, 26.6690, 3.4800, 1.4000, 1.5600, -1.6900], - [-8.1621, 1.6218, 31.3198, 3.7700, 1.4800, 1.7400, 2.7900]]) + expected_tensor = torch.tensor([ + [-2.5162, 1.7501, 1.7802, 1.7500, 1.6500, 3.3900, -1.4800 - np.pi / 2], + [-2.4567, 1.6357, 8.9594, 1.5400, 1.5700, 4.0100, -1.6200 - np.pi / 2], + [0.5558, 1.3033, 28.2967, 1.4700, 1.4800, 2.2300, 1.5700 - np.pi / 2], + [ + -21.8230, 1.7361, 26.6690, 1.5600, 1.4000, 3.4800, + 1.6900 - np.pi / 2 + ], + [ + -8.1621, 1.6218, 31.3198, 1.7400, 1.4800, 3.7700, + -2.7900 - np.pi / 2 + ] + ]) + expected_tensor[:, -1:] = limit_period( + expected_tensor[:, -1:], period=np.pi * 2) assert torch.allclose(expected_tensor, convert_cam_boxes.tensor, 1e-3) convert_depth_boxes = Coord3DMode.convert(lidar_boxes, Coord3DMode.LIDAR, Coord3DMode.DEPTH) - expected_tensor = torch.tensor( - [[-2.5162, 1.7802, -1.7501, 3.3900, 1.7500, 1.6500, 1.4800], - [-2.4567, 8.9594, -1.6357, 4.0100, 1.5400, 1.5700, 1.6200], - [0.5558, 28.2967, -1.3033, 2.2300, 1.4700, 1.4800, -1.5700], - [-21.8230, 26.6690, -1.7361, 3.4800, 1.5600, 1.4000, -1.6900], - [-8.1621, 31.3198, -1.6218, 3.7700, 1.7400, 1.4800, 2.7900]]) + expected_tensor = torch.tensor([[ + -2.5162, 1.7802, -1.7501, 1.7500, 3.3900, 1.6500, 1.4800 + np.pi / 2 + ], [-2.4567, 8.9594, -1.6357, 1.5400, 4.0100, 1.5700, 1.6200 + np.pi / 2], + [ + 0.5558, 28.2967, -1.3033, 1.4700, + 2.2300, 1.4800, -1.5700 + np.pi / 2 + ], + [ + -21.8230, 26.6690, -1.7361, 1.5600, + 3.4800, 1.4000, -1.6900 + np.pi / 2 + ], + [ + -8.1621, 31.3198, -1.6218, 1.7400, + 3.7700, 1.4800, 2.7900 + np.pi / 2 + ]]) + expected_tensor[:, -1:] = limit_period( + expected_tensor[:, -1:], period=np.pi * 2) assert torch.allclose(expected_tensor, convert_depth_boxes.tensor, 1e-3) # test DEPTH to CAM and LIDAR @@ -296,19 +326,25 @@ def test_boxes_conversion(): convert_cam_boxes = Coord3DMode.convert(depth_boxes, Coord3DMode.DEPTH, Coord3DMode.CAM) expected_tensor = torch.tensor( - [[1.7802, -1.7501, -2.5162, 1.7500, 1.6500, 3.3900, 1.4800], - [8.9594, -1.6357, -2.4567, 1.5400, 1.5700, 4.0100, 1.6200], - [28.2967, -1.3033, 0.5558, 1.4700, 1.4800, 2.2300, -1.5700], - [26.6690, -1.7361, -21.8230, 1.5600, 1.4000, 3.4800, -1.6900], - [31.3198, -1.6218, -8.1621, 1.7400, 1.4800, 3.7700, 2.7900]]) + [[1.7802, -1.7501, -2.5162, 1.7500, 1.6500, 3.3900, -1.4800], + [8.9594, -1.6357, -2.4567, 1.5400, 1.5700, 4.0100, -1.6200], + [28.2967, -1.3033, 0.5558, 1.4700, 1.4800, 2.2300, 1.5700], + [26.6690, -1.7361, -21.8230, 1.5600, 1.4000, 3.4800, 1.6900], + [31.3198, -1.6218, -8.1621, 1.7400, 1.4800, 3.7700, -2.7900]]) assert torch.allclose(expected_tensor, convert_cam_boxes.tensor, 1e-3) convert_lidar_boxes = Coord3DMode.convert(depth_boxes, Coord3DMode.DEPTH, Coord3DMode.LIDAR) - expected_tensor = torch.tensor( - [[2.5162, -1.7802, -1.7501, 3.3900, 1.7500, 1.6500, 1.4800], - [2.4567, -8.9594, -1.6357, 4.0100, 1.5400, 1.5700, 1.6200], - [-0.5558, -28.2967, -1.3033, 2.2300, 1.4700, 1.4800, -1.5700], - [21.8230, -26.6690, -1.7361, 3.4800, 1.5600, 1.4000, -1.6900], - [8.1621, -31.3198, -1.6218, 3.7700, 1.7400, 1.4800, 2.7900]]) + expected_tensor = torch.tensor([[ + 2.5162, -1.7802, -1.7501, 1.7500, 3.3900, 1.6500, 1.4800 - np.pi / 2 + ], [ + 2.4567, -8.9594, -1.6357, 1.5400, 4.0100, 1.5700, 1.6200 - np.pi / 2 + ], [ + -0.5558, -28.2967, -1.3033, 1.4700, 2.2300, 1.4800, -1.5700 - np.pi / 2 + ], [ + 21.8230, -26.6690, -1.7361, 1.5600, 3.4800, 1.4000, -1.6900 - np.pi / 2 + ], [8.1621, -31.3198, -1.6218, 1.7400, 3.7700, 1.4800, + 2.7900 - np.pi / 2]]) + expected_tensor[:, -1:] = limit_period( + expected_tensor[:, -1:], period=np.pi * 2) assert torch.allclose(expected_tensor, convert_lidar_boxes.tensor, 1e-3) diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 7493dc83e5..00a313018f 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -1,6 +1,8 @@ +import numpy as np +import pytest import torch -from mmdet3d.core import draw_heatmap_gaussian +from mmdet3d.core import array_converter, draw_heatmap_gaussian def test_gaussian(): @@ -9,3 +11,169 @@ def test_gaussian(): radius = 2 draw_heatmap_gaussian(heatmap, ct_int, radius) assert torch.isclose(torch.sum(heatmap), torch.tensor(4.3505), atol=1e-3) + + +def test_array_converter(): + # to torch + @array_converter(to_torch=True, apply_to=('array_a', 'array_b')) + def test_func_1(array_a, array_b, container): + container.append(array_a) + container.append(array_b) + return array_a.clone(), array_b.clone() + + np_array_a = np.array([0.0]) + np_array_b = np.array([0.0]) + container = [] + new_array_a, new_array_b = test_func_1(np_array_a, np_array_b, container) + + assert isinstance(new_array_a, np.ndarray) + assert isinstance(new_array_b, np.ndarray) + assert isinstance(container[0], torch.Tensor) + assert isinstance(container[1], torch.Tensor) + + # one to torch and one not + @array_converter(to_torch=True, apply_to=('array_a', )) + def test_func_2(array_a, array_b): + return torch.cat([array_a, array_b]) + + with pytest.raises(TypeError): + _ = test_func_2(np_array_a, np_array_b) + + # wrong template_arg_name_ + @array_converter( + to_torch=True, apply_to=('array_a', ), template_arg_name_='array_c') + def test_func_3(array_a, array_b): + return torch.cat([array_a, array_b]) + + with pytest.raises(ValueError): + _ = test_func_3(np_array_a, np_array_b) + + # wrong apply_to + @array_converter(to_torch=True, apply_to=('array_a', 'array_c')) + def test_func_4(array_a, array_b): + return torch.cat([array_a, array_b]) + + with pytest.raises(ValueError): + _ = test_func_4(np_array_a, np_array_b) + + # to numpy + @array_converter(to_torch=False, apply_to=('array_a', 'array_b')) + def test_func_5(array_a, array_b, container): + container.append(array_a) + container.append(array_b) + return array_a.copy(), array_b.copy() + + pt_array_a = torch.tensor([0.0]) + pt_array_b = torch.tensor([0.0]) + container = [] + new_array_a, new_array_b = test_func_5(pt_array_a, pt_array_b, container) + + assert isinstance(container[0], np.ndarray) + assert isinstance(container[1], np.ndarray) + assert isinstance(new_array_a, torch.Tensor) + assert isinstance(new_array_b, torch.Tensor) + + # apply_to = None + @array_converter(to_torch=False) + def test_func_6(array_a, array_b, container): + container.append(array_a) + container.append(array_b) + return array_a.clone(), array_b.clone() + + container = [] + new_array_a, new_array_b = test_func_6(pt_array_a, pt_array_b, container) + + assert isinstance(container[0], torch.Tensor) + assert isinstance(container[1], torch.Tensor) + assert isinstance(new_array_a, torch.Tensor) + assert isinstance(new_array_b, torch.Tensor) + + # with default arg + @array_converter(to_torch=True, apply_to=('array_a', 'array_b')) + def test_func_7(array_a, container, array_b=np.array([2.])): + container.append(array_a) + container.append(array_b) + return array_a.clone(), array_b.clone() + + container = [] + new_array_a, new_array_b = test_func_7(np_array_a, container) + + assert isinstance(container[0], torch.Tensor) + assert isinstance(container[1], torch.Tensor) + assert isinstance(new_array_a, np.ndarray) + assert isinstance(new_array_b, np.ndarray) + assert np.allclose(new_array_b, np.array([2.]), 1e-3) + + # override default arg + + container = [] + new_array_a, new_array_b = test_func_7(np_array_a, container, + np.array([4.])) + + assert isinstance(container[0], torch.Tensor) + assert isinstance(container[1], torch.Tensor) + assert isinstance(new_array_a, np.ndarray) + assert np.allclose(new_array_b, np.array([4.]), 1e-3) + + # list arg + @array_converter(to_torch=True, apply_to=('array_a', 'array_b')) + def test_func_8(container, array_a, array_b=[2.]): + container.append(array_a) + container.append(array_b) + return array_a.clone(), array_b.clone() + + container = [] + new_array_a, new_array_b = test_func_8(container, [3.]) + + assert isinstance(container[0], torch.Tensor) + assert isinstance(container[1], torch.Tensor) + assert np.allclose(new_array_a, np.array([3.]), 1e-3) + assert np.allclose(new_array_b, np.array([2.]), 1e-3) + + # number arg + @array_converter(to_torch=True, apply_to=('array_a', 'array_b')) + def test_func_9(container, array_a, array_b=1): + container.append(array_a) + container.append(array_b) + return array_a.clone(), array_b.clone() + + container = [] + new_array_a, new_array_b = test_func_9(container, np_array_a) + + assert isinstance(container[0], torch.FloatTensor) + assert isinstance(container[1], torch.FloatTensor) + assert np.allclose(new_array_a, np_array_a, 1e-3) + assert np.allclose(new_array_b, np.array(1.0), 1e-3) + + # feed kwargs + container = [] + kwargs = {'array_a': [5.], 'array_b': [6.]} + new_array_a, new_array_b = test_func_8(container, **kwargs) + + assert isinstance(container[0], torch.Tensor) + assert isinstance(container[1], torch.Tensor) + assert np.allclose(new_array_a, np.array([5.]), 1e-3) + assert np.allclose(new_array_b, np.array([6.]), 1e-3) + + # feed args and kwargs + container = [] + kwargs = {'array_b': [7.]} + args = (container, [8.]) + new_array_a, new_array_b = test_func_8(*args, **kwargs) + + assert isinstance(container[0], torch.Tensor) + assert isinstance(container[1], torch.Tensor) + assert np.allclose(new_array_a, np.array([8.]), 1e-3) + assert np.allclose(new_array_b, np.array([7.]), 1e-3) + + # wrong template arg type + with pytest.raises(TypeError): + new_array_a, new_array_b = test_func_9(container, 3 + 4j) + + with pytest.raises(TypeError): + new_array_a, new_array_b = test_func_9(container, {}) + + # invalid template arg list + with pytest.raises(TypeError): + new_array_a, new_array_b = test_func_9(container, + [True, np.array([3.0])]) diff --git a/tools/create_data.py b/tools/create_data.py index b761f3e377..9935d6cfb4 100644 --- a/tools/create_data.py +++ b/tools/create_data.py @@ -227,7 +227,7 @@ def waymo_data_prep(root_path, '--out-dir', type=str, default='./data/kitti', - required='False', + required=False, help='name of info pkl') parser.add_argument('--extra-tag', type=str, default='kitti') parser.add_argument( diff --git a/tools/data_converter/kitti_converter.py b/tools/data_converter/kitti_converter.py index 68a96e618a..796db9a02b 100644 --- a/tools/data_converter/kitti_converter.py +++ b/tools/data_converter/kitti_converter.py @@ -4,7 +4,7 @@ from nuscenes.utils.geometry_utils import view_points from pathlib import Path -from mmdet3d.core.bbox import box_np_ops +from mmdet3d.core.bbox import box_np_ops, points_cam2img from .kitti_data_utils import get_kitti_image_info, get_waymo_image_info from .nuscenes_converter import post_process_coords @@ -470,7 +470,7 @@ def get_2d_boxes(info, occluded, mono3d=True): repro_rec['velo_cam3d'] = -1 # no velocity in KITTI center3d = np.array(loc).reshape([1, 3]) - center2d = box_np_ops.points_cam2img( + center2d = points_cam2img( center3d, camera_intrinsic, with_depth=True) repro_rec['center2d'] = center2d.squeeze().tolist() # normalized center2D + depth diff --git a/tools/data_converter/lyft_converter.py b/tools/data_converter/lyft_converter.py index 7d4517fda7..be26f5b8e7 100644 --- a/tools/data_converter/lyft_converter.py +++ b/tools/data_converter/lyft_converter.py @@ -190,8 +190,10 @@ def _fill_trainval_infos(lyft, names[i] = LyftDataset.NameMapping[names[i]] names = np.array(names) - # we need to convert rot to SECOND format. - gt_boxes = np.concatenate([locs, dims, -rots - np.pi / 2], axis=1) + # we need to convert box size to + # the format of our lidar coordinate system + # which is dx, dy, dz (corresponding to l, w, h) + gt_boxes = np.concatenate([locs, dims[:, [1, 0, 2]], rots], axis=1) assert len(gt_boxes) == len( annotations), f'{len(gt_boxes)}, {len(annotations)}' info['gt_boxes'] = gt_boxes diff --git a/tools/data_converter/nuscenes_converter.py b/tools/data_converter/nuscenes_converter.py index f585a11969..a43c8e28d4 100644 --- a/tools/data_converter/nuscenes_converter.py +++ b/tools/data_converter/nuscenes_converter.py @@ -9,7 +9,7 @@ from shapely.geometry import MultiPoint, box from typing import List, Tuple, Union -from mmdet3d.core.bbox.box_np_ops import points_cam2img +from mmdet3d.core.bbox import points_cam2img from mmdet3d.datasets import NuScenesDataset nus_categories = ('car', 'truck', 'trailer', 'bus', 'construction_vehicle', @@ -248,8 +248,10 @@ def _fill_trainval_infos(nusc, if names[i] in NuScenesDataset.NameMapping: names[i] = NuScenesDataset.NameMapping[names[i]] names = np.array(names) - # we need to convert rot to SECOND format. - gt_boxes = np.concatenate([locs, dims, -rots - np.pi / 2], axis=1) + # we need to convert box size to + # the format of our lidar coordinate system + # which is dx, dy, dz (corresponding to l, w, h) + gt_boxes = np.concatenate([locs, dims[:, [1, 0, 2]], rots], axis=1) assert len(gt_boxes) == len( annotations), f'{len(gt_boxes)}, {len(annotations)}' info['gt_boxes'] = gt_boxes diff --git a/tools/data_converter/sunrgbd_data_utils.py b/tools/data_converter/sunrgbd_data_utils.py index 7d2995b93c..8efc7ea140 100644 --- a/tools/data_converter/sunrgbd_data_utils.py +++ b/tools/data_converter/sunrgbd_data_utils.py @@ -41,18 +41,17 @@ def __init__(self, line): self.ymax = data[2] + data[4] self.box2d = np.array([self.xmin, self.ymin, self.xmax, self.ymax]) self.centroid = np.array([data[5], data[6], data[7]]) - self.w = data[8] - self.l = data[9] # noqa: E741 - self.h = data[10] + # data[9] is dx (l), data[8] is dy (w), data[10] is dz (h) + # in our depth coordinate system, + # l corresponds to the size along the x axis + self.size = np.array([data[9], data[8], data[10]]) * 2 self.orientation = np.zeros((3, )) self.orientation[0] = data[11] self.orientation[1] = data[12] - self.heading_angle = -1 * np.arctan2(self.orientation[1], - self.orientation[0]) - self.box3d = np.concatenate([ - self.centroid, - np.array([self.l * 2, self.w * 2, self.h * 2, self.heading_angle]) - ]) + self.heading_angle = np.arctan2(self.orientation[1], + self.orientation[0]) + self.box3d = np.concatenate( + [self.centroid, self.size, self.heading_angle[None]]) class SUNRGBDData(object): From 687723232125ea5306b7ff6c031e291869997edf Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Thu, 29 Jul 2021 15:35:31 +0800 Subject: [PATCH 02/51] [Enhance] Add script for data update (#774) * Fixed wrong config paths and fixed a bug in test * Fixed metafile * Coord sys refactor (main code) * Update test_waymo_dataset.py * Manually resolve conflict * Removed unused lines and fixed imports * remove coord2box and box2coord * update dir_limit_offset * Some minor improvements * Removed some \s in comments * Revert a change * Change Box3DMode to Coord3DMode where points are converted * Fix points_in_bbox function * Fix Imvoxelnet config * Revert adding a line * Fix rotation bug when batch size is 0 * Keep sign of dir_scores as before * Fix several comments * Add a comment * Fix docstring * Add data update scripts * Fix comments --- docs/data_preparation.md | 12 ++- tools/create_data.sh | 11 ++- tools/update_data_coords.py | 167 ++++++++++++++++++++++++++++++++++++ tools/update_data_coords.sh | 22 +++++ 4 files changed, 205 insertions(+), 7 deletions(-) create mode 100644 tools/update_data_coords.py create mode 100644 tools/update_data_coords.sh diff --git a/docs/data_preparation.md b/docs/data_preparation.md index 970f594238..159f248cc9 100644 --- a/docs/data_preparation.md +++ b/docs/data_preparation.md @@ -78,7 +78,7 @@ mmdetection3d ### KITTI -Download KITTI 3D detection data [HERE](http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d). Prepare KITTI data by running +Download KITTI 3D detection data [HERE](http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d). Prepare KITTI data splits by running ```bash mkdir ./data/kitti/ && mkdir ./data/kitti/ImageSets @@ -88,10 +88,20 @@ wget -c https://raw.githubusercontent.com/traveller59/second.pytorch/master/sec wget -c https://raw.githubusercontent.com/traveller59/second.pytorch/master/second/data/ImageSets/train.txt --no-check-certificate --content-disposition -O ./data/kitti/ImageSets/train.txt wget -c https://raw.githubusercontent.com/traveller59/second.pytorch/master/second/data/ImageSets/val.txt --no-check-certificate --content-disposition -O ./data/kitti/ImageSets/val.txt wget -c https://raw.githubusercontent.com/traveller59/second.pytorch/master/second/data/ImageSets/trainval.txt --no-check-certificate --content-disposition -O ./data/kitti/ImageSets/trainval.txt +``` + +Then generate info files by running +``` python tools/create_data.py kitti --root-path ./data/kitti --out-dir ./data/kitti --extra-tag kitti ``` +In an environment using slurm, users may run the following command instead + +``` +sh tools/create_data.sh kitti +``` + ### Waymo Download Waymo open dataset V1.2 [HERE](https://waymo.com/open/download/) and its data split [HERE](https://drive.google.com/drive/folders/18BVuF_RYJF0NjZpt8SnfzANiakoRMf0o?usp=sharing). Then put tfrecord files into corresponding folders in `data/waymo/waymo_format/` and put the data split txt files into `data/waymo/kitti_format/ImageSets`. Download ground truth bin file for validation set [HERE](https://console.cloud.google.com/storage/browser/waymo_open_dataset_v_1_2_0/validation/ground_truth_objects) and put it into `data/waymo/waymo_format/`. A tip is that you can use `gsutil` to download the large-scale dataset with commands. You can take this [tool](https://github.com/RalphMao/Waymo-Dataset-Tool) as an example for more details. Subsequently, prepare waymo data by running diff --git a/tools/create_data.sh b/tools/create_data.sh index 4007de4095..9a57852f71 100755 --- a/tools/create_data.sh +++ b/tools/create_data.sh @@ -5,8 +5,7 @@ export PYTHONPATH=`pwd`:$PYTHONPATH PARTITION=$1 JOB_NAME=$2 -CONFIG=$3 -WORK_DIR=$4 +DATASET=$3 GPUS=${GPUS:-1} GPUS_PER_NODE=${GPUS_PER_NODE:-1} SRUN_ARGS=${SRUN_ARGS:-""} @@ -19,7 +18,7 @@ srun -p ${PARTITION} \ --ntasks-per-node=${GPUS_PER_NODE} \ --kill-on-bad-exit=1 \ ${SRUN_ARGS} \ - python -u tools/create_data.py kitti \ - --root-path ./data/kitti \ - --out-dir ./data/kitti \ - --extra-tag kitti + python -u tools/create_data.py ${DATASET} \ + --root-path ./data/${DATASET} \ + --out-dir ./data/${DATASET} \ + --extra-tag ${DATASET} diff --git a/tools/update_data_coords.py b/tools/update_data_coords.py new file mode 100644 index 0000000000..87165aa808 --- /dev/null +++ b/tools/update_data_coords.py @@ -0,0 +1,167 @@ +import argparse +import mmcv +import numpy as np +import time +from os import path as osp + +from mmdet3d.core.bbox import limit_period + + +def update_sunrgbd_infos(root_dir, out_dir, pkl_files): + print(f'{pkl_files} will be modified because ' + f'of the refactor of the Depth coordinate system.') + if root_dir == out_dir: + print(f'Warning, you are overwriting ' + f'the original data under {root_dir}.') + time.sleep(3) + for pkl_file in pkl_files: + in_path = osp.join(root_dir, pkl_file) + print(f'Reading from input file: {in_path}.') + a = mmcv.load(in_path) + print('Start updating:') + for item in mmcv.track_iter_progress(a): + if 'rotation_y' in item['annos']: + item['annos']['rotation_y'] = -item['annos']['rotation_y'] + item['annos']['gt_boxes_upright_depth'][:, -1:] = \ + -item['annos']['gt_boxes_upright_depth'][:, -1:] + + out_path = osp.join(out_dir, pkl_file) + print(f'Writing to output file: {out_path}.') + mmcv.dump(a, out_path, 'pkl') + + +def update_outdoor_dbinfos(root_dir, out_dir, pkl_files): + print(f'{pkl_files} will be modified because ' + f'of the refactor of the LIDAR coordinate system.') + if root_dir == out_dir: + print(f'Warning, you are overwriting ' + f'the original data under {root_dir}.') + time.sleep(3) + for pkl_file in pkl_files: + in_path = osp.join(root_dir, pkl_file) + print(f'Reading from input file: {in_path}.') + a = mmcv.load(in_path) + print('Start updating:') + for k in a.keys(): + print(f'Updating samples of class {k}:') + for item in mmcv.track_iter_progress(a[k]): + boxes = item['box3d_lidar'].copy() + # swap l, w (or dx, dy) + item['box3d_lidar'][3] = boxes[4] + item['box3d_lidar'][4] = boxes[3] + # change yaw + item['box3d_lidar'][6] = -boxes[6] - np.pi / 2 + item['box3d_lidar'][6] = limit_period( + item['box3d_lidar'][6], period=np.pi * 2) + + out_path = osp.join(out_dir, pkl_file) + print(f'Writing to output file: {out_path}.') + mmcv.dump(a, out_path, 'pkl') + + +def update_nuscenes_or_lyft_infos(root_dir, out_dir, pkl_files): + + print(f'{pkl_files} will be modified because ' + f'of the refactor of the LIDAR coordinate system.') + if root_dir == out_dir: + print(f'Warning, you are overwriting ' + f'the original data under {root_dir}.') + time.sleep(3) + for pkl_file in pkl_files: + in_path = osp.join(root_dir, pkl_file) + print(f'Reading from input file: {in_path}.') + a = mmcv.load(in_path) + print('Start updating:') + for item in mmcv.track_iter_progress(a['infos']): + boxes = item['gt_boxes'].copy() + # swap l, w (or dx, dy) + item['gt_boxes'][:, 3] = boxes[:, 4] + item['gt_boxes'][:, 4] = boxes[:, 3] + # change yaw + item['gt_boxes'][:, 6] = -boxes[:, 6] - np.pi / 2 + item['gt_boxes'][:, 6] = limit_period( + item['gt_boxes'][:, 6], period=np.pi * 2) + + out_path = osp.join(out_dir, pkl_file) + print(f'Writing to output file: {out_path}.') + mmcv.dump(a, out_path, 'pkl') + + +parser = argparse.ArgumentParser(description='Arg parser for data coords ' + 'update due to coords sys refactor.') +parser.add_argument('dataset', metavar='kitti', help='name of the dataset') +parser.add_argument( + '--root-dir', + type=str, + default='./data/kitti', + help='specify the root dir of dataset') +parser.add_argument( + '--version', + type=str, + default='v1.0', + required=False, + help='specify the dataset version, no need for kitti') +parser.add_argument( + '--out-dir', + type=str, + default=None, + required=False, + help='name of info pkl') +args = parser.parse_args() + +if __name__ == '__main__': + if args.out_dir is None: + args.out_dir = args.root_dir + if args.dataset == 'kitti': + # KITTI infos is in CAM coord sys (unchanged) + # KITTI dbinfos is in LIDAR coord sys (changed) + # so we only update dbinfos + pkl_files = ['kitti_dbinfos_train.pkl'] + update_outdoor_dbinfos( + root_dir=args.root_dir, out_dir=args.out_dir, pkl_files=pkl_files) + elif args.dataset == 'nuscenes': + # nuScenes infos is in LIDAR coord sys (changed) + # nuScenes dbinfos is in LIDAR coord sys (changed) + # so we update both infos and dbinfos + pkl_files = ['nuscenes_infos_val.pkl'] + if args.version != 'v1.0-mini': + pkl_files.append('nuscenes_infos_train.pkl') + else: + pkl_files.append('nuscenes_infos_train_tiny.pkl') + update_nuscenes_or_lyft_infos( + root_dir=args.root_dir, out_dir=args.out_dir, pkl_files=pkl_files) + if args.version != 'v1.0-mini': + pkl_files = ['nuscenes_dbinfos_train.pkl'] + update_outdoor_dbinfos( + root_dir=args.root_dir, + out_dir=args.out_dir, + pkl_files=pkl_files) + elif args.dataset == 'lyft': + # Lyft infos is in LIDAR coord sys (changed) + # Lyft has no dbinfos + # so we update infos + pkl_files = ['lyft_infos_train.pkl', 'lyft_infos_val.pkl'] + update_nuscenes_or_lyft_infos( + root_dir=args.root_dir, out_dir=args.out_dir, pkl_files=pkl_files) + elif args.dataset == 'waymo': + # Waymo infos is in CAM coord sys (unchanged) + # Waymo dbinfos is in LIDAR coord sys (changed) + # so we only update dbinfos + pkl_files = ['waymo_dbinfos_train.pkl'] + update_outdoor_dbinfos( + root_dir=args.root_dir, out_dir=args.out_dir, pkl_files=pkl_files) + elif args.dataset == 'scannet': + # ScanNet infos is in DEPTH coord sys (changed) + # but bbox is without yaw + # so ScanNet is unaffected + pass + elif args.dataset == 's3dis': + # Segmentation datasets are not affected + pass + elif args.dataset == 'sunrgbd': + # SUNRGBD infos is in DEPTH coord sys (changed) + # and bbox is with yaw + # so we update infos + pkl_files = ['sunrgbd_infos_train.pkl', 'sunrgbd_infos_val.pkl'] + update_sunrgbd_infos( + root_dir=args.root_dir, out_dir=args.out_dir, pkl_files=pkl_files) diff --git a/tools/update_data_coords.sh b/tools/update_data_coords.sh new file mode 100644 index 0000000000..bd8db62838 --- /dev/null +++ b/tools/update_data_coords.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -x +export PYTHONPATH=`pwd`:$PYTHONPATH + +PARTITION=$1 +DATASET=$2 +GPUS=${GPUS:-1} +GPUS_PER_NODE=${GPUS_PER_NODE:-1} +SRUN_ARGS=${SRUN_ARGS:-""} +JOB_NAME=update_data_coords + +srun -p ${PARTITION} \ + --job-name=${JOB_NAME} \ + --gres=gpu:${GPUS_PER_NODE} \ + --ntasks=${GPUS} \ + --ntasks-per-node=${GPUS_PER_NODE} \ + --kill-on-bad-exit=1 \ + ${SRUN_ARGS} \ + python -u tools/update_data_coords.py ${DATASET} \ + --root-dir ./data/${DATASET} \ + --out-dir ./data/${DATASET} From febd2ebb00b94eb76be2c75ebb6c0ef3545bfae6 Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Thu, 5 Aug 2021 20:38:32 +0800 Subject: [PATCH 03/51] fix import (#839) --- mmdet3d/apis/inference.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mmdet3d/apis/inference.py b/mmdet3d/apis/inference.py index cbd4b14fe6..3230891289 100644 --- a/mmdet3d/apis/inference.py +++ b/mmdet3d/apis/inference.py @@ -7,9 +7,10 @@ from mmcv.runner import load_checkpoint from os import path as osp -from mmdet3d.core import (Box3DMode, Coord3DMode, DepthInstance3DBoxes, - LiDARInstance3DBoxes, show_multi_modality_result, - show_result, show_seg_result) +from mmdet3d.core import (Box3DMode, CameraInstance3DBoxes, Coord3DMode, + DepthInstance3DBoxes, LiDARInstance3DBoxes, + show_multi_modality_result, show_result, + show_seg_result) from mmdet3d.core.bbox import get_box_type from mmdet3d.datasets.pipelines import Compose from mmdet3d.models import build_model From 3f64754eb9112228495651edd0d9ae2aec764f13 Mon Sep 17 00:00:00 2001 From: Xi Liu <75658786+xiliu8006@users.noreply.github.com> Date: Mon, 9 Aug 2021 15:26:31 +0800 Subject: [PATCH 04/51] [Enhance] refactor iou_neg_piecewise_sampler.py (#842) * [Refactor] Main code modification for coordinate system refactor (#677) * [Enhance] Add script for data update (#774) * Fixed wrong config paths and fixed a bug in test * Fixed metafile * Coord sys refactor (main code) * Update test_waymo_dataset.py * Manually resolve conflict * Removed unused lines and fixed imports * remove coord2box and box2coord * update dir_limit_offset * Some minor improvements * Removed some \s in comments * Revert a change * Change Box3DMode to Coord3DMode where points are converted * Fix points_in_bbox function * Fix Imvoxelnet config * Revert adding a line * Fix rotation bug when batch size is 0 * Keep sign of dir_scores as before * Fix several comments * Add a comment * Fix docstring * Add data update scripts * Fix comments * fix import * refactor iou_neg_piecewise_sampler.py * add docstring * modify docstring Co-authored-by: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Co-authored-by: THU17cyz --- .../samplers/iou_neg_piecewise_sampler.py | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py index 955f1546df..cc8ba8bfed 100644 --- a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py +++ b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py @@ -58,8 +58,10 @@ def _sample_neg(self, assign_result, num_expected, **kwargs): neg_inds = torch.nonzero(assign_result.gt_inds == 0, as_tuple=False) if neg_inds.numel() != 0: neg_inds = neg_inds.squeeze(1) - if len(neg_inds) <= num_expected: - return neg_inds + if len(neg_inds) <= 0: + raise NotImplementedError( + 'Not support sampling the negative samples when the length ' + 'of negative samples is 0') else: neg_inds_choice = neg_inds.new_zeros([0]) extend_num = 0 @@ -87,12 +89,38 @@ def _sample_neg(self, assign_result, num_expected, **kwargs): neg_inds_choice = torch.cat( [neg_inds_choice, neg_inds[piece_neg_inds]], dim=0) extend_num += piece_expected_num - len(piece_neg_inds) + + # for the last piece + if piece_inds == self.neg_piece_num - 1: + extend_neg_num = num_expected - len(neg_inds_choice) + # if the numbers of nagetive samples > 0, we will + # randomly select num_expected samples in last piece + if piece_neg_inds.numel() > 0: + rand_idx = torch.randint( + low=0, + high=piece_neg_inds.numel(), + size=(extend_neg_num, )).long() + neg_inds_choice = torch.cat( + [neg_inds_choice, piece_neg_inds[rand_idx]], + dim=0) + # if the numbers of nagetive samples == 0, we will + # randomly select num_expected samples in all + # previous pieces + else: + rand_idx = torch.randint( + low=0, + high=neg_inds_choice.numel(), + size=(extend_neg_num, )).long() + neg_inds_choice = torch.cat( + [neg_inds_choice, neg_inds_choice[rand_idx]], + dim=0) else: piece_choice = self.random_choice(piece_neg_inds, piece_expected_num) neg_inds_choice = torch.cat( [neg_inds_choice, neg_inds[piece_choice]], dim=0) extend_num = 0 + assert len(neg_inds_choice) == num_expected return neg_inds_choice def sample(self, @@ -144,7 +172,6 @@ def sample(self, num_expected_neg = neg_upper_bound neg_inds = self.neg_sampler._sample_neg( assign_result, num_expected_neg, bboxes=bboxes, **kwargs) - neg_inds = neg_inds.unique() sampling_result = SamplingResult(pos_inds, neg_inds, bboxes, gt_bboxes, assign_result, gt_flags) From b2abf1e3b3b01d63d7f017bf39aaaf31598ad8d2 Mon Sep 17 00:00:00 2001 From: Xi Liu <75658786+xiliu8006@users.noreply.github.com> Date: Mon, 9 Aug 2021 15:55:02 +0800 Subject: [PATCH 05/51] [Feature] Add roipooling cuda ops (#843) * [Refactor] Main code modification for coordinate system refactor (#677) * [Enhance] Add script for data update (#774) * Fixed wrong config paths and fixed a bug in test * Fixed metafile * Coord sys refactor (main code) * Update test_waymo_dataset.py * Manually resolve conflict * Removed unused lines and fixed imports * remove coord2box and box2coord * update dir_limit_offset * Some minor improvements * Removed some \s in comments * Revert a change * Change Box3DMode to Coord3DMode where points are converted * Fix points_in_bbox function * Fix Imvoxelnet config * Revert adding a line * Fix rotation bug when batch size is 0 * Keep sign of dir_scores as before * Fix several comments * Add a comment * Fix docstring * Add data update scripts * Fix comments * fix import * add roipooling cuda ops * add roi extractor * add test_roi_extractor unittest * Modify setup.py to install roipooling ops * modify docstring * remove enlarge bbox in roipoint pooling * add_roipooling_ops * modify docstring Co-authored-by: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Co-authored-by: THU17cyz --- .../roi_heads/roi_extractors/__init__.py | 6 +- .../single_roipoint_extractor.py | 63 +++++++ mmdet3d/ops/roipoint_pool3d/__init__.py | 3 + .../ops/roipoint_pool3d/roipoint_pool3d.py | 71 ++++++++ .../roipoint_pool3d/src/roipoint_pool3d.cpp | 66 +++++++ .../src/roipoint_pool3d_kernel.cu | 168 ++++++++++++++++++ setup.py | 5 + .../test_heads/test_roi_extractors.py | 27 ++- 8 files changed, 407 insertions(+), 2 deletions(-) create mode 100644 mmdet3d/models/roi_heads/roi_extractors/single_roipoint_extractor.py create mode 100644 mmdet3d/ops/roipoint_pool3d/__init__.py create mode 100644 mmdet3d/ops/roipoint_pool3d/roipoint_pool3d.py create mode 100644 mmdet3d/ops/roipoint_pool3d/src/roipoint_pool3d.cpp create mode 100644 mmdet3d/ops/roipoint_pool3d/src/roipoint_pool3d_kernel.cu diff --git a/mmdet3d/models/roi_heads/roi_extractors/__init__.py b/mmdet3d/models/roi_heads/roi_extractors/__init__.py index dc504d220d..24a4991047 100644 --- a/mmdet3d/models/roi_heads/roi_extractors/__init__.py +++ b/mmdet3d/models/roi_heads/roi_extractors/__init__.py @@ -1,4 +1,8 @@ from mmdet.models.roi_heads.roi_extractors import SingleRoIExtractor from .single_roiaware_extractor import Single3DRoIAwareExtractor +from .single_roipoint_extractor import Single3DRoIPointExtractor -__all__ = ['SingleRoIExtractor', 'Single3DRoIAwareExtractor'] +__all__ = [ + 'SingleRoIExtractor', 'Single3DRoIAwareExtractor', + 'Single3DRoIPointExtractor' +] diff --git a/mmdet3d/models/roi_heads/roi_extractors/single_roipoint_extractor.py b/mmdet3d/models/roi_heads/roi_extractors/single_roipoint_extractor.py new file mode 100644 index 0000000000..b6b9d8be2e --- /dev/null +++ b/mmdet3d/models/roi_heads/roi_extractors/single_roipoint_extractor.py @@ -0,0 +1,63 @@ +import torch +from torch import nn as nn + +from mmdet3d import ops +from mmdet3d.core.bbox.structures import rotation_3d_in_axis +from mmdet.models.builder import ROI_EXTRACTORS + + +@ROI_EXTRACTORS.register_module() +class Single3DRoIPointExtractor(nn.Module): + """Point-wise roi-aware Extractor. + + Extract Point-wise roi features. + + Args: + roi_layer (dict): The config of roi layer. + """ + + def __init__(self, roi_layer=None): + super(Single3DRoIPointExtractor, self).__init__() + self.roi_layer = self.build_roi_layers(roi_layer) + + def build_roi_layers(self, layer_cfg): + """Build roi layers using `layer_cfg`""" + cfg = layer_cfg.copy() + layer_type = cfg.pop('type') + assert hasattr(ops, layer_type) + layer_cls = getattr(ops, layer_type) + roi_layers = layer_cls(**cfg) + return roi_layers + + def forward(self, feats, coordinate, batch_inds, rois): + """Extract point-wise roi features. + + Args: + feats (torch.FloatTensor): Point-wise features with + shape (batch, npoints, channels) for pooling. + coordinate (torch.FloatTensor): Coordinate of each point. + batch_inds (torch.LongTensor): Indicate the batch of each point. + rois (torch.FloatTensor): Roi boxes with batch indices. + + Returns: + torch.FloatTensor: Pooled features + """ + rois = rois[..., 1:] + rois = rois.view(batch_inds, -1, rois.shape[-1]) + with torch.no_grad(): + pooled_roi_feat, pooled_empty_flag = self.roi_layer( + coordinate, feats, rois) + + # canonical transformation + roi_center = rois[:, :, 0:3] + pooled_roi_feat[:, :, :, 0:3] -= roi_center.unsqueeze(dim=2) + pooled_roi_feat = pooled_roi_feat.view(-1, + pooled_roi_feat.shape[-2], + pooled_roi_feat.shape[-1]) + pooled_roi_feat[:, :, 0:3] = rotation_3d_in_axis( + pooled_roi_feat[:, :, 0:3], + -(rois.view(-1, rois.shape[-1])[:, 6]), + axis=2) + pooled_roi_feat[pooled_empty_flag.view(-1) > 0] = 0 + + return pooled_roi_feat diff --git a/mmdet3d/ops/roipoint_pool3d/__init__.py b/mmdet3d/ops/roipoint_pool3d/__init__.py new file mode 100644 index 0000000000..bbd2f9a0bb --- /dev/null +++ b/mmdet3d/ops/roipoint_pool3d/__init__.py @@ -0,0 +1,3 @@ +from .roipoint_pool3d import RoIPointPool3d + +__all__ = ['RoIPointPool3d'] diff --git a/mmdet3d/ops/roipoint_pool3d/roipoint_pool3d.py b/mmdet3d/ops/roipoint_pool3d/roipoint_pool3d.py new file mode 100644 index 0000000000..9ff10c53a0 --- /dev/null +++ b/mmdet3d/ops/roipoint_pool3d/roipoint_pool3d.py @@ -0,0 +1,71 @@ +from torch import nn as nn +from torch.autograd import Function + +from . import roipoint_pool3d_ext + + +class RoIPointPool3d(nn.Module): + + def __init__(self, num_sampled_points=512): + super().__init__() + """ + Args: + num_sampled_points (int): Number of samples in each roi + """ + self.num_sampled_points = num_sampled_points + + def forward(self, points, point_features, boxes3d): + """ + Args: + points (torch.Tensor): Input points whose shape is BxNx3 + point_features: (B, N, C) + boxes3d: (B, M, 7), [x, y, z, dx, dy, dz, heading] + + Returns: + torch.Tensor: (B, M, 512, 3 + C) pooled_features + torch.Tensor: (B, M) pooled_empty_flag + """ + return RoIPointPool3dFunction.apply(points, point_features, boxes3d, + self.num_sampled_points) + + +class RoIPointPool3dFunction(Function): + + @staticmethod + def forward(ctx, points, point_features, boxes3d, num_sampled_points=512): + """ + Args: + points (torch.Tensor): Input points whose shape is (B, N, 3) + point_features (torch.Tensor): Input points features shape is \ + (B, N, C) + boxes3d (torch.Tensor): Input bounding boxes whose shape is \ + (B, M, 7) + num_sampled_points (int): the num of sampled points + + Returns: + torch.Tensor: (B, M, 512, 3 + C) pooled_features + torch.Tensor: (B, M) pooled_empty_flag + """ + assert points.shape.__len__() == 3 and points.shape[2] == 3 + batch_size, boxes_num, feature_len = points.shape[0], boxes3d.shape[ + 1], point_features.shape[2] + pooled_boxes3d = boxes3d.view(batch_size, -1, 7) + pooled_features = point_features.new_zeros( + (batch_size, boxes_num, num_sampled_points, 3 + feature_len)) + pooled_empty_flag = point_features.new_zeros( + (batch_size, boxes_num)).int() + + roipoint_pool3d_ext.forward(points.contiguous(), + pooled_boxes3d.contiguous(), + point_features.contiguous(), + pooled_features, pooled_empty_flag) + + return pooled_features, pooled_empty_flag + + @staticmethod + def backward(ctx, grad_out): + raise NotImplementedError + + +if __name__ == '__main__': + pass diff --git a/mmdet3d/ops/roipoint_pool3d/src/roipoint_pool3d.cpp b/mmdet3d/ops/roipoint_pool3d/src/roipoint_pool3d.cpp new file mode 100644 index 0000000000..9369b98482 --- /dev/null +++ b/mmdet3d/ops/roipoint_pool3d/src/roipoint_pool3d.cpp @@ -0,0 +1,66 @@ +/* +Modified for +https://github.com/open-mmlab/OpenPCDet/blob/master/pcdet/ops/roipoint_pool3d/src/roipoint_pool3d_kernel.cu +Point cloud feature pooling +Written by Shaoshuai Shi +All Rights Reserved 2018. +*/ +#include +#include + +#define CHECK_CUDA(x) do { \ + if (!x.type().is_cuda()) { \ + fprintf(stderr, "%s must be CUDA tensor at %s:%d\n", #x, __FILE__, __LINE__); \ + exit(-1); \ + } \ +} while (0) +#define CHECK_CONTIGUOUS(x) do { \ + if (!x.is_contiguous()) { \ + fprintf(stderr, "%s must be contiguous tensor at %s:%d\n", #x, __FILE__, __LINE__); \ + exit(-1); \ + } \ +} while (0) +#define CHECK_INPUT(x) CHECK_CUDA(x);CHECK_CONTIGUOUS(x) + + +void roipool3dLauncher(int batch_size, int pts_num, int boxes_num, int feature_in_len, int sampled_pts_num, + const float *xyz, const float *boxes3d, const float *pts_feature, float *pooled_features, int *pooled_empty_flag); + + +int roipool3d_gpu(at::Tensor xyz, at::Tensor boxes3d, at::Tensor pts_feature, at::Tensor pooled_features, at::Tensor pooled_empty_flag){ + // params xyz: (B, N, 3) + // params boxes3d: (B, M, 7) + // params pts_feature: (B, N, C) + // params pooled_features: (B, M, 512, 3+C) + // params pooled_empty_flag: (B, M) + CHECK_INPUT(xyz); + CHECK_INPUT(boxes3d); + CHECK_INPUT(pts_feature); + CHECK_INPUT(pooled_features); + CHECK_INPUT(pooled_empty_flag); + + int batch_size = xyz.size(0); + int pts_num = xyz.size(1); + int boxes_num = boxes3d.size(1); + int feature_in_len = pts_feature.size(2); + int sampled_pts_num = pooled_features.size(2); + + + const float * xyz_data = xyz.data(); + const float * boxes3d_data = boxes3d.data(); + const float * pts_feature_data = pts_feature.data(); + float * pooled_features_data = pooled_features.data(); + int * pooled_empty_flag_data = pooled_empty_flag.data(); + + roipool3dLauncher(batch_size, pts_num, boxes_num, feature_in_len, sampled_pts_num, + xyz_data, boxes3d_data, pts_feature_data, pooled_features_data, pooled_empty_flag_data); + + + + return 1; +} + + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", &roipool3d_gpu, "roipool3d forward (CUDA)"); +} diff --git a/mmdet3d/ops/roipoint_pool3d/src/roipoint_pool3d_kernel.cu b/mmdet3d/ops/roipoint_pool3d/src/roipoint_pool3d_kernel.cu new file mode 100644 index 0000000000..a63a4c7ec4 --- /dev/null +++ b/mmdet3d/ops/roipoint_pool3d/src/roipoint_pool3d_kernel.cu @@ -0,0 +1,168 @@ +/* +Modified from +https://github.com/sshaoshuai/PCDet/blob/master/pcdet/ops/roipoint_pool3d/src/roipoint_pool3d_kernel.cu +Point cloud feature pooling +Written by Shaoshuai Shi +All Rights Reserved 2018. +*/ + +#include +#include + +#define THREADS_PER_BLOCK 256 +#define DIVUP(m,n) ((m) / (n) + ((m) % (n) > 0)) +// #define DEBUG + +__device__ inline void lidar_to_local_coords(float shift_x, float shift_y, + float rz, float &local_x, + float &local_y) { + float cosa = cos(-rz), sina = sin(-rz); + local_x = shift_x * cosa + shift_y * (-sina); + local_y = shift_x * sina + shift_y * cosa; +} + +__device__ inline int check_pt_in_box3d(const float *pt, const float *box3d, + float &local_x, float &local_y) { + // param pt: (x, y, z) + // param box3d: (cx, cy, cz, dx, dy, dz, rz) in LiDAR coordinate, cz in the + // bottom center + float x = pt[0], y = pt[1], z = pt[2]; + float cx = box3d[0], cy = box3d[1], cz = box3d[2]; + float dx = box3d[3], dy = box3d[4], dz = box3d[5], rz = box3d[6]; + cz += dz / 2.0; // shift to the center since cz in box3d is the bottom center + + if (fabsf(z - cz) > dz / 2.0) return 0; + lidar_to_local_coords(x - cx, y - cy, rz, local_x, local_y); + float in_flag = (local_x > -dx / 2.0) & (local_x < dx / 2.0) & + (local_y > -dy / 2.0) & (local_y < dy / 2.0); + return in_flag; +} + +__global__ void assign_pts_to_box3d(int batch_size, int pts_num, int boxes_num, const float *xyz, const float *boxes3d, int *pts_assign){ + // params xyz: (B, N, 3) + // params boxes3d: (B, M, 7) + // params pts_assign: (B, N, M): idx of the corresponding box3d, -1 means background points + int pt_idx = blockIdx.x * blockDim.x + threadIdx.x; + int box_idx = blockIdx.y; + int bs_idx = blockIdx.z; + + if (pt_idx >= pts_num || box_idx >= boxes_num || bs_idx >= batch_size){ + return; + } + int assign_idx = bs_idx * pts_num * boxes_num + pt_idx * boxes_num + box_idx; + pts_assign[assign_idx] = 0; + + int box_offset = bs_idx * boxes_num * 7 + box_idx * 7; + int pt_offset = bs_idx * pts_num * 3 + pt_idx * 3; + + + float local_x = 0, local_y = 0; + int cur_in_flag = check_pt_in_box3d(xyz + pt_offset, boxes3d + box_offset, local_x, local_y); + pts_assign[assign_idx] = cur_in_flag; + // printf("bs=%d, pt=%d, in=%d\n", bs_idx, pt_idx, pts_assign[bs_idx * pts_num + pt_idx]); +} + + +__global__ void get_pooled_idx(int batch_size, int pts_num, int boxes_num, int sampled_pts_num, + const int *pts_assign, int *pts_idx, int *pooled_empty_flag){ + // params xyz: (B, N, 3) + // params pts_feature: (B, N, C) + // params pts_assign: (B, N) + // params pts_idx: (B, M, 512) + // params pooled_empty_flag: (B, M) + + int boxes_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (boxes_idx >= boxes_num){ + return; + } + + int bs_idx = blockIdx.y; + + int cnt = 0; + for (int k = 0; k < pts_num; k++){ + if (pts_assign[bs_idx * pts_num * boxes_num + k * boxes_num + boxes_idx]){ + if (cnt < sampled_pts_num){ + pts_idx[bs_idx * boxes_num * sampled_pts_num + boxes_idx * sampled_pts_num + cnt] = k; + cnt++; + } + else break; + } + } + + if (cnt == 0){ + pooled_empty_flag[bs_idx * boxes_num + boxes_idx] = 1; + } + else if (cnt < sampled_pts_num){ + // duplicate same points for sampling + for (int k = cnt; k < sampled_pts_num; k++){ + int duplicate_idx = k % cnt; + int base_offset = bs_idx * boxes_num * sampled_pts_num + boxes_idx * sampled_pts_num; + pts_idx[base_offset + k] = pts_idx[base_offset + duplicate_idx]; + } + } +} + + +__global__ void roipool3d_forward(int batch_size, int pts_num, int boxes_num, int feature_in_len, int sampled_pts_num, + const float *xyz, const int *pts_idx, const float *pts_feature, + float *pooled_features, int *pooled_empty_flag){ + // params xyz: (B, N, 3) + // params pts_idx: (B, M, 512) + // params pts_feature: (B, N, C) + // params pooled_features: (B, M, 512, 3+C) + // params pooled_empty_flag: (B, M) + + int sample_pt_idx = blockIdx.x * blockDim.x + threadIdx.x; + int box_idx = blockIdx.y; + int bs_idx = blockIdx.z; + + if (sample_pt_idx >= sampled_pts_num || box_idx >= boxes_num || bs_idx >= batch_size){ + return; + } + + if (pooled_empty_flag[bs_idx * boxes_num + box_idx]){ + return; + } + + int temp_idx = bs_idx * boxes_num * sampled_pts_num + box_idx * sampled_pts_num + sample_pt_idx; + int src_pt_idx = pts_idx[temp_idx]; + int dst_feature_offset = temp_idx * (3 + feature_in_len); + + for (int j = 0; j < 3; j++) + pooled_features[dst_feature_offset + j] = xyz[bs_idx * pts_num * 3 + src_pt_idx * 3 + j]; + + int src_feature_offset = bs_idx * pts_num * feature_in_len + src_pt_idx * feature_in_len; + for (int j = 0; j < feature_in_len; j++) + pooled_features[dst_feature_offset + 3 + j] = pts_feature[src_feature_offset + j]; +} + + +void roipool3dLauncher(int batch_size, int pts_num, int boxes_num, int feature_in_len, int sampled_pts_num, + const float *xyz, const float *boxes3d, const float *pts_feature, float *pooled_features, int *pooled_empty_flag){ + + // printf("batch_size=%d, pts_num=%d, boxes_num=%d\n", batch_size, pts_num, boxes_num); + int *pts_assign = NULL; + cudaMalloc(&pts_assign, batch_size * pts_num * boxes_num * sizeof(int)); // (batch_size, N, M) + // cudaMemset(&pts_assign, -1, batch_size * pts_num * boxes_num * sizeof(int)); + + dim3 blocks(DIVUP(pts_num, THREADS_PER_BLOCK), boxes_num, batch_size); // blockIdx.x(col), blockIdx.y(row) + dim3 threads(THREADS_PER_BLOCK); + assign_pts_to_box3d<<>>(batch_size, pts_num, boxes_num, xyz, boxes3d, pts_assign); + + int *pts_idx = NULL; + cudaMalloc(&pts_idx, batch_size * boxes_num * sampled_pts_num * sizeof(int)); // (batch_size, M, sampled_pts_num) + + dim3 blocks2(DIVUP(boxes_num, THREADS_PER_BLOCK), batch_size); // blockIdx.x(col), blockIdx.y(row) + get_pooled_idx<<>>(batch_size, pts_num, boxes_num, sampled_pts_num, pts_assign, pts_idx, pooled_empty_flag); + + dim3 blocks_pool(DIVUP(sampled_pts_num, THREADS_PER_BLOCK), boxes_num, batch_size); + roipool3d_forward<<>>(batch_size, pts_num, boxes_num, feature_in_len, sampled_pts_num, + xyz, pts_idx, pts_feature, pooled_features, pooled_empty_flag); + + cudaFree(pts_assign); + cudaFree(pts_idx); + +#ifdef DEBUG + cudaDeviceSynchronize(); // for using printf in kernel function +#endif +} diff --git a/setup.py b/setup.py index e0d80c8a6c..e46739a631 100644 --- a/setup.py +++ b/setup.py @@ -270,6 +270,11 @@ def add_mim_extention(): 'src/roiaware_pool3d_kernel.cu', 'src/points_in_boxes_cuda.cu', ]), + make_cuda_ext( + name='roipoint_pool3d_ext', + module='mmdet3d.ops.roipoint_pool3d', + sources=['src/roipoint_pool3d.cpp'], + sources_cuda=['src/roipoint_pool3d_kernel.cu']), make_cuda_ext( name='ball_query_ext', module='mmdet3d.ops.ball_query', diff --git a/tests/test_models/test_heads/test_roi_extractors.py b/tests/test_models/test_heads/test_roi_extractors.py index 1316aa3594..8ff941443d 100644 --- a/tests/test_models/test_heads/test_roi_extractors.py +++ b/tests/test_models/test_heads/test_roi_extractors.py @@ -2,7 +2,8 @@ import pytest import torch -from mmdet3d.models.roi_heads.roi_extractors import Single3DRoIAwareExtractor +from mmdet3d.models.roi_heads.roi_extractors import (Single3DRoIAwareExtractor, + Single3DRoIPointExtractor) def test_single_roiaware_extractor(): @@ -29,3 +30,27 @@ def test_single_roiaware_extractor(): assert pooled_feats.shape == torch.Size([2, 4, 4, 4, 3]) assert torch.allclose(pooled_feats.sum(), torch.tensor(51.100).cuda(), 1e-3) + + +def test_single_roipoint_extractor(): + if not torch.cuda.is_available(): + pytest.skip('test requires GPU and torch+cuda') + + roi_layer_cfg = dict( + type='RoIPointPool3d', num_sampled_points=512, pool_extra_width=0) + + self = Single3DRoIPointExtractor(roi_layer=roi_layer_cfg) + + feats = torch.tensor( + [[1, 2, 3.3], [1.2, 2.5, 3.0], [0.8, 2.1, 3.5], [1.6, 2.6, 3.6], + [0.8, 1.2, 3.9], [-9.2, 21.0, 18.2], [3.8, 7.9, 6.3], + [4.7, 3.5, -12.2], [3.8, 7.6, -2], [-10.6, -12.9, -20], [-16, -18, 9], + [-21.3, -52, -5], [0, 0, 0], [6, 7, 8], [-2, -3, -4]], + dtype=torch.float32).unsqueeze(0).cuda() + points = feats.clone() + batch_inds = feats.shape[0] + rois = torch.tensor([[0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 0.3], + [0, -10.0, 23.0, 16.0, 10, 20, 20, 0.5]], + dtype=torch.float32).cuda() + pooled_feats = self(feats, points, batch_inds, rois) + assert pooled_feats.shape == torch.Size([2, 512, 6]) From 5a07dfe1d23c515bc8070e770c8bf8c67e2c6254 Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Wed, 11 Aug 2021 22:12:59 +0800 Subject: [PATCH 06/51] [Refactor] Refactor code structure and docstrings (#803) * refactor points_in_boxes * Merge same functions of three boxes * More docstring fixes and unify x/y/z size * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Remove None in function param type * Fix unittest * Add comments for NMS functions * Merge methods of Points * Add unittest * Add optional and default value * Fix box conversion and add unittest * Fix comments * Add unit test * Indent * Fix CI * Remove useless \\ * Remove useless \\ * Remove useless \\ * Remove useless \\ * Remove useless \\ * Add unit test for box bev * More unit tests and refine docstrings in box_np_ops * Fix comment * Add deprecation warning --- docs/datasets/scannet_det.md | 4 +- mmdet3d/apis/inference.py | 20 +- mmdet3d/apis/test.py | 4 +- mmdet3d/core/anchor/anchor_3d_generator.py | 89 +++++---- mmdet3d/core/bbox/box_np_ops.py | 54 +++-- .../bbox/coders/anchor_free_bbox_coder.py | 2 +- .../bbox/coders/centerpoint_bbox_coders.py | 21 +- .../bbox/coders/delta_xyzwhlr_bbox_coder.py | 12 +- .../bbox/coders/groupfree3d_bbox_coder.py | 11 +- .../coders/partial_bin_based_bbox_coder.py | 2 +- .../bbox/iou_calculators/iou3d_calculator.py | 40 ++-- .../samplers/iou_neg_piecewise_sampler.py | 2 +- mmdet3d/core/bbox/structures/base_box3d.py | 184 ++++++++++++----- mmdet3d/core/bbox/structures/box_3d_mode.py | 11 +- mmdet3d/core/bbox/structures/cam_box3d.py | 125 +++++------- mmdet3d/core/bbox/structures/coord_3d_mode.py | 15 +- mmdet3d/core/bbox/structures/depth_box3d.py | 71 ++----- mmdet3d/core/bbox/structures/lidar_box3d.py | 61 +----- mmdet3d/core/bbox/transforms.py | 8 +- mmdet3d/core/evaluation/indoor_eval.py | 17 +- .../core/evaluation/kitti_utils/rotate_iou.py | 9 +- mmdet3d/core/evaluation/lyft_eval.py | 8 +- mmdet3d/core/evaluation/seg_eval.py | 2 +- .../waymo_utils/prediction_kitti_to_waymo.py | 2 +- mmdet3d/core/points/base_points.py | 59 ++++-- mmdet3d/core/points/cam_points.py | 43 ++-- mmdet3d/core/points/depth_points.py | 40 ++-- mmdet3d/core/points/lidar_points.py | 40 ++-- mmdet3d/core/post_processing/box3d_nms.py | 16 +- mmdet3d/core/utils/array_converter.py | 20 +- mmdet3d/core/utils/gaussian.py | 6 +- mmdet3d/core/visualizer/image_vis.py | 14 +- mmdet3d/core/visualizer/open3d_vis.py | 144 +++++++------ mmdet3d/core/visualizer/show_result.py | 30 +-- mmdet3d/core/voxel/voxel_generator.py | 40 ++-- mmdet3d/datasets/custom_3d.py | 23 ++- mmdet3d/datasets/custom_3d_seg.py | 18 +- mmdet3d/datasets/kitti2d_dataset.py | 5 +- mmdet3d/datasets/kitti_dataset.py | 50 ++--- mmdet3d/datasets/kitti_mono_dataset.py | 39 ++-- mmdet3d/datasets/lyft_dataset.py | 38 ++-- mmdet3d/datasets/nuscenes_dataset.py | 39 ++-- mmdet3d/datasets/nuscenes_mono_dataset.py | 43 ++-- .../datasets/pipelines/data_augment_utils.py | 25 +-- mmdet3d/datasets/pipelines/dbsampler.py | 18 +- mmdet3d/datasets/pipelines/formating.py | 29 ++- mmdet3d/datasets/pipelines/loading.py | 57 +++--- mmdet3d/datasets/pipelines/test_time_aug.py | 21 +- mmdet3d/datasets/pipelines/transforms_3d.py | 89 +++++---- mmdet3d/datasets/scannet_dataset.py | 14 +- mmdet3d/datasets/sunrgbd_dataset.py | 19 +- mmdet3d/datasets/utils.py | 6 +- mmdet3d/datasets/waymo_dataset.py | 31 +-- mmdet3d/models/backbones/pointnet2_sa_msg.py | 2 +- mmdet3d/models/backbones/pointnet2_sa_ssg.py | 6 +- mmdet3d/models/decode_heads/decode_head.py | 19 +- mmdet3d/models/decode_heads/paconv_head.py | 2 +- mmdet3d/models/decode_heads/pointnet2_head.py | 2 +- mmdet3d/models/dense_heads/anchor3d_head.py | 22 +- .../dense_heads/anchor_free_mono3d_head.py | 87 ++++---- .../models/dense_heads/centerpoint_head.py | 110 +++++----- .../models/dense_heads/fcos_mono3d_head.py | 54 ++--- .../models/dense_heads/groupfree3d_head.py | 32 +-- mmdet3d/models/dense_heads/parta2_rpn_head.py | 12 +- .../models/dense_heads/shape_aware_head.py | 48 ++--- mmdet3d/models/dense_heads/ssd_3d_head.py | 24 +-- mmdet3d/models/dense_heads/train_mixins.py | 2 +- mmdet3d/models/dense_heads/vote_head.py | 24 +-- mmdet3d/models/detectors/centerpoint.py | 3 +- mmdet3d/models/detectors/groupfree3dnet.py | 6 +- mmdet3d/models/detectors/h3dnet.py | 6 +- mmdet3d/models/detectors/imvotenet.py | 20 +- mmdet3d/models/detectors/mvx_two_stage.py | 14 +- .../models/detectors/single_stage_mono3d.py | 5 +- mmdet3d/models/detectors/votenet.py | 6 +- mmdet3d/models/fusion_layers/point_fusion.py | 4 +- .../models/losses/axis_aligned_iou_loss.py | 2 +- mmdet3d/models/losses/chamfer_distance.py | 16 +- .../models/middle_encoders/sparse_encoder.py | 26 +-- mmdet3d/models/model_utils/transformer.py | 47 ++--- mmdet3d/models/model_utils/vote_module.py | 33 +-- .../roi_heads/bbox_heads/h3d_bbox_head.py | 48 ++--- mmdet3d/models/roi_heads/h3d_roi_head.py | 10 +- .../mask_heads/pointwise_semantic_head.py | 22 +- .../roi_heads/mask_heads/primitive_head.py | 22 +- mmdet3d/models/segmentors/base.py | 2 +- mmdet3d/models/segmentors/encoder_decoder.py | 12 +- mmdet3d/models/utils/clip_sigmoid.py | 4 +- mmdet3d/models/utils/mlp.py | 10 +- .../models/voxel_encoders/pillar_encoder.py | 2 +- mmdet3d/models/voxel_encoders/utils.py | 11 +- .../models/voxel_encoders/voxel_encoder.py | 81 ++++---- mmdet3d/ops/__init__.py | 8 +- .../furthest_point_sample/points_sampler.py | 6 +- mmdet3d/ops/furthest_point_sample/utils.py | 2 +- mmdet3d/ops/group_points/group_points.py | 20 +- mmdet3d/ops/iou3d/iou3d_utils.py | 27 ++- .../ops/pointnet_modules/paconv_sa_module.py | 7 +- .../ops/pointnet_modules/point_fp_module.py | 2 +- .../ops/pointnet_modules/point_sa_module.py | 79 ++++---- mmdet3d/ops/roiaware_pool3d/__init__.py | 8 +- .../ops/roiaware_pool3d/points_in_boxes.py | 33 +-- .../src/points_in_boxes_cpu.cpp | 14 +- .../src/points_in_boxes_cuda.cu | 74 +++---- .../roiaware_pool3d/src/roiaware_pool3d.cpp | 18 +- .../src/roiaware_pool3d_kernel.cu | 28 +-- mmdet3d/ops/sparse_block.py | 24 +-- .../test_roiaware_pool3d.py | 18 +- tests/test_models/test_forward.py | 2 +- tests/test_models/test_heads/test_heads.py | 16 +- tests/test_utils/test_box3d.py | 189 ++++++++++++++++-- tests/test_utils/test_box_np_ops.py | 18 ++ tests/test_utils/test_coord_3d_mode.py | 20 +- tests/test_utils/test_points.py | 4 + tools/create_data.py | 11 +- tools/data_converter/create_gt_database.py | 14 +- tools/data_converter/indoor_converter.py | 9 +- tools/data_converter/kitti_converter.py | 35 ++-- tools/data_converter/lyft_converter.py | 14 +- tools/data_converter/nuscenes_converter.py | 21 +- tools/data_converter/s3dis_data_utils.py | 21 +- tools/data_converter/scannet_data_utils.py | 21 +- tools/data_converter/sunrgbd_data_utils.py | 14 +- tools/data_converter/waymo_converter.py | 8 +- 124 files changed, 1850 insertions(+), 1583 deletions(-) diff --git a/docs/datasets/scannet_det.md b/docs/datasets/scannet_det.md index 48f0c90063..b8075b3590 100644 --- a/docs/datasets/scannet_det.md +++ b/docs/datasets/scannet_det.md @@ -110,8 +110,8 @@ def export(mesh_file, instance_ids[verts] = object_id if object_id not in object_id_to_label_id: object_id_to_label_id[object_id] = label_ids[verts][0] - # bbox format is [x, y, z, dx, dy, dz, label_id] - # [x, y, z] is gravity center of bbox, [dx, dy, dz] is axis-aligned + # bbox format is [x, y, z, x_size, y_size, z_size, label_id] + # [x, y, z] is gravity center of bbox, [x_size, y_size, z_size] is axis-aligned # [label_id] is semantic label id in 'nyu40id' standard # Note: since 3D bbox is axis-aligned, the yaw is 0. unaligned_bboxes = extract_bbox(mesh_vertices, object_id_to_segs, diff --git a/mmdet3d/apis/inference.py b/mmdet3d/apis/inference.py index 3230891289..35d10bad54 100644 --- a/mmdet3d/apis/inference.py +++ b/mmdet3d/apis/inference.py @@ -459,15 +459,17 @@ def show_result_meshlab(data, data (dict): Contain data from pipeline. result (dict): Predicted result from model. out_dir (str): Directory to save visualized result. - score_thr (float): Minimum score of bboxes to be shown. Default: 0.0 - show (bool): Visualize the results online. Defaults to False. - snapshot (bool): Whether to save the online results. Defaults to False. - task (str): Distinguish which task result to visualize. Currently we - support 3D detection, multi-modality detection and 3D segmentation. - Defaults to 'det'. - palette (list[list[int]]] | np.ndarray | None): The palette of - segmentation map. If None is given, random palette will be - generated. Defaults to None. + score_thr (float, optional): Minimum score of bboxes to be shown. + Default: 0.0 + show (bool, optional): Visualize the results online. Defaults to False. + snapshot (bool, optional): Whether to save the online results. + Defaults to False. + task (str, optional): Distinguish which task result to visualize. + Currently we support 3D detection, multi-modality detection and + 3D segmentation. Defaults to 'det'. + palette (list[list[int]]] | np.ndarray, optional): The palette + of segmentation map. If None is given, random palette will be + generated. Defaults to None. """ assert task in ['det', 'multi_modality-det', 'seg', 'mono-det'], \ f'unsupported visualization task {task}' diff --git a/mmdet3d/apis/test.py b/mmdet3d/apis/test.py index f60c56a119..858fe3dd26 100644 --- a/mmdet3d/apis/test.py +++ b/mmdet3d/apis/test.py @@ -21,9 +21,9 @@ def single_gpu_test(model, Args: model (nn.Module): Model to be tested. data_loader (nn.Dataloader): Pytorch data loader. - show (bool): Whether to save viualization results. + show (bool, optional): Whether to save viualization results. Default: True. - out_dir (str): The path to save visualization results. + out_dir (str, optional): The path to save visualization results. Default: None. Returns: diff --git a/mmdet3d/core/anchor/anchor_3d_generator.py b/mmdet3d/core/anchor/anchor_3d_generator.py index 752a5cbcba..1ea8426e51 100644 --- a/mmdet3d/core/anchor/anchor_3d_generator.py +++ b/mmdet3d/core/anchor/anchor_3d_generator.py @@ -18,15 +18,21 @@ class Anchor3DRangeGenerator(object): ranges (list[list[float]]): Ranges of different anchors. The ranges are the same across different feature levels. But may vary for different anchor sizes if size_per_range is True. - sizes (list[list[float]]): 3D sizes of anchors. - scales (list[int]): Scales of anchors in different feature levels. - rotations (list[float]): Rotations of anchors in a feature grid. - custom_values (tuple[float]): Customized values of that anchor. For - example, in nuScenes the anchors have velocities. - reshape_out (bool): Whether to reshape the output into (N x 4). - size_per_range: Whether to use separate ranges for different sizes. - If size_per_range is True, the ranges should have the same length - as the sizes, if not, it will be duplicated. + sizes (list[list[float]], optional): 3D sizes of anchors. + Defaults to [[3.9, 1.6, 1.56]]. + scales (list[int], optional): Scales of anchors in different feature + levels. Defaults to [1]. + rotations (list[float], optional): Rotations of anchors in a feature + grid. Defaults to [0, 1.5707963]. + custom_values (tuple[float], optional): Customized values of that + anchor. For example, in nuScenes the anchors have velocities. + Defaults to (). + reshape_out (bool, optional): Whether to reshape the output into + (N x 4). Defaults to True. + size_per_range (bool, optional): Whether to use separate ranges for + different sizes. If size_per_range is True, the ranges should have + the same length as the sizes, if not, it will be duplicated. + Defaults to True. """ def __init__(self, @@ -85,13 +91,14 @@ def grid_anchors(self, featmap_sizes, device='cuda'): Args: featmap_sizes (list[tuple]): List of feature map sizes in multiple feature levels. - device (str): Device where the anchors will be put on. + device (str, optional): Device where the anchors will be put on. + Defaults to 'cuda'. Returns: - list[torch.Tensor]: Anchors in multiple feature levels. \ - The sizes of each tensor should be [N, 4], where \ - N = width * height * num_base_anchors, width and height \ - are the sizes of the corresponding feature lavel, \ + list[torch.Tensor]: Anchors in multiple feature levels. + The sizes of each tensor should be [N, 4], where + N = width * height * num_base_anchors, width and height + are the sizes of the corresponding feature lavel, num_base_anchors is the number of anchors for that level. """ assert self.num_levels == len(featmap_sizes) @@ -160,14 +167,18 @@ def anchors_single_range(self, shape [6]. The order is consistent with that of anchors, i.e., (x_min, y_min, z_min, x_max, y_max, z_max). scale (float | int, optional): The scale factor of anchors. - sizes (list[list] | np.ndarray | torch.Tensor): Anchor size with - shape [N, 3], in order of x, y, z. - rotations (list[float] | np.ndarray | torch.Tensor): Rotations of - anchors in a single feature grid. + Defaults to 1. + sizes (list[list] | np.ndarray | torch.Tensor, optional): + Anchor size with shape [N, 3], in order of x, y, z. + Defaults to [[3.9, 1.6, 1.56]]. + rotations (list[float] | np.ndarray | torch.Tensor, optional): + Rotations of anchors in a single feature grid. + Defaults to [0, 1.5707963]. device (str): Devices that the anchors will be put on. + Defaults to 'cuda'. Returns: - torch.Tensor: Anchors with shape \ + torch.Tensor: Anchors with shape [*feature_size, num_sizes, num_rots, 7]. """ if len(feature_size) == 2: @@ -230,10 +241,10 @@ class AlignedAnchor3DRangeGenerator(Anchor3DRangeGenerator): up corner to distribute anchors. Args: - anchor_corner (bool): Whether to align with the corner of the voxel - grid. By default it is False and the anchor's center will be + anchor_corner (bool, optional): Whether to align with the corner of the + voxel grid. By default it is False and the anchor's center will be the same as the corresponding voxel's center, which is also the - center of the corresponding greature grid. + center of the corresponding greature grid. Defaults to False. """ def __init__(self, align_corner=False, **kwargs): @@ -255,15 +266,18 @@ def anchors_single_range(self, anchor_range (torch.Tensor | list[float]): Range of anchors with shape [6]. The order is consistent with that of anchors, i.e., (x_min, y_min, z_min, x_max, y_max, z_max). - scale (float | int, optional): The scale factor of anchors. - sizes (list[list] | np.ndarray | torch.Tensor): Anchor size with - shape [N, 3], in order of x, y, z. - rotations (list[float] | np.ndarray | torch.Tensor): Rotations of - anchors in a single feature grid. - device (str): Devices that the anchors will be put on. + scale (float | int): The scale factor of anchors. + sizes (list[list] | np.ndarray | torch.Tensor, optional): + Anchor size with shape [N, 3], in order of x, y, z. + Defaults to [[3.9, 1.6, 1.56]]. + rotations (list[float] | np.ndarray | torch.Tensor, optional): + Rotations of anchors in a single feature grid. + Defaults to [0, 1.5707963]. + device (str, optional): Devices that the anchors will be put on. + Defaults to 'cuda'. Returns: - torch.Tensor: Anchors with shape \ + torch.Tensor: Anchors with shape [*feature_size, num_sizes, num_rots, 7]. """ if len(feature_size) == 2: @@ -333,7 +347,7 @@ class AlignedAnchor3DRangeGeneratorPerCls(AlignedAnchor3DRangeGenerator): Note that feature maps of different classes may be different. Args: - kwargs (dict): Arguments are the same as those in \ + kwargs (dict): Arguments are the same as those in :class:`AlignedAnchor3DRangeGenerator`. """ @@ -346,15 +360,16 @@ def grid_anchors(self, featmap_sizes, device='cuda'): """Generate grid anchors in multiple feature levels. Args: - featmap_sizes (list[tuple]): List of feature map sizes for \ + featmap_sizes (list[tuple]): List of feature map sizes for different classes in a single feature level. - device (str): Device where the anchors will be put on. + device (str, optional): Device where the anchors will be put on. + Defaults to 'cuda'. Returns: - list[list[torch.Tensor]]: Anchors in multiple feature levels. \ - Note that in this anchor generator, we currently only \ - support single feature level. The sizes of each tensor \ - should be [num_sizes/ranges*num_rots*featmap_size, \ + list[list[torch.Tensor]]: Anchors in multiple feature levels. + Note that in this anchor generator, we currently only + support single feature level. The sizes of each tensor + should be [num_sizes/ranges*num_rots*featmap_size, box_code_size]. """ multi_level_anchors = [] @@ -370,7 +385,7 @@ def multi_cls_grid_anchors(self, featmap_sizes, scale, device='cuda'): This function is usually called by method ``self.grid_anchors``. Args: - featmap_sizes (list[tuple]): List of feature map sizes for \ + featmap_sizes (list[tuple]): List of feature map sizes for different classes in a single feature level. scale (float): Scale factor of the anchors in the current level. device (str, optional): Device the tensor will be put on. diff --git a/mmdet3d/core/bbox/box_np_ops.py b/mmdet3d/core/bbox/box_np_ops.py index c8d55d6654..8990713a5d 100644 --- a/mmdet3d/core/bbox/box_np_ops.py +++ b/mmdet3d/core/bbox/box_np_ops.py @@ -1,5 +1,8 @@ # TODO: clean the functions in this file and move the APIs into box structures # in the future +# NOTICE: All functions in this file are valid for LiDAR or depth boxes only +# if we use default parameters. + import numba import numpy as np @@ -46,13 +49,13 @@ def box_camera_to_lidar(data, r_rect, velo2cam): np.ndarray, shape=[N, 3]: Boxes in lidar coordinate. """ xyz = data[:, 0:3] - dx, dy, dz = data[:, 3:4], data[:, 4:5], data[:, 5:6] + x_size, y_size, z_size = data[:, 3:4], data[:, 4:5], data[:, 5:6] r = data[:, 6:7] xyz_lidar = camera_to_lidar(xyz, r_rect, velo2cam) # yaw and dims also needs to be converted r_new = -r - np.pi / 2 r_new = limit_period(r_new, period=np.pi * 2) - return np.concatenate([xyz_lidar, dx, dz, dy, r_new], axis=1) + return np.concatenate([xyz_lidar, x_size, z_size, y_size, r_new], axis=1) def corners_nd(dims, origin=0.5): @@ -91,7 +94,7 @@ def corners_nd(dims, origin=0.5): def center_to_corner_box2d(centers, dims, angles=None, origin=0.5): """Convert kitti locations, dimensions and angles to corners. - format: center(xy), dims(xy), angles(clockwise when positive) + format: center(xy), dims(xy), angles(counterclockwise when positive) Args: centers (np.ndarray): Locations in kitti label file with shape (N, 2). @@ -186,7 +189,7 @@ def center_to_corner_box3d(centers, np.ndarray: Corners with the shape of (N, 8, 3). """ # 'length' in kitti format is in x axis. - # yzx(hwl)(kitti label file)<->xyz(lhw)(camera)<->z(-x)(-y)(wlh)(lidar) + # yzx(hwl)(kitti label file)<->xyz(lhw)(camera)<->z(-x)(-y)(lwh)(lidar) # center in kitti format is [0.5, 1.0, 0.5] in xyz. corners = corners_nd(dims, origin=origin) # corners: [N, 8, 3] @@ -347,7 +350,10 @@ def corner_to_surfaces_3d(corners): def points_in_rbbox(points, rbbox, z_axis=2, origin=(0.5, 0.5, 0)): - """Check points in rotated bbox and return indicces. + """Check points in rotated bbox and return indices. + + Note: + This function is for counterclockwise boxes. Args: points (np.ndarray, shape=[N, 3+dim]): Points to query. @@ -403,7 +409,7 @@ def create_anchors_3d_range(feature_size, rotations (list[float] | np.ndarray | torch.Tensor, optional): Rotations of anchors in a single feature grid. Defaults to (0, np.pi / 2). - dtype (type, optional): Data type. Default to np.float32. + dtype (type, optional): Data type. Defaults to np.float32. Returns: np.ndarray: Range based anchors with shape of @@ -477,6 +483,9 @@ def iou_jit(boxes, query_boxes, mode='iou', eps=0.0): """Calculate box iou. Note that jit version runs ~10x faster than the box_overlaps function in mmdet3d.core.evaluation. + Note: + This function is for counterclockwise boxes. + Args: boxes (np.ndarray): Input bounding boxes with shape of (N, 4). query_boxes (np.ndarray): Query boxes with shape of (K, 4). @@ -514,7 +523,10 @@ def iou_jit(boxes, query_boxes, mode='iou', eps=0.0): def projection_matrix_to_CRT_kitti(proj): - """Split projection matrix of kitti. + """Split projection matrix of KITTI. + + Note: + This function is for KITTI only. P = C @ [R|T] C is upper triangular matrix, so we need to inverse CR and use QR @@ -540,6 +552,9 @@ def projection_matrix_to_CRT_kitti(proj): def remove_outside_points(points, rect, Trv2c, P2, image_shape): """Remove points which are outside of image. + Note: + This function is for KITTI only. + Args: points (np.ndarray, shape=[N, 3+dims]): Total points. rect (np.ndarray, shape=[4, 4]): Matrix to project points in @@ -690,7 +705,7 @@ def points_in_convex_polygon_3d_jit(points, @numba.jit -def points_in_convex_polygon_jit(points, polygon, clockwise=True): +def points_in_convex_polygon_jit(points, polygon, clockwise=False): """Check points is in 2d convex polygons. True when point in polygon. Args: @@ -746,10 +761,13 @@ def boxes3d_to_corners3d_lidar(boxes3d, bottom_center=True): |/ |/ 2 -------- 1 + Note: + This function is for LiDAR boxes only. + Args: boxes3d (np.ndarray): Boxes with shape of (N, 7) - [x, y, z, dx, dy, dz, ry] in LiDAR coords, see the definition of - ry in KITTI dataset. + [x, y, z, x_size, y_size, z_size, ry] in LiDAR coords, + see the definition of ry in KITTI dataset. bottom_center (bool, optional): Whether z is on the bottom center of object. Defaults to True. @@ -757,25 +775,25 @@ def boxes3d_to_corners3d_lidar(boxes3d, bottom_center=True): np.ndarray: Box corners with the shape of [N, 8, 3]. """ boxes_num = boxes3d.shape[0] - dx, dy, dz = boxes3d[:, 3], boxes3d[:, 4], boxes3d[:, 5] + x_size, y_size, z_size = boxes3d[:, 3], boxes3d[:, 4], boxes3d[:, 5] x_corners = np.array([ - dx / 2., -dx / 2., -dx / 2., dx / 2., dx / 2., -dx / 2., -dx / 2., - dx / 2. + x_size / 2., -x_size / 2., -x_size / 2., x_size / 2., x_size / 2., + -x_size / 2., -x_size / 2., x_size / 2. ], dtype=np.float32).T y_corners = np.array([ - -dy / 2., -dy / 2., dy / 2., dy / 2., -dy / 2., -dy / 2., dy / 2., - dy / 2. + -y_size / 2., -y_size / 2., y_size / 2., y_size / 2., -y_size / 2., + -y_size / 2., y_size / 2., y_size / 2. ], dtype=np.float32).T if bottom_center: z_corners = np.zeros((boxes_num, 8), dtype=np.float32) - z_corners[:, 4:8] = dz.reshape(boxes_num, 1).repeat( + z_corners[:, 4:8] = z_size.reshape(boxes_num, 1).repeat( 4, axis=1) # (N, 8) else: z_corners = np.array([ - -dz / 2., -dz / 2., -dz / 2., -dz / 2., dz / 2., dz / 2., dz / 2., - dz / 2. + -z_size / 2., -z_size / 2., -z_size / 2., -z_size / 2., + z_size / 2., z_size / 2., z_size / 2., z_size / 2. ], dtype=np.float32).T diff --git a/mmdet3d/core/bbox/coders/anchor_free_bbox_coder.py b/mmdet3d/core/bbox/coders/anchor_free_bbox_coder.py index 8bf8160434..b42e02b716 100644 --- a/mmdet3d/core/bbox/coders/anchor_free_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/anchor_free_bbox_coder.py @@ -24,7 +24,7 @@ def encode(self, gt_bboxes_3d, gt_labels_3d): """Encode ground truth to prediction targets. Args: - gt_bboxes_3d (BaseInstance3DBoxes): Ground truth bboxes \ + gt_bboxes_3d (BaseInstance3DBoxes): Ground truth bboxes with shape (n, 7). gt_labels_3d (torch.Tensor): Ground truth classes. diff --git a/mmdet3d/core/bbox/coders/centerpoint_bbox_coders.py b/mmdet3d/core/bbox/coders/centerpoint_bbox_coders.py index 8180ff8faf..9506de3229 100644 --- a/mmdet3d/core/bbox/coders/centerpoint_bbox_coders.py +++ b/mmdet3d/core/bbox/coders/centerpoint_bbox_coders.py @@ -12,12 +12,12 @@ class CenterPointBBoxCoder(BaseBBoxCoder): pc_range (list[float]): Range of point cloud. out_size_factor (int): Downsample factor of the model. voxel_size (list[float]): Size of voxel. - post_center_range (list[float]): Limit of the center. + post_center_range (list[float], optional): Limit of the center. Default: None. - max_num (int): Max number to be kept. Default: 100. - score_threshold (float): Threshold to filter boxes based on score. - Default: None. - code_size (int): Code size of bboxes. Default: 9 + max_num (int, optional): Max number to be kept. Default: 100. + score_threshold (float, optional): Threshold to filter boxes + based on score. Default: None. + code_size (int, optional): Code size of bboxes. Default: 9 """ def __init__(self, @@ -44,7 +44,8 @@ def _gather_feat(self, feats, inds, feat_masks=None): feats (torch.Tensor): Features to be transposed and gathered with the shape of [B, 2, W, H]. inds (torch.Tensor): Indexes with the shape of [B, N]. - feat_masks (torch.Tensor): Mask of the feats. Default: None. + feat_masks (torch.Tensor, optional): Mask of the feats. + Default: None. Returns: torch.Tensor: Gathered feats. @@ -63,7 +64,7 @@ def _topk(self, scores, K=80): Args: scores (torch.Tensor): scores with the shape of [B, N, W, H]. - K (int): Number to be kept. Defaults to 80. + K (int, optional): Number to be kept. Defaults to 80. Returns: tuple[torch.Tensor] @@ -134,9 +135,9 @@ def decode(self, dim (torch.Tensor): Dim of the boxes with the shape of [B, 1, W, H]. vel (torch.Tensor): Velocity with the shape of [B, 1, W, H]. - reg (torch.Tensor): Regression value of the boxes in 2D with - the shape of [B, 2, W, H]. Default: None. - task_id (int): Index of task. Default: -1. + reg (torch.Tensor, optional): Regression value of the boxes in + 2D with the shape of [B, 2, W, H]. Default: None. + task_id (int, optional): Index of task. Default: -1. Returns: list[dict]: Decoded boxes. diff --git a/mmdet3d/core/bbox/coders/delta_xyzwhlr_bbox_coder.py b/mmdet3d/core/bbox/coders/delta_xyzwhlr_bbox_coder.py index 251b957ddf..28b6f5b3ec 100644 --- a/mmdet3d/core/bbox/coders/delta_xyzwhlr_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/delta_xyzwhlr_bbox_coder.py @@ -18,9 +18,9 @@ def __init__(self, code_size=7): @staticmethod def encode(src_boxes, dst_boxes): - """Get box regression transformation deltas (dx, dy, dz, dw, dh, dl, - dr, dv*) that can be used to transform the `src_boxes` into the - `target_boxes`. + """Get box regression transformation deltas (dx, dy, dz, dx_size, + dy_size, dz_size, dr, dv*) that can be used to transform the + `src_boxes` into the `target_boxes`. Args: src_boxes (torch.Tensor): source boxes, e.g., object proposals. @@ -55,13 +55,13 @@ def encode(src_boxes, dst_boxes): @staticmethod def decode(anchors, deltas): - """Apply transformation `deltas` (dx, dy, dz, dw, dh, dl, dr, dv*) to - `boxes`. + """Apply transformation `deltas` (dx, dy, dz, dx_size, dy_size, + dz_size, dr, dv*) to `boxes`. Args: anchors (torch.Tensor): Parameters of anchors with shape (N, 7). deltas (torch.Tensor): Encoded boxes with shape - (N, 7+n) [x, y, z, w, l, h, r, velo*]. + (N, 7+n) [x, y, z, x_size, y_size, z_size, r, velo*]. Returns: torch.Tensor: Decoded boxes. diff --git a/mmdet3d/core/bbox/coders/groupfree3d_bbox_coder.py b/mmdet3d/core/bbox/coders/groupfree3d_bbox_coder.py index 0732af0754..3b52696751 100644 --- a/mmdet3d/core/bbox/coders/groupfree3d_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/groupfree3d_bbox_coder.py @@ -13,9 +13,10 @@ class GroupFree3DBBoxCoder(PartialBinBasedBBoxCoder): num_dir_bins (int): Number of bins to encode direction angle. num_sizes (int): Number of size clusters. mean_sizes (list[list[int]]): Mean size of bboxes in each class. - with_rot (bool): Whether the bbox is with rotation. Defaults to True. - size_cls_agnostic (bool): Whether the predicted size is class-agnostic. + with_rot (bool, optional): Whether the bbox is with rotation. Defaults to True. + size_cls_agnostic (bool, optional): Whether the predicted size is + class-agnostic. Defaults to True. """ def __init__(self, @@ -35,7 +36,7 @@ def encode(self, gt_bboxes_3d, gt_labels_3d): """Encode ground truth to prediction targets. Args: - gt_bboxes_3d (BaseInstance3DBoxes): Ground truth bboxes \ + gt_bboxes_3d (BaseInstance3DBoxes): Ground truth bboxes with shape (n, 7). gt_labels_3d (torch.Tensor): Ground truth classes. @@ -75,7 +76,7 @@ def decode(self, bbox_out, prefix=''): - size_class: predicted bbox size class. - size_res: predicted bbox size residual. - size: predicted class-agnostic bbox size - prefix (str): Decode predictions with specific prefix. + prefix (str, optional): Decode predictions with specific prefix. Defaults to ''. Returns: @@ -121,7 +122,7 @@ def split_pred(self, cls_preds, reg_preds, base_xyz, prefix=''): cls_preds (torch.Tensor): Class predicted features to split. reg_preds (torch.Tensor): Regression predicted features to split. base_xyz (torch.Tensor): Coordinates of points. - prefix (str): Decode predictions with specific prefix. + prefix (str, optional): Decode predictions with specific prefix. Defaults to ''. Returns: diff --git a/mmdet3d/core/bbox/coders/partial_bin_based_bbox_coder.py b/mmdet3d/core/bbox/coders/partial_bin_based_bbox_coder.py index b20afae101..0fa9dc921c 100644 --- a/mmdet3d/core/bbox/coders/partial_bin_based_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/partial_bin_based_bbox_coder.py @@ -28,7 +28,7 @@ def encode(self, gt_bboxes_3d, gt_labels_3d): """Encode ground truth to prediction targets. Args: - gt_bboxes_3d (BaseInstance3DBoxes): Ground truth bboxes \ + gt_bboxes_3d (BaseInstance3DBoxes): Ground truth bboxes with shape (n, 7). gt_labels_3d (torch.Tensor): Ground truth classes. diff --git a/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py b/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py index 6a930afe41..211f9522ad 100644 --- a/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py +++ b/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py @@ -31,16 +31,16 @@ def __call__(self, bboxes1, bboxes2, mode='iou', is_aligned=False): Args: bboxes1 (torch.Tensor): shape (N, 7+N) - [x, y, z, dx, dy, dz, ry, v]. + [x, y, z, x_size, y_size, z_size, ry, v]. bboxes2 (torch.Tensor): shape (M, 7+N) - [x, y, z, dx, dy, dz, ry, v]. + [x, y, z, x_size, y_size, z_size, ry, v]. mode (str): "iou" (intersection over union) or iof (intersection over foreground). is_aligned (bool): Whether the calculation is aligned. Return: - torch.Tensor: If ``is_aligned`` is ``True``, return ious between \ - bboxes1 and bboxes2 with shape (M, N). If ``is_aligned`` is \ + torch.Tensor: If ``is_aligned`` is ``True``, return ious between + bboxes1 and bboxes2 with shape (M, N). If ``is_aligned`` is ``False``, return shape is M. """ return bbox_overlaps_nearest_3d(bboxes1, bboxes2, mode, is_aligned, @@ -75,13 +75,15 @@ def __call__(self, bboxes1, bboxes2, mode='iou'): calculate the actual 3D IoUs of boxes. Args: - bboxes1 (torch.Tensor): shape (N, 7+C) [x, y, z, dx, dy, dz, ry]. - bboxes2 (torch.Tensor): shape (M, 7+C) [x, y, z, dx, dy, dz, ry]. + bboxes1 (torch.Tensor): with shape (N, 7+C), + (x, y, z, x_size, y_size, z_size, ry, v*). + bboxes2 (torch.Tensor): with shape (M, 7+C), + (x, y, z, x_size, y_size, z_size, ry, v*). mode (str): "iou" (intersection over union) or iof (intersection over foreground). Return: - torch.Tensor: Bbox overlaps results of bboxes1 and bboxes2 \ + torch.Tensor: Bbox overlaps results of bboxes1 and bboxes2 with shape (M, N) (aligned mode is not supported currently). """ return bbox_overlaps_3d(bboxes1, bboxes2, mode, self.coordinate) @@ -111,15 +113,17 @@ def bbox_overlaps_nearest_3d(bboxes1, aligned pair of bboxes1 and bboxes2. Args: - bboxes1 (torch.Tensor): shape (N, 7+C) [x, y, z, dx, dy, dz, ry, v]. - bboxes2 (torch.Tensor): shape (M, 7+C) [x, y, z, dx, dy, dz, ry, v]. + bboxes1 (torch.Tensor): with shape (N, 7+C), + (x, y, z, x_size, y_size, z_size, ry, v*). + bboxes2 (torch.Tensor): with shape (M, 7+C), + (x, y, z, x_size, y_size, z_size, ry, v*). mode (str): "iou" (intersection over union) or iof (intersection over foreground). is_aligned (bool): Whether the calculation is aligned Return: - torch.Tensor: If ``is_aligned`` is ``True``, return ious between \ - bboxes1 and bboxes2 with shape (M, N). If ``is_aligned`` is \ + torch.Tensor: If ``is_aligned`` is ``True``, return ious between + bboxes1 and bboxes2 with shape (M, N). If ``is_aligned`` is ``False``, return shape is M. """ assert bboxes1.size(-1) == bboxes2.size(-1) >= 7 @@ -149,14 +153,16 @@ def bbox_overlaps_3d(bboxes1, bboxes2, mode='iou', coordinate='camera'): calculate the actual IoUs of boxes. Args: - bboxes1 (torch.Tensor): shape (N, 7+C) [x, y, z, dx, dy, dz, ry]. - bboxes2 (torch.Tensor): shape (M, 7+C) [x, y, z, dx, dy, dz, ry]. + bboxes1 (torch.Tensor): with shape (N, 7+C), + (x, y, z, x_size, y_size, z_size, ry, v*). + bboxes2 (torch.Tensor): with shape (M, 7+C), + (x, y, z, x_size, y_size, z_size, ry, v*). mode (str): "iou" (intersection over union) or iof (intersection over foreground). coordinate (str): 'camera' or 'lidar' coordinate system. Return: - torch.Tensor: Bbox overlaps results of bboxes1 and bboxes2 \ + torch.Tensor: Bbox overlaps results of bboxes1 and bboxes2 with shape (M, N) (aligned mode is not supported currently). """ assert bboxes1.size(-1) == bboxes2.size(-1) >= 7 @@ -186,7 +192,7 @@ def __call__(self, bboxes1, bboxes2, mode='iou', is_aligned=False): mode (str): "iou" (intersection over union) or "giou" (generalized intersection over union). is_aligned (bool, optional): If True, then m and n must be equal. - Default False. + Defaults to False. Returns: Tensor: shape (m, n) if ``is_aligned`` is False else shape (m,) """ @@ -220,9 +226,9 @@ def axis_aligned_bbox_overlaps_3d(bboxes1, mode (str): "iou" (intersection over union) or "giou" (generalized intersection over union). is_aligned (bool, optional): If True, then m and n must be equal. - Default False. + Defaults to False. eps (float, optional): A value added to the denominator for numerical - stability. Default 1e-6. + stability. Defaults to 1e-6. Returns: Tensor: shape (m, n) if ``is_aligned`` is False else shape (m,) diff --git a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py index cc8ba8bfed..b74fd5a8a5 100644 --- a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py +++ b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py @@ -138,7 +138,7 @@ def sample(self, assign_result (:obj:`AssignResult`): Bbox assigning results. bboxes (torch.Tensor): Boxes to be sampled from. gt_bboxes (torch.Tensor): Ground truth bboxes. - gt_labels (torch.Tensor, optional): Class labels of ground truth \ + gt_labels (torch.Tensor, optional): Class labels of ground truth bboxes. Returns: diff --git a/mmdet3d/core/bbox/structures/base_box3d.py b/mmdet3d/core/bbox/structures/base_box3d.py index b3ee156e85..37c7dc8709 100644 --- a/mmdet3d/core/bbox/structures/base_box3d.py +++ b/mmdet3d/core/bbox/structures/base_box3d.py @@ -1,8 +1,9 @@ import numpy as np import torch +import warnings from abc import abstractmethod -from mmdet3d.ops import points_in_boxes_batch, points_in_boxes_gpu +from mmdet3d.ops import points_in_boxes_all, points_in_boxes_part from mmdet3d.ops.iou3d import iou3d_cuda from .utils import limit_period, xywhr2xyxyr @@ -18,12 +19,12 @@ class BaseInstance3DBoxes(object): tensor (torch.Tensor | np.ndarray | list): a N x box_dim matrix. box_dim (int): Number of the dimension of a box. Each row is (x, y, z, x_size, y_size, z_size, yaw). - Default to 7. + Defaults to 7. with_yaw (bool): Whether the box is with yaw rotation. If False, the value of yaw will be set to 0 as minmax boxes. - Default to True. - origin (tuple[float]): The relative position of origin in the box. - Default to (0.5, 0.5, 0). This will guide the box be converted to + Defaults to True. + origin (tuple[float], optional): Relative position of the box origin. + Defaults to (0.5, 0.5, 0). This will guide the box be converted to (0.5, 0.5, 0) mode. Attributes: @@ -72,27 +73,29 @@ def volume(self): @property def dims(self): - """torch.Tensor: Corners of each box with size (N, 8, 3).""" + """torch.Tensor: Size dimensions of each box in shape (N, 3).""" return self.tensor[:, 3:6] @property def yaw(self): - """torch.Tensor: A vector with yaw of each box.""" + """torch.Tensor: A vector with yaw of each box in shape (N, ).""" return self.tensor[:, 6] @property def height(self): - """torch.Tensor: A vector with height of each box.""" + """torch.Tensor: A vector with height of each box in shape (N, ).""" return self.tensor[:, 5] @property def top_height(self): - """torch.Tensor: A vector with the top height of each box.""" + """torch.Tensor: + A vector with the top height of each box in shape (N, ).""" return self.bottom_height + self.height @property def bottom_height(self): - """torch.Tensor: A vector with bottom's height of each box.""" + """torch.Tensor: + A vector with bottom's height of each box in shape (N, ).""" return self.tensor[:, 2] @property @@ -100,35 +103,84 @@ def center(self): """Calculate the center of all the boxes. Note: - In the MMDetection3D's convention, the bottom center is + In MMDetection3D's convention, the bottom center is usually taken as the default center. The relative position of the centers in different kinds of boxes are different, e.g., the relative center of a boxes is (0.5, 1.0, 0.5) in camera and (0.5, 0.5, 0) in lidar. It is recommended to use ``bottom_center`` or ``gravity_center`` - for more clear usage. + for clearer usage. Returns: - torch.Tensor: A tensor with center of each box. + torch.Tensor: A tensor with center of each box in shape (N, 3). """ return self.bottom_center @property def bottom_center(self): - """torch.Tensor: A tensor with center of each box.""" + """torch.Tensor: A tensor with center of each box in shape (N, 3).""" return self.tensor[:, :3] @property def gravity_center(self): - """torch.Tensor: A tensor with center of each box.""" + """torch.Tensor: A tensor with center of each box in shape (N, 3).""" pass @property def corners(self): - """torch.Tensor: a tensor with 8 corners of each box.""" + """torch.Tensor: + a tensor with 8 corners of each box in shape (N, 8, 3).""" pass + @property + def bev(self): + """torch.Tensor: 2D BEV box of each box with rotation + in XYWHR format, in shape (N, 5).""" + return self.tensor[:, [0, 1, 3, 4, 6]] + + @property + def nearest_bev(self): + """torch.Tensor: A tensor of 2D BEV box of each box + without rotation.""" + # Obtain BEV boxes with rotation in XYWHR format + bev_rotated_boxes = self.bev + # convert the rotation to a valid range + rotations = bev_rotated_boxes[:, -1] + normed_rotations = torch.abs(limit_period(rotations, 0.5, np.pi)) + + # find the center of boxes + conditions = (normed_rotations > np.pi / 4)[..., None] + bboxes_xywh = torch.where(conditions, bev_rotated_boxes[:, + [0, 1, 3, 2]], + bev_rotated_boxes[:, :4]) + + centers = bboxes_xywh[:, :2] + dims = bboxes_xywh[:, 2:] + bev_boxes = torch.cat([centers - dims / 2, centers + dims / 2], dim=-1) + return bev_boxes + + def in_range_bev(self, box_range): + """Check whether the boxes are in the given range. + + Args: + box_range (list | torch.Tensor): the range of box + (x_min, y_min, x_max, y_max) + + Note: + The original implementation of SECOND checks whether boxes in + a range by checking whether the points are in a convex + polygon, we reduce the burden for simpler cases. + + Returns: + torch.Tensor: Whether each box is inside the reference range. + """ + in_range_flags = ((self.bev[:, 0] > box_range[0]) + & (self.bev[:, 1] > box_range[1]) + & (self.bev[:, 0] < box_range[2]) + & (self.bev[:, 1] < box_range[3])) + return in_range_flags + @abstractmethod def rotate(self, angle, points=None): """Rotate boxes with points (optional) with the given angle or rotation @@ -137,21 +189,28 @@ def rotate(self, angle, points=None): Args: angle (float | torch.Tensor | np.ndarray): Rotation angle or rotation matrix. - points (torch.Tensor, numpy.ndarray, :obj:`BasePoints`, optional): + points (torch.Tensor | numpy.ndarray | + :obj:`BasePoints`, optional): Points to rotate. Defaults to None. """ pass @abstractmethod def flip(self, bev_direction='horizontal'): - """Flip the boxes in BEV along given BEV direction.""" + """Flip the boxes in BEV along given BEV direction. + + Args: + bev_direction (str, optional): Direction by which to flip. + Can be chosen from 'horizontal' and 'vertical'. + Defaults to 'horizontal'. + """ pass def translate(self, trans_vector): """Translate boxes with the given translation vector. Args: - trans_vector (torch.Tensor): Translation vector of size 1x3. + trans_vector (torch.Tensor): Translation vector of size (1, 3). """ if not isinstance(trans_vector, torch.Tensor): trans_vector = self.tensor.new_tensor(trans_vector) @@ -181,28 +240,15 @@ def in_range_3d(self, box_range): & (self.tensor[:, 2] < box_range[5])) return in_range_flags - @abstractmethod - def in_range_bev(self, box_range): - """Check whether the boxes are in the given range. - - Args: - box_range (list | torch.Tensor): The range of box - in order of (x_min, y_min, x_max, y_max). - - Returns: - torch.Tensor: Indicating whether each box is inside - the reference range. - """ - pass - @abstractmethod def convert_to(self, dst, rt_mat=None): """Convert self to ``dst`` mode. Args: dst (:obj:`Box3DMode`): The target Box mode. - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from `src` coordinates to `dst` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. @@ -220,25 +266,26 @@ def scale(self, scale_factor): scale_factors (float): Scale factors to scale the boxes. """ self.tensor[:, :6] *= scale_factor - self.tensor[:, 7:] *= scale_factor + self.tensor[:, 7:] *= scale_factor # velocity def limit_yaw(self, offset=0.5, period=np.pi): """Limit the yaw to a given period and offset. Args: - offset (float): The offset of the yaw. - period (float): The expected period. + offset (float, optional): The offset of the yaw. Defaults to 0.5. + period (float, optional): The expected period. Defaults to np.pi. """ self.tensor[:, 6] = limit_period(self.tensor[:, 6], offset, period) - def nonempty(self, threshold: float = 0.0): + def nonempty(self, threshold=0.0): """Find boxes that are non-empty. A box is considered empty, if either of its side is no larger than threshold. Args: - threshold (float): The threshold of minimal sizes. + threshold (float, optional): The threshold of minimal sizes. + Defaults to 0.0. Returns: torch.Tensor: A binary vector which represents whether each @@ -363,7 +410,7 @@ def height_overlaps(cls, boxes1, boxes2, mode='iou'): Args: boxes1 (:obj:`BaseInstance3DBoxes`): Boxes 1 contain N boxes. boxes2 (:obj:`BaseInstance3DBoxes`): Boxes 2 contain M boxes. - mode (str, optional): Mode of iou calculation. Defaults to 'iou'. + mode (str, optional): Mode of IoU calculation. Defaults to 'iou'. Returns: torch.Tensor: Calculated iou of boxes. @@ -460,34 +507,49 @@ def new_box(self, data): return original_type( new_tensor, box_dim=self.box_dim, with_yaw=self.with_yaw) - def points_in_boxes(self, points, boxes_override=None): - """Find the box which the points are in. + def points_in_boxes_part(self, points, boxes_override=None): + """Find the box in which each point is. Args: - points (torch.Tensor): Points in shape (N, 3). + points (torch.Tensor): Points in shape (1, M, 3) or (M, 3), + 3 dimensions are (x, y, z) in LiDAR or depth coordinate. + boxes_override (torch.Tensor, optional): Boxes to override + `self.tensor`. Defaults to None. Returns: - torch.Tensor: The index of box where each point are in. + torch.Tensor: The index of the first box that each point + is in, in shape (M, ). Default value is -1 + (if the point is not enclosed by any box). + + Note: + If a point is enclosed by multiple boxes, the index of the + first box will be returned. """ if boxes_override is not None: boxes = boxes_override else: boxes = self.tensor - box_idx = points_in_boxes_gpu( - points.unsqueeze(0), - boxes.unsqueeze(0).to(points.device)).squeeze(0) + if points.dim() == 2: + points = points.unsqueeze(0) + box_idx = points_in_boxes_part(points, + boxes.unsqueeze(0).to( + points.device)).squeeze(0) return box_idx - def points_in_boxes_batch(self, points, boxes_override=None): - """Find points that are in boxes (CUDA). + def points_in_boxes_all(self, points, boxes_override=None): + """Find all boxes in which each point is. Args: - points (torch.Tensor): Points in shape [1, M, 3] or [M, 3], - 3 dimensions are [x, y, z] in LiDAR coordinate. + points (torch.Tensor): Points in shape (1, M, 3) or (M, 3), + 3 dimensions are (x, y, z) in LiDAR or depth coordinate. + boxes_override (torch.Tensor, optional): Boxes to override + `self.tensor`. Defaults to None. Returns: - torch.Tensor: The index of boxes each point lies in with shape - of (B, M, T). + torch.Tensor: A tensor indicating whether a point is in a box, + in shape (M, T). T is the number of boxes. Denote this + tensor as A, if the m^th point is in the t^th box, then + `A[m, t] == 1`, elsewise `A[m, t] == 0`. """ if boxes_override is not None: boxes = boxes_override @@ -501,6 +563,18 @@ def points_in_boxes_batch(self, points, boxes_override=None): assert points_clone.dim() == 3 and points_clone.shape[0] == 1 boxes = boxes.to(points_clone.device).unsqueeze(0) - box_idxs_of_pts = points_in_boxes_batch(points_clone, boxes) + box_idxs_of_pts = points_in_boxes_all(points_clone, boxes) return box_idxs_of_pts.squeeze(0) + + def points_in_boxes(self, points, boxes_override=None): + warnings.warn('DeprecationWarning: points_in_boxes is a ' + 'deprecated method, please consider using ' + 'points_in_boxes_part.') + return self.points_in_boxes_part(points, boxes_override) + + def points_in_boxes_batch(self, points, boxes_override=None): + warnings.warn('DeprecationWarning: points_in_boxes_batch is a ' + 'deprecated method, please consider using ' + 'points_in_boxes_all.') + return self.points_in_boxes_all(points, boxes_override) diff --git a/mmdet3d/core/bbox/structures/box_3d_mode.py b/mmdet3d/core/bbox/structures/box_3d_mode.py index 0e98cd8112..1406b42175 100644 --- a/mmdet3d/core/bbox/structures/box_3d_mode.py +++ b/mmdet3d/core/bbox/structures/box_3d_mode.py @@ -70,12 +70,13 @@ def convert(box, src, dst, rt_mat=None, with_yaw=True): Can be a k-tuple, k-list or an Nxk array/tensor, where k = 7. src (:obj:`Box3DMode`): The src Box mode. dst (:obj:`Box3DMode`): The target Box mode. - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from `src` coordinates to `dst` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. - with_yaw (bool): If `box` is an instance of + with_yaw (bool, optional): If `box` is an instance of :obj:`BaseInstance3DBoxes`, whether or not it has a yaw angle. Defaults to True. @@ -127,13 +128,13 @@ def convert(box, src, dst, rt_mat=None, with_yaw=True): yaw = limit_period(yaw, period=np.pi * 2) elif src == Box3DMode.DEPTH and dst == Box3DMode.CAM: if rt_mat is None: - rt_mat = arr.new_tensor([[1, 0, 0], [0, 0, 1], [0, -1, 0]]) + rt_mat = arr.new_tensor([[1, 0, 0], [0, 0, -1], [0, 1, 0]]) xyz_size = torch.cat([x_size, z_size, y_size], dim=-1) if with_yaw: yaw = -yaw elif src == Box3DMode.CAM and dst == Box3DMode.DEPTH: if rt_mat is None: - rt_mat = arr.new_tensor([[1, 0, 0], [0, 0, -1], [0, 1, 0]]) + rt_mat = arr.new_tensor([[1, 0, 0], [0, 0, 1], [0, -1, 0]]) xyz_size = torch.cat([x_size, z_size, y_size], dim=-1) if with_yaw: yaw = -yaw diff --git a/mmdet3d/core/bbox/structures/cam_box3d.py b/mmdet3d/core/bbox/structures/cam_box3d.py index 7ac29619ed..509de233ba 100644 --- a/mmdet3d/core/bbox/structures/cam_box3d.py +++ b/mmdet3d/core/bbox/structures/cam_box3d.py @@ -3,7 +3,7 @@ from ...points import BasePoints from .base_box3d import BaseInstance3DBoxes -from .utils import limit_period, rotation_3d_in_axis +from .utils import rotation_3d_in_axis class CameraInstance3DBoxes(BaseInstance3DBoxes): @@ -27,15 +27,12 @@ class CameraInstance3DBoxes(BaseInstance3DBoxes): The yaw is 0 at the positive direction of x axis, and decreases from the positive direction of x to the positive direction of z. - A refactor is ongoing to make the three coordinate systems - easier to understand and convert between each other. - Attributes: - tensor (torch.Tensor): Float matrix of N x box_dim. - box_dim (int): Integer indicates the dimension of a box + tensor (torch.Tensor): Float matrix in shape (N, box_dim). + box_dim (int): Integer indicating the dimension of a box Each row is (x, y, z, x_size, y_size, z_size, yaw, ...). - with_yaw (bool): If True, the value of yaw will be set to 0 as minmax - boxes. + with_yaw (bool): If True, the value of yaw will be set to 0 as + axis-aligned boxes tightly enclosing the original boxes. """ YAW_AXIS = 1 @@ -76,23 +73,25 @@ def __init__(self, @property def height(self): - """torch.Tensor: A vector with height of each box.""" + """torch.Tensor: A vector with height of each box in shape (N, ).""" return self.tensor[:, 4] @property def top_height(self): - """torch.Tensor: A vector with the top height of each box.""" + """torch.Tensor: + A vector with the top height of each box in shape (N, ).""" # the positive direction is down rather than up return self.bottom_height - self.height @property def bottom_height(self): - """torch.Tensor: A vector with bottom's height of each box.""" + """torch.Tensor: + A vector with bottom's height of each box in shape (N, ).""" return self.tensor[:, 1] @property def gravity_center(self): - """torch.Tensor: A tensor with center of each box.""" + """torch.Tensor: A tensor with center of each box in shape (N, 3).""" bottom_center = self.bottom_center gravity_center = torch.zeros_like(bottom_center) gravity_center[:, [0, 2]] = bottom_center[:, [0, 2]] @@ -147,8 +146,8 @@ def corners(self): @property def bev(self): - """torch.Tensor: A n x 5 tensor of 2D BEV box of each box - with rotation in XYWHR format.""" + """torch.Tensor: 2D BEV box of each box with rotation + in XYWHR format, in shape (N, 5).""" bev = self.tensor[:, [0, 2, 3, 5, 6]].clone() # positive direction of the gravity axis # in cam coord system points to the earth @@ -156,27 +155,6 @@ def bev(self): bev[:, -1] = -bev[:, -1] return bev - @property - def nearest_bev(self): - """torch.Tensor: A tensor of 2D BEV box of each box - without rotation.""" - # Obtain BEV boxes with rotation in XZWHR format - bev_rotated_boxes = self.bev - # convert the rotation to a valid range - rotations = bev_rotated_boxes[:, -1] - normed_rotations = torch.abs(limit_period(rotations, 0.5, np.pi)) - - # find the center of boxes - conditions = (normed_rotations > np.pi / 4)[..., None] - bboxes_xywh = torch.where(conditions, bev_rotated_boxes[:, - [0, 1, 3, 2]], - bev_rotated_boxes[:, :4]) - - centers = bboxes_xywh[:, :2] - dims = bboxes_xywh[:, 2:] - bev_boxes = torch.cat([centers - dims / 2, centers + dims / 2], dim=-1) - return bev_boxes - def rotate(self, angle, points=None): """Rotate boxes with points (optional) with the given angle or rotation matrix. @@ -184,7 +162,7 @@ def rotate(self, angle, points=None): Args: angle (float | torch.Tensor | np.ndarray): Rotation angle or rotation matrix. - points (torch.Tensor, numpy.ndarray, :obj:`BasePoints`, optional): + points (torch.Tensor | np.ndarray | :obj:`BasePoints`, optional): Points to rotate. Defaults to None. Returns: @@ -236,7 +214,7 @@ def flip(self, bev_direction='horizontal', points=None): Args: bev_direction (str): Flip direction (horizontal or vertical). - points (torch.Tensor, numpy.ndarray, :obj:`BasePoints`, None): + points (torch.Tensor | np.ndarray | :obj:`BasePoints`, optional): Points to flip. Defaults to None. Returns: @@ -263,28 +241,6 @@ def flip(self, bev_direction='horizontal', points=None): points.flip(bev_direction) return points - def in_range_bev(self, box_range): - """Check whether the boxes are in the given range. - - Args: - box_range (list | torch.Tensor): The range of box - (x_min, z_min, x_max, z_max). - - Note: - The original implementation of SECOND checks whether boxes in - a range by checking whether the points are in a convex - polygon, we reduce the burden for simpler cases. - - Returns: - torch.Tensor: Indicating whether each box is inside - the reference range. - """ - in_range_flags = ((self.tensor[:, 0] > box_range[0]) - & (self.tensor[:, 2] > box_range[1]) - & (self.tensor[:, 0] < box_range[2]) - & (self.tensor[:, 2] < box_range[3])) - return in_range_flags - @classmethod def height_overlaps(cls, boxes1, boxes2, mode='iou'): """Calculate height overlaps of two boxes. @@ -321,8 +277,9 @@ def convert_to(self, dst, rt_mat=None): Args: dst (:obj:`Box3DMode`): The target Box mode. - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from ``src`` coordinates to ``dst`` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. @@ -335,43 +292,55 @@ def convert_to(self, dst, rt_mat=None): return Box3DMode.convert( box=self, src=Box3DMode.CAM, dst=dst, rt_mat=rt_mat) - def points_in_boxes(self, points): - """Find the box which the points are in. + def points_in_boxes_part(self, points, boxes_override=None): + """Find the box in which each point is. Args: - points (torch.Tensor): Points in shape (N, 3). + points (torch.Tensor): Points in shape (1, M, 3) or (M, 3), + 3 dimensions are (x, y, z) in LiDAR or depth coordinate. + boxes_override (torch.Tensor, optional): Boxes to override + `self.tensor `. Defaults to None. Returns: - torch.Tensor: The index of box where each point are in. + torch.Tensor: The index of the box in which + each point is, in shape (M, ). Default value is -1 + (if the point is not enclosed by any box). """ from .coord_3d_mode import Coord3DMode points_lidar = Coord3DMode.convert(points, Coord3DMode.CAM, Coord3DMode.LIDAR) - boxes_lidar = Coord3DMode.convert(self.tensor, Coord3DMode.CAM, - Coord3DMode.LIDAR) + if boxes_override is not None: + boxes_lidar = boxes_override + else: + boxes_lidar = Coord3DMode.convert(self.tensor, Coord3DMode.CAM, + Coord3DMode.LIDAR) - box_idx = super().points_in_boxes(self, points_lidar, boxes_lidar) + box_idx = super().points_in_boxes_part(points_lidar, boxes_lidar) return box_idx - def points_in_boxes_batch(self, points): - """Find points that are in boxes (CUDA). + def points_in_boxes_all(self, points, boxes_override=None): + """Find all boxes in which each point is. Args: - points (torch.Tensor): Points in shape [1, M, 3] or [M, 3], - 3 dimensions are [x, y, z] in LiDAR coordinate. + points (torch.Tensor): Points in shape (1, M, 3) or (M, 3), + 3 dimensions are (x, y, z) in LiDAR or depth coordinate. + boxes_override (torch.Tensor, optional): Boxes to override + `self.tensor `. Defaults to None. Returns: - torch.Tensor: The index of boxes each point lies in with shape - of (B, M, T). + torch.Tensor: The index of all boxes in which each point is, + in shape (B, M, T). """ from .coord_3d_mode import Coord3DMode points_lidar = Coord3DMode.convert(points, Coord3DMode.CAM, Coord3DMode.LIDAR) - boxes_lidar = Coord3DMode.convert(self.tensor, Coord3DMode.CAM, - Coord3DMode.LIDAR) + if boxes_override is not None: + boxes_lidar = boxes_override + else: + boxes_lidar = Coord3DMode.convert(self.tensor, Coord3DMode.CAM, + Coord3DMode.LIDAR) - box_idx = super().points_in_boxes_batch(self, points_lidar, - boxes_lidar) + box_idx = super().points_in_boxes_all(points_lidar, boxes_lidar) return box_idx diff --git a/mmdet3d/core/bbox/structures/coord_3d_mode.py b/mmdet3d/core/bbox/structures/coord_3d_mode.py index b42726461c..1fb0566d0b 100644 --- a/mmdet3d/core/bbox/structures/coord_3d_mode.py +++ b/mmdet3d/core/bbox/structures/coord_3d_mode.py @@ -69,8 +69,9 @@ def convert(input, src, dst, rt_mat=None, with_yaw=True, is_point=True): Can be a k-tuple, k-list or an Nxk array/tensor, where k = 7. src (:obj:`Box3DMode` | :obj:`Coord3DMode`): The source mode. dst (:obj:`Box3DMode` | :obj:`Coord3DMode`): The target mode. - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from `src` coordinates to `dst` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. @@ -112,8 +113,9 @@ def convert_box(box, src, dst, rt_mat=None, with_yaw=True): Can be a k-tuple, k-list or an Nxk array/tensor, where k = 7. src (:obj:`Box3DMode`): The src Box mode. dst (:obj:`Box3DMode`): The target Box mode. - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from `src` coordinates to `dst` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. @@ -138,8 +140,9 @@ def convert_point(point, src, dst, rt_mat=None): Can be a k-tuple, k-list or an Nxk array/tensor. src (:obj:`CoordMode`): The src Point mode. dst (:obj:`CoordMode`): The target Point mode. - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from `src` coordinates to `dst` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. diff --git a/mmdet3d/core/bbox/structures/depth_box3d.py b/mmdet3d/core/bbox/structures/depth_box3d.py index d732f714ff..fb92aff827 100644 --- a/mmdet3d/core/bbox/structures/depth_box3d.py +++ b/mmdet3d/core/bbox/structures/depth_box3d.py @@ -3,7 +3,7 @@ from mmdet3d.core.points import BasePoints from .base_box3d import BaseInstance3DBoxes -from .utils import limit_period, rotation_3d_in_axis +from .utils import rotation_3d_in_axis class DepthInstance3DBoxes(BaseInstance3DBoxes): @@ -40,7 +40,7 @@ class DepthInstance3DBoxes(BaseInstance3DBoxes): @property def gravity_center(self): - """torch.Tensor: A tensor with center of each box.""" + """torch.Tensor: A tensor with center of each box in shape (N, 3).""" bottom_center = self.bottom_center gravity_center = torch.zeros_like(bottom_center) gravity_center[:, :2] = bottom_center[:, :2] @@ -70,8 +70,6 @@ def corners(self): (x0, y0, z0) + ----------- + --------> right x (x1, y0, z0) """ - # TODO: rotation_3d_in_axis function do not support - # empty tensor currently. assert len(self.tensor) != 0 dims = self.dims corners_norm = torch.from_numpy( @@ -89,33 +87,6 @@ def corners(self): corners += self.tensor[:, :3].view(-1, 1, 3) return corners - @property - def bev(self): - """torch.Tensor: A n x 5 tensor of 2D BEV box of each box - in XYWHR format.""" - return self.tensor[:, [0, 1, 3, 4, 6]] - - @property - def nearest_bev(self): - """torch.Tensor: A tensor of 2D BEV box of each box - without rotation.""" - # Obtain BEV boxes with rotation in XYWHR format - bev_rotated_boxes = self.bev - # convert the rotation to a valid range - rotations = bev_rotated_boxes[:, -1] - normed_rotations = torch.abs(limit_period(rotations, 0.5, np.pi)) - - # find the center of boxes - conditions = (normed_rotations > np.pi / 4)[..., None] - bboxes_xywh = torch.where(conditions, bev_rotated_boxes[:, - [0, 1, 3, 2]], - bev_rotated_boxes[:, :4]) - - centers = bboxes_xywh[:, :2] - dims = bboxes_xywh[:, 2:] - bev_boxes = torch.cat([centers - dims / 2, centers + dims / 2], dim=-1) - return bev_boxes - def rotate(self, angle, points=None): """Rotate boxes with points (optional) with the given angle or rotation matrix. @@ -123,7 +94,7 @@ def rotate(self, angle, points=None): Args: angle (float | torch.Tensor | np.ndarray): Rotation angle or rotation matrix. - points (torch.Tensor, numpy.ndarray, :obj:`BasePoints`, optional): + points (torch.Tensor | np.ndarray | :obj:`BasePoints`, optional): Points to rotate. Defaults to None. Returns: @@ -153,6 +124,8 @@ def rotate(self, angle, points=None): if self.with_yaw: self.tensor[:, 6] += angle else: + # for axis-aligned boxes, we take the new + # enclosing axis-aligned boxes after rotation corners_rot = self.corners @ rot_mat_T new_x_size = corners_rot[..., 0].max( dim=1, keepdim=True)[0] - corners_rot[..., 0].min( @@ -180,8 +153,9 @@ def flip(self, bev_direction='horizontal', points=None): In Depth coordinates, it flips x (horizontal) or y (vertical) axis. Args: - bev_direction (str): Flip direction (horizontal or vertical). - points (torch.Tensor, numpy.ndarray, :obj:`BasePoints`, None): + bev_direction (str, optional): Flip direction + (horizontal or vertical). Defaults to 'horizontal'. + points (torch.Tensor | np.ndarray | :obj:`BasePoints`, optional): Points to flip. Defaults to None. Returns: @@ -208,35 +182,14 @@ def flip(self, bev_direction='horizontal', points=None): points.flip(bev_direction) return points - def in_range_bev(self, box_range): - """Check whether the boxes are in the given range. - - Args: - box_range (list | torch.Tensor): The range of box - (x_min, y_min, x_max, y_max). - - Note: - In the original implementation of SECOND, checking whether - a box in the range checks whether the points are in a convex - polygon, we try to reduce the burdun for simpler cases. - - Returns: - torch.Tensor: Indicating whether each box is inside - the reference range. - """ - in_range_flags = ((self.tensor[:, 0] > box_range[0]) - & (self.tensor[:, 1] > box_range[1]) - & (self.tensor[:, 0] < box_range[2]) - & (self.tensor[:, 1] < box_range[3])) - return in_range_flags - def convert_to(self, dst, rt_mat=None): """Convert self to ``dst`` mode. Args: dst (:obj:`Box3DMode`): The target Box mode. - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from ``src`` coordinates to ``dst`` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. @@ -256,7 +209,7 @@ def enlarged_box(self, extra_width): extra_width (float | torch.Tensor): Extra width to enlarge the box. Returns: - :obj:`LiDARInstance3DBoxes`: Enlarged boxes. + :obj:`DepthInstance3DBoxes`: Enlarged boxes. """ enlarged_boxes = self.tensor.clone() enlarged_boxes[:, 3:6] += extra_width * 2 diff --git a/mmdet3d/core/bbox/structures/lidar_box3d.py b/mmdet3d/core/bbox/structures/lidar_box3d.py index 6adfa3ad0a..24efa32384 100644 --- a/mmdet3d/core/bbox/structures/lidar_box3d.py +++ b/mmdet3d/core/bbox/structures/lidar_box3d.py @@ -3,7 +3,7 @@ from mmdet3d.core.points import BasePoints from .base_box3d import BaseInstance3DBoxes -from .utils import limit_period, rotation_3d_in_axis +from .utils import rotation_3d_in_axis class LiDARInstance3DBoxes(BaseInstance3DBoxes): @@ -38,7 +38,7 @@ class LiDARInstance3DBoxes(BaseInstance3DBoxes): @property def gravity_center(self): - """torch.Tensor: A tensor with center of each box.""" + """torch.Tensor: A tensor with center of each box in shape (N, 3).""" bottom_center = self.bottom_center gravity_center = torch.zeros_like(bottom_center) gravity_center[:, :2] = bottom_center[:, :2] @@ -87,33 +87,6 @@ def corners(self): corners += self.tensor[:, :3].view(-1, 1, 3) return corners - @property - def bev(self): - """torch.Tensor: 2D BEV box of each box with rotation - in XYWHR format.""" - return self.tensor[:, [0, 1, 3, 4, 6]] - - @property - def nearest_bev(self): - """torch.Tensor: A tensor of 2D BEV box of each box - without rotation.""" - # Obtain BEV boxes with rotation in XYWHR format - bev_rotated_boxes = self.bev - # convert the rotation to a valid range - rotations = bev_rotated_boxes[:, -1] - normed_rotations = torch.abs(limit_period(rotations, 0.5, np.pi)) - - # find the center of boxes - conditions = (normed_rotations > np.pi / 4)[..., None] - bboxes_xywh = torch.where(conditions, bev_rotated_boxes[:, - [0, 1, 3, 2]], - bev_rotated_boxes[:, :4]) - - centers = bboxes_xywh[:, :2] - dims = bboxes_xywh[:, 2:] - bev_boxes = torch.cat([centers - dims / 2, centers + dims / 2], dim=-1) - return bev_boxes - def rotate(self, angle, points=None): """Rotate boxes with points (optional) with the given angle or rotation matrix. @@ -121,7 +94,7 @@ def rotate(self, angle, points=None): Args: angles (float | torch.Tensor | np.ndarray): Rotation angle or rotation matrix. - points (torch.Tensor, numpy.ndarray, :obj:`BasePoints`, optional): + points (torch.Tensor | np.ndarray | :obj:`BasePoints`, optional): Points to rotate. Defaults to None. Returns: @@ -173,7 +146,7 @@ def flip(self, bev_direction='horizontal', points=None): Args: bev_direction (str): Flip direction (horizontal or vertical). - points (torch.Tensor, numpy.ndarray, :obj:`BasePoints`, None): + points (torch.Tensor | np.ndarray | :obj:`BasePoints`, optional): Points to flip. Defaults to None. Returns: @@ -200,34 +173,14 @@ def flip(self, bev_direction='horizontal', points=None): points.flip(bev_direction) return points - def in_range_bev(self, box_range): - """Check whether the boxes are in the given range. - - Args: - box_range (list | torch.Tensor): the range of box - (x_min, y_min, x_max, y_max) - - Note: - The original implementation of SECOND checks whether boxes in - a range by checking whether the points are in a convex - polygon, we reduce the burden for simpler cases. - - Returns: - torch.Tensor: Whether each box is inside the reference range. - """ - in_range_flags = ((self.tensor[:, 0] > box_range[0]) - & (self.tensor[:, 1] > box_range[1]) - & (self.tensor[:, 0] < box_range[2]) - & (self.tensor[:, 1] < box_range[3])) - return in_range_flags - def convert_to(self, dst, rt_mat=None): """Convert self to ``dst`` mode. Args: dst (:obj:`Box3DMode`): the target Box mode - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from ``src`` coordinates to ``dst`` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. diff --git a/mmdet3d/core/bbox/transforms.py b/mmdet3d/core/bbox/transforms.py index e5f5778b6a..e002924f10 100644 --- a/mmdet3d/core/bbox/transforms.py +++ b/mmdet3d/core/bbox/transforms.py @@ -50,10 +50,10 @@ def bbox3d2result(bboxes, scores, labels, attrs=None): """Convert detection results to a list of numpy arrays. Args: - bboxes (torch.Tensor): Bounding boxes with shape of (n, 5). - labels (torch.Tensor): Labels with shape of (n, ). - scores (torch.Tensor): Scores with shape of (n, ). - attrs (torch.Tensor, optional): Attributes with shape of (n, ). + bboxes (torch.Tensor): Bounding boxes with shape (N, 5). + labels (torch.Tensor): Labels with shape (N, ). + scores (torch.Tensor): Scores with shape (N, ). + attrs (torch.Tensor, optional): Attributes with shape (N, ). Defaults to None. Returns: diff --git a/mmdet3d/core/evaluation/indoor_eval.py b/mmdet3d/core/evaluation/indoor_eval.py index 69c8adb8d4..d80506225b 100644 --- a/mmdet3d/core/evaluation/indoor_eval.py +++ b/mmdet3d/core/evaluation/indoor_eval.py @@ -8,9 +8,9 @@ def average_precision(recalls, precisions, mode='area'): """Calculate average precision (for single or multiple scales). Args: - recalls (np.ndarray): Recalls with shape of (num_scales, num_dets) \ + recalls (np.ndarray): Recalls with shape of (num_scales, num_dets) or (num_dets, ). - precisions (np.ndarray): Precisions with shape of \ + precisions (np.ndarray): Precisions with shape of (num_scales, num_dets) or (num_dets, ). mode (str): 'area' or '11points', 'area' means calculating the area under precision-recall curve, '11points' means calculating @@ -57,13 +57,13 @@ def eval_det_cls(pred, gt, iou_thr=None): single class. Args: - pred (dict): Predictions mapping from image id to bounding boxes \ + pred (dict): Predictions mapping from image id to bounding boxes and scores. gt (dict): Ground truths mapping from image id to bounding boxes. iou_thr (list[float]): A list of iou thresholds. Return: - tuple (np.ndarray, np.ndarray, float): Recalls, precisions and \ + tuple (np.ndarray, np.ndarray, float): Recalls, precisions and average precision. """ @@ -169,10 +169,9 @@ def eval_map_recall(pred, gt, ovthresh=None): Args: pred (dict): Information of detection results, which maps class_id and predictions. - gt (dict): Information of ground truths, which maps class_id and \ + gt (dict): Information of ground truths, which maps class_id and ground truths. - ovthresh (list[float]): iou threshold. - Default: None. + ovthresh (list[float], optional): iou threshold. Default: None. Return: tuple[dict]: dict results of recall, AP, and precision for all classes. @@ -217,12 +216,12 @@ def indoor_eval(gt_annos, includes the following keys - labels_3d (torch.Tensor): Labels of boxes. - - boxes_3d (:obj:`BaseInstance3DBoxes`): \ + - boxes_3d (:obj:`BaseInstance3DBoxes`): 3D bounding boxes in Depth coordinate. - scores_3d (torch.Tensor): Scores of boxes. metric (list[float]): IoU thresholds for computing average precisions. label2cat (dict): Map from label to category. - logger (logging.Logger | str | None): The way to print the mAP + logger (logging.Logger | str, optional): The way to print the mAP summary. See `mmdet.utils.print_log()` for details. Default: None. Return: diff --git a/mmdet3d/core/evaluation/kitti_utils/rotate_iou.py b/mmdet3d/core/evaluation/kitti_utils/rotate_iou.py index cc2cca58f8..33dccfd7f9 100644 --- a/mmdet3d/core/evaluation/kitti_utils/rotate_iou.py +++ b/mmdet3d/core/evaluation/kitti_utils/rotate_iou.py @@ -290,7 +290,8 @@ def rotate_iou_kernel_eval(N, dev_query_boxes, dev_iou, criterion=-1): - """Kernel of computing rotated iou. + """Kernel of computing rotated IoU. This function is for bev boxes in + camera coordinate system ONLY (the rotation is clockwise). Args: N (int): The number of boxes. @@ -342,10 +343,14 @@ def rotate_iou_gpu_eval(boxes, query_boxes, criterion=-1, device_id=0): in one example with numba.cuda code). convert from [this project]( https://github.com/hongzhenwang/RRPN-revise/tree/master/lib/rotation). + This function is for bev boxes in camera coordinate system ONLY + (the rotation is clockwise). + Args: boxes (torch.Tensor): rbboxes. format: centers, dims, angles(clockwise when positive) with the shape of [N, 5]. - query_boxes (float tensor: [K, 5]): rbboxes to compute iou with boxes. + query_boxes (torch.FloatTensor, shape=(K, 5)): + rbboxes to compute iou with boxes. device_id (int, optional): Defaults to 0. Device to use. criterion (int, optional): Indicate different type of iou. -1 indicate `area_inter / (area1 + area2 - area_inter)`, diff --git a/mmdet3d/core/evaluation/lyft_eval.py b/mmdet3d/core/evaluation/lyft_eval.py index 6842eee831..71a73d09b5 100644 --- a/mmdet3d/core/evaluation/lyft_eval.py +++ b/mmdet3d/core/evaluation/lyft_eval.py @@ -17,7 +17,7 @@ def load_lyft_gts(lyft, data_root, eval_split, logger=None): lyft (:obj:`LyftDataset`): Lyft class in the sdk. data_root (str): Root of data for reading splits. eval_split (str): Name of the split for evaluation. - logger (logging.Logger | str | None): Logger used for printing + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. Returns: @@ -95,7 +95,7 @@ def lyft_eval(lyft, data_root, res_path, eval_set, output_dir, logger=None): res_path (str): Path of result json file recording detections. eval_set (str): Name of the split for evaluation. output_dir (str): Output directory for output json files. - logger (logging.Logger | str | None): Logger used for printing + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. Returns: @@ -201,9 +201,9 @@ def get_single_class_aps(gt, predictions, iou_thresholds): Args: gt (list[dict]): list of dictionaries in the format described above. - predictions (list[dict]): list of dictionaries in the format \ + predictions (list[dict]): list of dictionaries in the format described below. - iou_thresholds (list[float]): IOU thresholds used to calculate \ + iou_thresholds (list[float]): IOU thresholds used to calculate TP / FN Returns: diff --git a/mmdet3d/core/evaluation/seg_eval.py b/mmdet3d/core/evaluation/seg_eval.py index bb81807069..2582498bd5 100644 --- a/mmdet3d/core/evaluation/seg_eval.py +++ b/mmdet3d/core/evaluation/seg_eval.py @@ -76,7 +76,7 @@ def seg_eval(gt_labels, seg_preds, label2cat, ignore_index, logger=None): seg_preds (list[torch.Tensor]): Predictions. label2cat (dict): Map from label to category name. ignore_index (int): Index that will be ignored in evaluation. - logger (logging.Logger | str | None): The way to print the mAP + logger (logging.Logger | str, optional): The way to print the mAP summary. See `mmdet.utils.print_log()` for details. Default: None. Returns: diff --git a/mmdet3d/core/evaluation/waymo_utils/prediction_kitti_to_waymo.py b/mmdet3d/core/evaluation/waymo_utils/prediction_kitti_to_waymo.py index c0651011ee..86548a79ce 100644 --- a/mmdet3d/core/evaluation/waymo_utils/prediction_kitti_to_waymo.py +++ b/mmdet3d/core/evaluation/waymo_utils/prediction_kitti_to_waymo.py @@ -113,7 +113,7 @@ def parse_one_object(instance_idx): instance_idx (int): Index of the instance to be converted. Returns: - :obj:`Object`: Predicted instance in waymo dataset \ + :obj:`Object`: Predicted instance in waymo dataset Object proto. """ cls = kitti_result['name'][instance_idx] diff --git a/mmdet3d/core/points/base_points.py b/mmdet3d/core/points/base_points.py index 16fbe406ef..a43b03b909 100644 --- a/mmdet3d/core/points/base_points.py +++ b/mmdet3d/core/points/base_points.py @@ -11,17 +11,17 @@ class BasePoints(object): Args: tensor (torch.Tensor | np.ndarray | list): a N x points_dim matrix. - points_dim (int): Number of the dimension of a point. - Each row is (x, y, z). Default to 3. - attribute_dims (dict): Dictionary to indicate the meaning of extra - dimension. Default to None. + points_dim (int, optional): Number of the dimension of a point. + Each row is (x, y, z). Defaults to 3. + attribute_dims (dict, optional): Dictionary to indicate the + meaning of extra dimension. Defaults to None. Attributes: tensor (torch.Tensor): Float matrix of N x points_dim. points_dim (int): Integer indicating the dimension of a point. Each row is (x, y, z, ...). attribute_dims (bool): Dictionary to indicate the meaning of extra - dimension. Default to None. + dimension. Defaults to None. rotation_axis (int): Default rotation axis for points rotation. """ @@ -46,7 +46,7 @@ def __init__(self, tensor, points_dim=3, attribute_dims=None): @property def coord(self): - """torch.Tensor: Coordinates of each point with size (N, 3).""" + """torch.Tensor: Coordinates of each point in shape (N, 3).""" return self.tensor[:, :3] @coord.setter @@ -62,7 +62,8 @@ def coord(self, tensor): @property def height(self): - """torch.Tensor: A vector with height of each point.""" + """torch.Tensor: + A vector with height of each point in shape (N, 1), or None.""" if self.attribute_dims is not None and \ 'height' in self.attribute_dims.keys(): return self.tensor[:, self.attribute_dims['height']] @@ -92,7 +93,8 @@ def height(self, tensor): @property def color(self): - """torch.Tensor: A vector with color of each point.""" + """torch.Tensor: + A vector with color of each point in shape (N, 3), or None.""" if self.attribute_dims is not None and \ 'color' in self.attribute_dims.keys(): return self.tensor[:, self.attribute_dims['color']] @@ -144,7 +146,7 @@ def rotate(self, rotation, axis=None): Args: rotation (float | np.ndarray | torch.Tensor): Rotation matrix or angle. - axis (int): Axis to rotate at. Defaults to None. + axis (int, optional): Axis to rotate at. Defaults to None. """ if not isinstance(rotation, torch.Tensor): rotation = self.tensor.new_tensor(rotation) @@ -168,7 +170,11 @@ def rotate(self, rotation, axis=None): @abstractmethod def flip(self, bev_direction='horizontal'): - """Flip the points in BEV along given BEV direction.""" + """Flip the points along given BEV direction. + + Args: + bev_direction (str): Flip direction (horizontal or vertical). + """ pass def translate(self, trans_vector): @@ -205,7 +211,7 @@ def in_range_3d(self, point_range): polygon, we try to reduce the burden for simpler cases. Returns: - torch.Tensor: A binary vector indicating whether each point is \ + torch.Tensor: A binary vector indicating whether each point is inside the reference range. """ in_range_flags = ((self.tensor[:, 0] > point_range[0]) @@ -216,7 +222,11 @@ def in_range_3d(self, point_range): & (self.tensor[:, 2] < point_range[5])) return in_range_flags - @abstractmethod + @property + def bev(self): + """torch.Tensor: BEV of the points in shape (N, 2).""" + return self.tensor[:, [0, 1]] + def in_range_bev(self, point_range): """Check whether the points are in the given range. @@ -225,10 +235,14 @@ def in_range_bev(self, point_range): in order of (x_min, y_min, x_max, y_max). Returns: - torch.Tensor: Indicating whether each point is inside \ + torch.Tensor: Indicating whether each point is inside the reference range. """ - pass + in_range_flags = ((self.bev[:, 0] > point_range[0]) + & (self.bev[:, 1] > point_range[1]) + & (self.bev[:, 1] < point_range[2]) + & (self.bev[:, 1] < point_range[3])) + return in_range_flags @abstractmethod def convert_to(self, dst, rt_mat=None): @@ -236,14 +250,15 @@ def convert_to(self, dst, rt_mat=None): Args: dst (:obj:`CoordMode`): The target Box mode. - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from `src` coordinates to `dst` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. Returns: - :obj:`BasePoints`: The converted box of the same type \ + :obj:`BasePoints`: The converted box of the same type in the `dst` mode. """ pass @@ -275,7 +290,7 @@ def __getitem__(self, item): subject to Pytorch's indexing semantics. Returns: - :obj:`BasePoints`: A new object of \ + :obj:`BasePoints`: A new object of :class:`BasePoints` after indexing. """ original_type = type(self) @@ -366,7 +381,7 @@ def to(self, device): device (str | :obj:`torch.device`): The name of the device. Returns: - :obj:`BasePoints`: A new boxes object on the \ + :obj:`BasePoints`: A new boxes object on the specific device. """ original_type = type(self) @@ -379,7 +394,7 @@ def clone(self): """Clone the Points. Returns: - :obj:`BasePoints`: Box object with the same properties \ + :obj:`BasePoints`: Box object with the same properties as self. """ original_type = type(self) @@ -404,14 +419,14 @@ def __iter__(self): def new_point(self, data): """Create a new point object with data. - The new point and its tensor has the similar properties \ + The new point and its tensor has the similar properties as self and self.tensor, respectively. Args: data (torch.Tensor | numpy.array | list): Data to be copied. Returns: - :obj:`BasePoints`: A new point object with ``data``, \ + :obj:`BasePoints`: A new point object with ``data``, the object's other properties are similar to ``self``. """ new_tensor = self.tensor.new_tensor(data) \ diff --git a/mmdet3d/core/points/cam_points.py b/mmdet3d/core/points/cam_points.py index 185680158b..6f189269e6 100644 --- a/mmdet3d/core/points/cam_points.py +++ b/mmdet3d/core/points/cam_points.py @@ -6,17 +6,17 @@ class CameraPoints(BasePoints): Args: tensor (torch.Tensor | np.ndarray | list): a N x points_dim matrix. - points_dim (int): Number of the dimension of a point. - Each row is (x, y, z). Default to 3. - attribute_dims (dict): Dictionary to indicate the meaning of extra - dimension. Default to None. + points_dim (int, optional): Number of the dimension of a point. + Each row is (x, y, z). Defaults to 3. + attribute_dims (dict, optional): Dictionary to indicate the + meaning of extra dimension. Defaults to None. Attributes: tensor (torch.Tensor): Float matrix of N x points_dim. points_dim (int): Integer indicating the dimension of a point. Each row is (x, y, z, ...). attribute_dims (bool): Dictionary to indicate the meaning of extra - dimension. Default to None. + dimension. Defaults to None. rotation_axis (int): Default rotation axis for points rotation. """ @@ -26,42 +26,35 @@ def __init__(self, tensor, points_dim=3, attribute_dims=None): self.rotation_axis = 1 def flip(self, bev_direction='horizontal'): - """Flip the boxes in BEV along given BEV direction.""" + """Flip the points along given BEV direction. + + Args: + bev_direction (str): Flip direction (horizontal or vertical). + """ if bev_direction == 'horizontal': self.tensor[:, 0] = -self.tensor[:, 0] elif bev_direction == 'vertical': self.tensor[:, 2] = -self.tensor[:, 2] - def in_range_bev(self, point_range): - """Check whether the points are in the given range. - - Args: - point_range (list | torch.Tensor): The range of point - in order of (x_min, y_min, x_max, y_max). - - Returns: - torch.Tensor: Indicating whether each point is inside \ - the reference range. - """ - in_range_flags = ((self.tensor[:, 0] > point_range[0]) - & (self.tensor[:, 2] > point_range[1]) - & (self.tensor[:, 0] < point_range[2]) - & (self.tensor[:, 2] < point_range[3])) - return in_range_flags + @property + def bev(self): + """torch.Tensor: BEV of the points in shape (N, 2).""" + return self.tensor[:, [0, 2]] def convert_to(self, dst, rt_mat=None): """Convert self to ``dst`` mode. Args: dst (:obj:`CoordMode`): The target Point mode. - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from `src` coordinates to `dst` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. Returns: - :obj:`BasePoints`: The converted point of the same type \ + :obj:`BasePoints`: The converted point of the same type in the `dst` mode. """ from mmdet3d.core.bbox import Coord3DMode diff --git a/mmdet3d/core/points/depth_points.py b/mmdet3d/core/points/depth_points.py index 3d194a1e03..de136de9ea 100644 --- a/mmdet3d/core/points/depth_points.py +++ b/mmdet3d/core/points/depth_points.py @@ -6,17 +6,17 @@ class DepthPoints(BasePoints): Args: tensor (torch.Tensor | np.ndarray | list): a N x points_dim matrix. - points_dim (int): Number of the dimension of a point. - Each row is (x, y, z). Default to 3. - attribute_dims (dict): Dictionary to indicate the meaning of extra - dimension. Default to None. + points_dim (int, optional): Number of the dimension of a point. + Each row is (x, y, z). Defaults to 3. + attribute_dims (dict, optional): Dictionary to indicate the + meaning of extra dimension. Defaults to None. Attributes: tensor (torch.Tensor): Float matrix of N x points_dim. points_dim (int): Integer indicating the dimension of a point. Each row is (x, y, z, ...). attribute_dims (bool): Dictionary to indicate the meaning of extra - dimension. Default to None. + dimension. Defaults to None. rotation_axis (int): Default rotation axis for points rotation. """ @@ -26,42 +26,30 @@ def __init__(self, tensor, points_dim=3, attribute_dims=None): self.rotation_axis = 2 def flip(self, bev_direction='horizontal'): - """Flip the boxes in BEV along given BEV direction.""" + """Flip the points along given BEV direction. + + Args: + bev_direction (str): Flip direction (horizontal or vertical). + """ if bev_direction == 'horizontal': self.tensor[:, 0] = -self.tensor[:, 0] elif bev_direction == 'vertical': self.tensor[:, 1] = -self.tensor[:, 1] - def in_range_bev(self, point_range): - """Check whether the points are in the given range. - - Args: - point_range (list | torch.Tensor): The range of point - in order of (x_min, y_min, x_max, y_max). - - Returns: - torch.Tensor: Indicating whether each point is inside \ - the reference range. - """ - in_range_flags = ((self.tensor[:, 0] > point_range[0]) - & (self.tensor[:, 1] > point_range[1]) - & (self.tensor[:, 0] < point_range[2]) - & (self.tensor[:, 1] < point_range[3])) - return in_range_flags - def convert_to(self, dst, rt_mat=None): """Convert self to ``dst`` mode. Args: dst (:obj:`CoordMode`): The target Point mode. - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from `src` coordinates to `dst` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. Returns: - :obj:`BasePoints`: The converted point of the same type \ + :obj:`BasePoints`: The converted point of the same type in the `dst` mode. """ from mmdet3d.core.bbox import Coord3DMode diff --git a/mmdet3d/core/points/lidar_points.py b/mmdet3d/core/points/lidar_points.py index f17323a3af..62df4bb183 100644 --- a/mmdet3d/core/points/lidar_points.py +++ b/mmdet3d/core/points/lidar_points.py @@ -6,17 +6,17 @@ class LiDARPoints(BasePoints): Args: tensor (torch.Tensor | np.ndarray | list): a N x points_dim matrix. - points_dim (int): Number of the dimension of a point. - Each row is (x, y, z). Default to 3. - attribute_dims (dict): Dictionary to indicate the meaning of extra - dimension. Default to None. + points_dim (int, optional): Number of the dimension of a point. + Each row is (x, y, z). Defaults to 3. + attribute_dims (dict, optional): Dictionary to indicate the + meaning of extra dimension. Defaults to None. Attributes: tensor (torch.Tensor): Float matrix of N x points_dim. points_dim (int): Integer indicating the dimension of a point. Each row is (x, y, z, ...). attribute_dims (bool): Dictionary to indicate the meaning of extra - dimension. Default to None. + dimension. Defaults to None. rotation_axis (int): Default rotation axis for points rotation. """ @@ -26,42 +26,30 @@ def __init__(self, tensor, points_dim=3, attribute_dims=None): self.rotation_axis = 2 def flip(self, bev_direction='horizontal'): - """Flip the boxes in BEV along given BEV direction.""" + """Flip the points along given BEV direction. + + Args: + bev_direction (str): Flip direction (horizontal or vertical). + """ if bev_direction == 'horizontal': self.tensor[:, 1] = -self.tensor[:, 1] elif bev_direction == 'vertical': self.tensor[:, 0] = -self.tensor[:, 0] - def in_range_bev(self, point_range): - """Check whether the points are in the given range. - - Args: - point_range (list | torch.Tensor): The range of point - in order of (x_min, y_min, x_max, y_max). - - Returns: - torch.Tensor: Indicating whether each point is inside \ - the reference range. - """ - in_range_flags = ((self.tensor[:, 0] > point_range[0]) - & (self.tensor[:, 1] > point_range[1]) - & (self.tensor[:, 0] < point_range[2]) - & (self.tensor[:, 1] < point_range[3])) - return in_range_flags - def convert_to(self, dst, rt_mat=None): """Convert self to ``dst`` mode. Args: dst (:obj:`CoordMode`): The target Point mode. - rt_mat (np.ndarray | torch.Tensor): The rotation and translation - matrix between different coordinates. Defaults to None. + rt_mat (np.ndarray | torch.Tensor, optional): The rotation and + translation matrix between different coordinates. + Defaults to None. The conversion from `src` coordinates to `dst` coordinates usually comes along the change of sensors, e.g., from camera to LiDAR. This requires a transformation matrix. Returns: - :obj:`BasePoints`: The converted point of the same type \ + :obj:`BasePoints`: The converted point of the same type in the `dst` mode. """ from mmdet3d.core.bbox import Coord3DMode diff --git a/mmdet3d/core/post_processing/box3d_nms.py b/mmdet3d/core/post_processing/box3d_nms.py index ebaa399e8f..4c0b0dc867 100644 --- a/mmdet3d/core/post_processing/box3d_nms.py +++ b/mmdet3d/core/post_processing/box3d_nms.py @@ -14,13 +14,15 @@ def box3d_multiclass_nms(mlvl_bboxes, mlvl_dir_scores=None, mlvl_attr_scores=None, mlvl_bboxes2d=None): - """Multi-class nms for 3D boxes. + """Multi-class NMS for 3D boxes. The IoU used for NMS is defined as the 2D + IoU between BEV boxes. Args: mlvl_bboxes (torch.Tensor): Multi-level boxes with shape (N, M). M is the dimensions of boxes. mlvl_bboxes_for_nms (torch.Tensor): Multi-level boxes with shape (N, 5) ([x1, y1, x2, y2, ry]). N is the number of boxes. + The coordinate system of the BEV boxes is counterclockwise. mlvl_scores (torch.Tensor): Multi-level boxes with shape (N, C + 1). N is the number of boxes. C is the number of classes. score_thr (float): Score thredhold to filter boxes with low @@ -35,8 +37,8 @@ def box3d_multiclass_nms(mlvl_bboxes, boxes. Defaults to None. Returns: - tuple[torch.Tensor]: Return results after nms, including 3D \ - bounding boxes, scores, labels, direction scores, attribute \ + tuple[torch.Tensor]: Return results after nms, including 3D + bounding boxes, scores, labels, direction scores, attribute scores (optional) and 2D bounding boxes (optional). """ # do multi class nms @@ -127,13 +129,13 @@ def box3d_multiclass_nms(mlvl_bboxes, def aligned_3d_nms(boxes, scores, classes, thresh): - """3d nms for aligned boxes. + """3D NMS for aligned boxes. Args: boxes (torch.Tensor): Aligned box with shape [n, 6]. scores (torch.Tensor): Scores of each box. classes (torch.Tensor): Class of each box. - thresh (float): Iou threshold for nms. + thresh (float): IoU threshold for nms. Returns: torch.Tensor: Indices of selected boxes. @@ -187,8 +189,8 @@ def circle_nms(dets, thresh, post_max_size=83): Args: dets (torch.Tensor): Detection results with the shape of [N, 3]. thresh (float): Value of threshold. - post_max_size (int): Max number of prediction to be kept. Defaults - to 83 + post_max_size (int, optional): Max number of prediction to be kept. + Defaults to 83. Returns: torch.Tensor: Indexes of the detections to be kept. diff --git a/mmdet3d/core/utils/array_converter.py b/mmdet3d/core/utils/array_converter.py index eeb3699973..fa623afee4 100644 --- a/mmdet3d/core/utils/array_converter.py +++ b/mmdet3d/core/utils/array_converter.py @@ -11,19 +11,20 @@ def array_converter(to_torch=True, """Wrapper function for data-type agnostic processing. First converts input arrays to PyTorch tensors or NumPy ndarrays - for middle calculation, then convert output to original data-type. + for middle calculation, then convert output to original data-type if + `recover=True`. Args: - to_torch (Bool): Whether convert to PyTorch tensors + to_torch (Bool, optional): Whether convert to PyTorch tensors for middle calculation. Defaults to True. - apply_to (tuple[str]): The arguments to which we apply data-type - conversion. Defaults to an empty tuple. - template_arg_name_ (str): Argument serving as the template ( + apply_to (tuple[str], optional): The arguments to which we apply + data-type conversion. Defaults to an empty tuple. + template_arg_name_ (str, optional): Argument serving as the template ( return arrays should have the same dtype and device as the template). Defaults to None. If None, we will use the first argument in `apply_to` as the template argument. - recover (Bool): Whether or not recover the wrapped function outputs - to the `template_arg_name_` type. Defaults to True. + recover (Bool, optional): Whether or not recover the wrapped function + outputs to the `template_arg_name_` type. Defaults to True. Raises: ValueError: When template_arg_name_ is not among all args, or @@ -254,9 +255,10 @@ def convert(self, input_array, target_type=None, target_array=None): input_array (tuple | list | np.ndarray | torch.Tensor | int | float ): Input array. Defaults to None. - target_type ( | ): + target_type ( | , + optional): Type to which input array is converted. Defaults to None. - target_array (np.ndarray | torch.Tensor): + target_array (np.ndarray | torch.Tensor, optional): Template array to which input array is converted. Defaults to None. diff --git a/mmdet3d/core/utils/gaussian.py b/mmdet3d/core/utils/gaussian.py index 28605f2601..aa1bc408a6 100644 --- a/mmdet3d/core/utils/gaussian.py +++ b/mmdet3d/core/utils/gaussian.py @@ -7,7 +7,7 @@ def gaussian_2d(shape, sigma=1): Args: shape (list[int]): Shape of the map. - sigma (float): Sigma to generate gaussian map. + sigma (float, optional): Sigma to generate gaussian map. Defaults to 1. Returns: @@ -28,7 +28,7 @@ def draw_heatmap_gaussian(heatmap, center, radius, k=1): heatmap (torch.Tensor): Heatmap to be masked. center (torch.Tensor): Center coord of the heatmap. radius (int): Radius of gausian. - K (int): Multiple of masked_gaussian. Defaults to 1. + K (int, optional): Multiple of masked_gaussian. Defaults to 1. Returns: torch.Tensor: Masked heatmap. @@ -58,7 +58,7 @@ def gaussian_radius(det_size, min_overlap=0.5): Args: det_size (tuple[torch.Tensor]): Size of the detection result. - min_overlap (float): Gaussian_overlap. Defaults to 0.5. + min_overlap (float, optional): Gaussian_overlap. Defaults to 0.5. Returns: torch.Tensor: Computed radius. diff --git a/mmdet3d/core/visualizer/image_vis.py b/mmdet3d/core/visualizer/image_vis.py index 94c415835e..7e4679c529 100644 --- a/mmdet3d/core/visualizer/image_vis.py +++ b/mmdet3d/core/visualizer/image_vis.py @@ -17,7 +17,7 @@ def project_pts_on_img(points, raw_img (numpy.array): The numpy array of image. lidar2img_rt (numpy.array, shape=[4, 4]): The projection matrix according to the camera intrinsic parameters. - max_distance (float): the max distance of the points cloud. + max_distance (float, optional): the max distance of the points cloud. Default: 70. thickness (int, optional): The thickness of 2D points. Default: -1. """ @@ -68,7 +68,8 @@ def plot_rect3d_on_img(img, num_rects (int): Number of 3D rectangulars. rect_corners (numpy.array): Coordinates of the corners of 3D rectangulars. Should be in the shape of [num_rect, 8, 2]. - color (tuple[int]): The color to draw bboxes. Default: (0, 255, 0). + color (tuple[int], optional): The color to draw bboxes. + Default: (0, 255, 0). thickness (int, optional): The thickness of bboxes. Default: 1. """ line_indices = ((0, 1), (0, 3), (0, 4), (1, 2), (1, 5), (3, 2), (3, 7), @@ -98,7 +99,8 @@ def draw_lidar_bbox3d_on_img(bboxes3d, lidar2img_rt (numpy.array, shape=[4, 4]): The projection matrix according to the camera intrinsic parameters. img_metas (dict): Useless here. - color (tuple[int]): The color to draw bboxes. Default: (0, 255, 0). + color (tuple[int], optional): The color to draw bboxes. + Default: (0, 255, 0). thickness (int, optional): The thickness of bboxes. Default: 1. """ img = raw_img.copy() @@ -135,7 +137,8 @@ def draw_depth_bbox3d_on_img(bboxes3d, raw_img (numpy.array): The numpy array of image. calibs (dict): Camera calibration information, Rt and K. img_metas (dict): Used in coordinates transformation. - color (tuple[int]): The color to draw bboxes. Default: (0, 255, 0). + color (tuple[int], optional): The color to draw bboxes. + Default: (0, 255, 0). thickness (int, optional): The thickness of bboxes. Default: 1. """ from mmdet3d.core.bbox import points_cam2img @@ -175,7 +178,8 @@ def draw_camera_bbox3d_on_img(bboxes3d, cam2img (dict): Camera intrinsic matrix, denoted as `K` in depth bbox coordinate system. img_metas (dict): Useless here. - color (tuple[int]): The color to draw bboxes. Default: (0, 255, 0). + color (tuple[int], optional): The color to draw bboxes. + Default: (0, 255, 0). thickness (int, optional): The thickness of bboxes. Default: 1. """ from mmdet3d.core.bbox import points_cam2img diff --git a/mmdet3d/core/visualizer/open3d_vis.py b/mmdet3d/core/visualizer/open3d_vis.py index 3a2933c3cd..b15a9d2a9a 100644 --- a/mmdet3d/core/visualizer/open3d_vis.py +++ b/mmdet3d/core/visualizer/open3d_vis.py @@ -21,12 +21,12 @@ def _draw_points(points, points (numpy.array | torch.tensor, shape=[N, 3+C]): points to visualize. vis (:obj:`open3d.visualization.Visualizer`): open3d visualizer. - points_size (int): the size of points to show on visualizer. + points_size (int, optional): the size of points to show on visualizer. Default: 2. - point_color (tuple[float]): the color of points. + point_color (tuple[float], optional): the color of points. Default: (0.5, 0.5, 0.5). - mode (str): indicate type of the input points, avaliable mode - ['xyz', 'xyzrgb']. Default: 'xyz'. + mode (str, optional): indicate type of the input points, + available mode ['xyz', 'xyzrgb']. Default: 'xyz'. Returns: tuple: points, color of each point. @@ -68,19 +68,21 @@ def _draw_bboxes(bbox3d, Args: bbox3d (numpy.array | torch.tensor, shape=[M, 7]): - 3d bbox (x, y, z, dx, dy, dz, yaw) to visualize. + 3d bbox (x, y, z, x_size, y_size, z_size, yaw) to visualize. vis (:obj:`open3d.visualization.Visualizer`): open3d visualizer. points_colors (numpy.array): color of each points. - pcd (:obj:`open3d.geometry.PointCloud`): point cloud. Default: None. - bbox_color (tuple[float]): the color of bbox. Default: (0, 1, 0). - points_in_box_color (tuple[float]): + pcd (:obj:`open3d.geometry.PointCloud`, optional): point cloud. + Default: None. + bbox_color (tuple[float], optional): the color of bbox. + Default: (0, 1, 0). + points_in_box_color (tuple[float], optional): the color of points inside bbox3d. Default: (1, 0, 0). - rot_axis (int): rotation axis of bbox. Default: 2. - center_mode (bool): indicate the center of bbox is bottom center - or gravity center. avaliable mode + rot_axis (int, optional): rotation axis of bbox. Default: 2. + center_mode (bool, optional): indicate the center of bbox is + bottom center or gravity center. avaliable mode ['lidar_bottom', 'camera_bottom']. Default: 'lidar_bottom'. - mode (str): indicate type of the input points, avaliable mode - ['xyz', 'xyzrgb']. Default: 'xyz'. + mode (str, optional): indicate type of the input points, + avaliable mode ['xyz', 'xyzrgb']. Default: 'xyz'. """ if isinstance(bbox3d, torch.Tensor): bbox3d = bbox3d.cpu().numpy() @@ -134,23 +136,27 @@ def show_pts_boxes(points, Args: points (numpy.array | torch.tensor, shape=[N, 3+C]): points to visualize. - bbox3d (numpy.array | torch.tensor, shape=[M, 7]): - 3d bbox (x, y, z, dx, dy, dz, yaw) to visualize. Default: None. - show (bool): whether to show the visualization results. Default: True. - save_path (str): path to save visualized results. Default: None. - points_size (int): the size of points to show on visualizer. + bbox3d (numpy.array | torch.tensor, shape=[M, 7], optional): + 3D bbox (x, y, z, x_size, y_size, z_size, yaw) to visualize. + Defaults to None. + show (bool, optional): whether to show the visualization results. + Default: True. + save_path (str, optional): path to save visualized results. + Default: None. + points_size (int, optional): the size of points to show on visualizer. Default: 2. - point_color (tuple[float]): the color of points. + point_color (tuple[float], optional): the color of points. Default: (0.5, 0.5, 0.5). - bbox_color (tuple[float]): the color of bbox. Default: (0, 1, 0). - points_in_box_color (tuple[float]): + bbox_color (tuple[float], optional): the color of bbox. + Default: (0, 1, 0). + points_in_box_color (tuple[float], optional): the color of points which are in bbox3d. Default: (1, 0, 0). - rot_axis (int): rotation axis of bbox. Default: 2. - center_mode (bool): indicate the center of bbox is bottom center - or gravity center. avaliable mode + rot_axis (int, optional): rotation axis of bbox. Default: 2. + center_mode (bool, optional): indicate the center of bbox is bottom + center or gravity center. avaliable mode ['lidar_bottom', 'camera_bottom']. Default: 'lidar_bottom'. - mode (str): indicate type of the input points, avaliable mode - ['xyz', 'xyzrgb']. Default: 'xyz'. + mode (str, optional): indicate type of the input points, avaliable + mode ['xyz', 'xyzrgb']. Default: 'xyz'. """ # TODO: support score and class info assert 0 <= rot_axis <= 2 @@ -195,21 +201,23 @@ def _draw_bboxes_ind(bbox3d, Args: bbox3d (numpy.array | torch.tensor, shape=[M, 7]): - 3d bbox (x, y, z, dx, dy, dz, yaw) to visualize. + 3d bbox (x, y, z, x_size, y_size, z_size, yaw) to visualize. vis (:obj:`open3d.visualization.Visualizer`): open3d visualizer. indices (numpy.array | torch.tensor, shape=[N, M]): indicate which bbox3d that each point lies in. points_colors (numpy.array): color of each points. - pcd (:obj:`open3d.geometry.PointCloud`): point cloud. Default: None. - bbox_color (tuple[float]): the color of bbox. Default: (0, 1, 0). - points_in_box_color (tuple[float]): + pcd (:obj:`open3d.geometry.PointCloud`, optional): point cloud. + Default: None. + bbox_color (tuple[float], optional): the color of bbox. + Default: (0, 1, 0). + points_in_box_color (tuple[float], optional): the color of points which are in bbox3d. Default: (1, 0, 0). - rot_axis (int): rotation axis of bbox. Default: 2. - center_mode (bool): indicate the center of bbox is bottom center - or gravity center. avaliable mode + rot_axis (int, optional): rotation axis of bbox. Default: 2. + center_mode (bool, optional): indicate the center of bbox is + bottom center or gravity center. avaliable mode ['lidar_bottom', 'camera_bottom']. Default: 'lidar_bottom'. - mode (str): indicate type of the input points, avaliable mode - ['xyz', 'xyzrgb']. Default: 'xyz'. + mode (str, optional): indicate type of the input points, + avaliable mode ['xyz', 'xyzrgb']. Default: 'xyz'. """ if isinstance(bbox3d, torch.Tensor): bbox3d = bbox3d.cpu().numpy() @@ -269,24 +277,28 @@ def show_pts_index_boxes(points, points (numpy.array | torch.tensor, shape=[N, 3+C]): points to visualize. bbox3d (numpy.array | torch.tensor, shape=[M, 7]): - 3d bbox (x, y, z, dx, dy, dz, yaw) to visualize. Default: None. - show (bool): whether to show the visualization results. Default: True. - indices (numpy.array | torch.tensor, shape=[N, M]): + 3D bbox (x, y, z, x_size, y_size, z_size, yaw) to visualize. + Defaults to None. + show (bool, optional): whether to show the visualization results. + Default: True. + indices (numpy.array | torch.tensor, shape=[N, M], optional): indicate which bbox3d that each point lies in. Default: None. - save_path (str): path to save visualized results. Default: None. - points_size (int): the size of points to show on visualizer. + save_path (str, optional): path to save visualized results. + Default: None. + points_size (int, optional): the size of points to show on visualizer. Default: 2. - point_color (tuple[float]): the color of points. + point_color (tuple[float], optional): the color of points. Default: (0.5, 0.5, 0.5). - bbox_color (tuple[float]): the color of bbox. Default: (0, 1, 0). - points_in_box_color (tuple[float]): + bbox_color (tuple[float], optional): the color of bbox. + Default: (0, 1, 0). + points_in_box_color (tuple[float], optional): the color of points which are in bbox3d. Default: (1, 0, 0). - rot_axis (int): rotation axis of bbox. Default: 2. - center_mode (bool): indicate the center of bbox is bottom center - or gravity center. avaliable mode + rot_axis (int, optional): rotation axis of bbox. Default: 2. + center_mode (bool, optional): indicate the center of bbox is + bottom center or gravity center. avaliable mode ['lidar_bottom', 'camera_bottom']. Default: 'lidar_bottom'. - mode (str): indicate type of the input points, avaliable mode - ['xyz', 'xyzrgb']. Default: 'xyz'. + mode (str, optional): indicate type of the input points, + avaliable mode ['xyz', 'xyzrgb']. Default: 'xyz'. """ # TODO: support score and class info assert 0 <= rot_axis <= 2 @@ -323,24 +335,27 @@ class Visualizer(object): points (numpy.array, shape=[N, 3+C]): Points to visualize. The Points cloud is in mode of Coord3DMode.DEPTH (please refer to core.structures.coord_3d_mode). - bbox3d (numpy.array, shape=[M, 7]): 3d bbox (x, y, z, dx, dy, dz, yaw) - to visualize. The 3d bbox is in mode of Box3DMode.DEPTH with + bbox3d (numpy.array, shape=[M, 7], optional): 3D bbox + (x, y, z, x_size, y_size, z_size, yaw) to visualize. + The 3D bbox is in mode of Box3DMode.DEPTH with gravity_center (please refer to core.structures.box_3d_mode). Default: None. - save_path (str): path to save visualized results. Default: None. - points_size (int): the size of points to show on visualizer. + save_path (str, optional): path to save visualized results. + Default: None. + points_size (int, optional): the size of points to show on visualizer. Default: 2. - point_color (tuple[float]): the color of points. + point_color (tuple[float], optional): the color of points. Default: (0.5, 0.5, 0.5). - bbox_color (tuple[float]): the color of bbox. Default: (0, 1, 0). - points_in_box_color (tuple[float]): + bbox_color (tuple[float], optional): the color of bbox. + Default: (0, 1, 0). + points_in_box_color (tuple[float], optional): the color of points which are in bbox3d. Default: (1, 0, 0). - rot_axis (int): rotation axis of bbox. Default: 2. - center_mode (bool): indicate the center of bbox is bottom center - or gravity center. avaliable mode + rot_axis (int, optional): rotation axis of bbox. Default: 2. + center_mode (bool, optional): indicate the center of bbox is + bottom center or gravity center. avaliable mode ['lidar_bottom', 'camera_bottom']. Default: 'lidar_bottom'. - mode (str): indicate type of the input points, avaliable mode - ['xyz', 'xyzrgb']. Default: 'xyz'. + mode (str, optional): indicate type of the input points, + avaliable mode ['xyz', 'xyzrgb']. Default: 'xyz'. """ def __init__(self, @@ -389,9 +404,10 @@ def add_bboxes(self, bbox3d, bbox_color=None, points_in_box_color=None): Args: bbox3d (numpy.array, shape=[M, 7]): - 3D bbox (x, y, z, dx, dy, dz, yaw) to be visualized. - The 3d bbox is in mode of Box3DMode.DEPTH with - gravity_center (please refer to core.structures.box_3d_mode). + 3D bbox (x, y, z, x_size, y_size, z_size, yaw) + to be visualized. The 3d bbox is in mode of + Box3DMode.DEPTH with gravity_center (please refer to + core.structures.box_3d_mode). bbox_color (tuple[float]): the color of bbox. Defaule: None. points_in_box_color (tuple[float]): the color of points which are in bbox3d. Defaule: None. @@ -430,7 +446,7 @@ def show(self, save_path=None): """Visualize the points cloud. Args: - save_path (str): path to save image. Default: None. + save_path (str, optional): path to save image. Default: None. """ self.o3d_visualizer.run() diff --git a/mmdet3d/core/visualizer/show_result.py b/mmdet3d/core/visualizer/show_result.py index 20e46ca7d4..fad828a4f9 100644 --- a/mmdet3d/core/visualizer/show_result.py +++ b/mmdet3d/core/visualizer/show_result.py @@ -34,7 +34,7 @@ def _write_oriented_bbox(scene_bbox, out_filename): Args: scene_bbox(list[ndarray] or ndarray): xyz pos of center and - 3 lengths (dx,dy,dz) and heading angle around Z axis. + 3 lengths (x_size, y_size, z_size) and heading angle around Z axis. Y forward, X right, Z upward. heading angle of positive X is 0, heading angle of positive Y is 90 degrees. out_filename(str): Filename. @@ -86,8 +86,10 @@ def show_result(points, pred_bboxes (np.ndarray): Predicted boxes. out_dir (str): Path of output directory filename (str): Filename of the current frame. - show (bool): Visualize the results online. Defaults to False. - snapshot (bool): Whether to save the online results. Defaults to False. + show (bool, optional): Visualize the results online. + Defaults to False. + snapshot (bool, optional): Whether to save the online results. + Defaults to False. """ result_path = osp.join(out_dir, filename) mmcv.mkdir_or_exist(result_path) @@ -140,10 +142,10 @@ def show_seg_result(points, out_dir (str): Path of output directory filename (str): Filename of the current frame. palette (np.ndarray): Mapping between class labels and colors. - ignore_index (int, optional): The label index to be ignored, e.g. \ + ignore_index (int, optional): The label index to be ignored, e.g. unannotated points. Defaults to None. show (bool, optional): Visualize the results online. Defaults to False. - snapshot (bool, optional): Whether to save the online results. \ + snapshot (bool, optional): Whether to save the online results. Defaults to False. """ # we need 3D coordinates to visualize segmentation mask @@ -218,14 +220,16 @@ def show_multi_modality_result(img, according to the camera intrinsic parameters. out_dir (str): Path of output directory. filename (str): Filename of the current frame. - box_mode (str): Coordinate system the boxes are in. Should be one of - 'depth', 'lidar' and 'camera'. Defaults to 'lidar'. - img_metas (dict): Used in projecting depth bbox. - show (bool): Visualize the results online. Defaults to False. - gt_bbox_color (str or tuple(int)): Color of bbox lines. - The tuple of color should be in BGR order. Default: (255, 102, 61) - pred_bbox_color (str or tuple(int)): Color of bbox lines. - The tuple of color should be in BGR order. Default: (72, 101, 241) + box_mode (str, optional): Coordinate system the boxes are in. + Should be one of 'depth', 'lidar' and 'camera'. + Defaults to 'lidar'. + img_metas (dict, optional): Used in projecting depth bbox. + Defaults to None. + show (bool, optional): Visualize the results online. Defaults to False. + gt_bbox_color (str or tuple(int), optional): Color of bbox lines. + The tuple of color should be in BGR order. Default: (255, 102, 61). + pred_bbox_color (str or tuple(int), optional): Color of bbox lines. + The tuple of color should be in BGR order. Default: (72, 101, 241). """ if box_mode == 'depth': draw_bbox = draw_depth_bbox3d_on_img diff --git a/mmdet3d/core/voxel/voxel_generator.py b/mmdet3d/core/voxel/voxel_generator.py index 7a6a5fecc8..748b347ef1 100644 --- a/mmdet3d/core/voxel/voxel_generator.py +++ b/mmdet3d/core/voxel/voxel_generator.py @@ -81,18 +81,18 @@ def points_to_voxel(points, """convert kitti points(N, >=3) to voxels. Args: - points (np.ndarray): [N, ndim]. points[:, :3] contain xyz points and \ + points (np.ndarray): [N, ndim]. points[:, :3] contain xyz points and points[:, 3:] contain other information such as reflectivity. voxel_size (list, tuple, np.ndarray): [3] xyz, indicate voxel size - coors_range (list[float | tuple[float] | ndarray]): Voxel range. \ + coors_range (list[float | tuple[float] | ndarray]): Voxel range. format: xyzxyz, minmax max_points (int): Indicate maximum points contained in a voxel. - reverse_index (bool): Whether return reversed coordinates. \ - if points has xyz format and reverse_index is True, output \ - coordinates will be zyx format, but points in features always \ + reverse_index (bool): Whether return reversed coordinates. + if points has xyz format and reverse_index is True, output + coordinates will be zyx format, but points in features always xyz format. - max_voxels (int): Maximum number of voxels this function creates. \ - For second, 20000 is a good choice. Points should be shuffled for \ + max_voxels (int): Maximum number of voxels this function creates. + For second, 20000 is a good choice. Points should be shuffled for randomness before this function because max_voxels drops points. Returns: @@ -146,20 +146,20 @@ def _points_to_voxel_reverse_kernel(points, """convert kitti points(N, >=3) to voxels. Args: - points (np.ndarray): [N, ndim]. points[:, :3] contain xyz points and \ + points (np.ndarray): [N, ndim]. points[:, :3] contain xyz points and points[:, 3:] contain other information such as reflectivity. - voxel_size (list, tuple, np.ndarray): [3] xyz, indicate voxel size \ - coors_range (list[float | tuple[float] | ndarray]): Range of voxels. \ + voxel_size (list, tuple, np.ndarray): [3] xyz, indicate voxel size + coors_range (list[float | tuple[float] | ndarray]): Range of voxels. format: xyzxyz, minmax num_points_per_voxel (int): Number of points per voxel. - coor_to_voxel_idx (np.ndarray): A voxel grid of shape (D, H, W), \ - which has the same shape as the complete voxel map. It indicates \ + coor_to_voxel_idx (np.ndarray): A voxel grid of shape (D, H, W), + which has the same shape as the complete voxel map. It indicates the index of each corresponding voxel. voxels (np.ndarray): Created empty voxels. coors (np.ndarray): Created coordinates of each voxel. max_points (int): Indicate maximum points contained in a voxel. - max_voxels (int): Maximum number of voxels this function create. \ - for second, 20000 is a good choice. Points should be shuffled for \ + max_voxels (int): Maximum number of voxels this function create. + for second, 20000 is a good choice. Points should be shuffled for randomness before this function because max_voxels drops points. Returns: @@ -220,20 +220,20 @@ def _points_to_voxel_kernel(points, """convert kitti points(N, >=3) to voxels. Args: - points (np.ndarray): [N, ndim]. points[:, :3] contain xyz points and \ + points (np.ndarray): [N, ndim]. points[:, :3] contain xyz points and points[:, 3:] contain other information such as reflectivity. voxel_size (list, tuple, np.ndarray): [3] xyz, indicate voxel size. - coors_range (list[float | tuple[float] | ndarray]): Range of voxels. \ + coors_range (list[float | tuple[float] | ndarray]): Range of voxels. format: xyzxyz, minmax num_points_per_voxel (int): Number of points per voxel. - coor_to_voxel_idx (np.ndarray): A voxel grid of shape (D, H, W), \ - which has the same shape as the complete voxel map. It indicates \ + coor_to_voxel_idx (np.ndarray): A voxel grid of shape (D, H, W), + which has the same shape as the complete voxel map. It indicates the index of each corresponding voxel. voxels (np.ndarray): Created empty voxels. coors (np.ndarray): Created coordinates of each voxel. max_points (int): Indicate maximum points contained in a voxel. - max_voxels (int): Maximum number of voxels this function create. \ - for second, 20000 is a good choice. Points should be shuffled for \ + max_voxels (int): Maximum number of voxels this function create. + for second, 20000 is a good choice. Points should be shuffled for randomness before this function because max_voxels drops points. Returns: diff --git a/mmdet3d/datasets/custom_3d.py b/mmdet3d/datasets/custom_3d.py index 2d3f9feda2..d21a1b0a19 100644 --- a/mmdet3d/datasets/custom_3d.py +++ b/mmdet3d/datasets/custom_3d.py @@ -87,7 +87,7 @@ def get_data_info(self, index): index (int): Index of the sample data to get. Returns: - dict: Data information that will be passed to the data \ + dict: Data information that will be passed to the data preprocessing pipelines. It includes the following keys: - sample_idx (str): Sample index. @@ -176,7 +176,7 @@ def get_classes(cls, classes=None): """Get class names of current dataset. Args: - classes (Sequence[str] | str | None): If classes is None, use + classes (Sequence[str] | str): If classes is None, use default CLASSES defined by builtin dataset. If classes is a string, take it as a file name. The file contains the name of classes where each line contains one class name. If classes is @@ -206,13 +206,13 @@ def format_results(self, Args: outputs (list[dict]): Testing results of the dataset. - pklfile_prefix (str | None): The prefix of pkl files. It includes + pklfile_prefix (str): The prefix of pkl files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. Returns: - tuple: (outputs, tmp_dir), outputs is the detection results, \ - tmp_dir is the temporal directory created for saving json \ + tuple: (outputs, tmp_dir), outputs is the detection results, + tmp_dir is the temporal directory created for saving json files when ``jsonfile_prefix`` is not specified. """ if pklfile_prefix is None: @@ -236,11 +236,14 @@ def evaluate(self, Args: results (list[dict]): List of results. - metric (str | list[str]): Metrics to be evaluated. - iou_thr (list[float]): AP IoU thresholds. - show (bool): Whether to visualize. + metric (str | list[str], optional): Metrics to be evaluated. + Defaults to None. + iou_thr (list[float]): AP IoU thresholds. Defaults to (0.25, 0.5). + logger (logging.Logger | str, optional): Logger used for printing + related information during evaluation. Defaults to None. + show (bool, optional): Whether to visualize. Default: False. - out_dir (str): Path to save the visualization results. + out_dir (str, optional): Path to save the visualization results. Default: None. pipeline (list[dict], optional): raw data loading for showing. Default: None. @@ -280,7 +283,7 @@ def _get_pipeline(self, pipeline): """Get data loading pipeline in self.show/evaluate function. Args: - pipeline (list[dict] | None): Input pipeline. If None is given, \ + pipeline (list[dict]): Input pipeline. If None is given, get from self.pipeline. """ if pipeline is None: diff --git a/mmdet3d/datasets/custom_3d_seg.py b/mmdet3d/datasets/custom_3d_seg.py index afd1b45051..60d1cee36e 100644 --- a/mmdet3d/datasets/custom_3d_seg.py +++ b/mmdet3d/datasets/custom_3d_seg.py @@ -31,7 +31,7 @@ class Custom3DSegDataset(Dataset): as input. Defaults to None. test_mode (bool, optional): Whether the dataset is in test mode. Defaults to False. - ignore_index (int, optional): The label index to be ignored, e.g. \ + ignore_index (int, optional): The label index to be ignored, e.g. unannotated points. If None is given, set to len(self.CLASSES) to be consistent with PointSegClassMapping function in pipeline. Defaults to None. @@ -101,7 +101,7 @@ def get_data_info(self, index): index (int): Index of the sample data to get. Returns: - dict: Data information that will be passed to the data \ + dict: Data information that will be passed to the data preprocessing pipelines. It includes the following keys: - sample_idx (str): Sample index. @@ -178,13 +178,13 @@ def get_classes_and_palette(self, classes=None, palette=None): This function is taken from MMSegmentation. Args: - classes (Sequence[str] | str | None): If classes is None, use + classes (Sequence[str] | str): If classes is None, use default CLASSES defined by builtin dataset. If classes is a string, take it as a file name. The file contains the name of classes where each line contains one class name. If classes is a tuple or list, override the CLASSES defined by the dataset. Defaults to None. - palette (Sequence[Sequence[int]]] | np.ndarray | None): + palette (Sequence[Sequence[int]]] | np.ndarray): The palette of segmentation map. If None is given, random palette will be generated. Defaults to None. """ @@ -275,13 +275,13 @@ def format_results(self, Args: outputs (list[dict]): Testing results of the dataset. - pklfile_prefix (str | None): The prefix of pkl files. It includes + pklfile_prefix (str): The prefix of pkl files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. Returns: - tuple: (outputs, tmp_dir), outputs is the detection results, \ - tmp_dir is the temporal directory created for saving json \ + tuple: (outputs, tmp_dir), outputs is the detection results, + tmp_dir is the temporal directory created for saving json files when ``jsonfile_prefix`` is not specified. """ if pklfile_prefix is None: @@ -305,7 +305,7 @@ def evaluate(self, Args: results (list[dict]): List of results. metric (str | list[str]): Metrics to be evaluated. - logger (logging.Logger | None | str): Logger used for printing + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Defaults to None. show (bool, optional): Whether to visualize. Defaults to False. @@ -363,7 +363,7 @@ def _get_pipeline(self, pipeline): """Get data loading pipeline in self.show/evaluate function. Args: - pipeline (list[dict] | None): Input pipeline. If None is given, \ + pipeline (list[dict]): Input pipeline. If None is given, get from self.pipeline. """ if pipeline is None: diff --git a/mmdet3d/datasets/kitti2d_dataset.py b/mmdet3d/datasets/kitti2d_dataset.py index 6fd887f3ab..2004f90101 100644 --- a/mmdet3d/datasets/kitti2d_dataset.py +++ b/mmdet3d/datasets/kitti2d_dataset.py @@ -205,7 +205,8 @@ def reformat_bbox(self, outputs, out=None): Args: outputs (list[np.ndarray]): List of arrays storing the inferenced bounding boxes and scores. - out (str | None): The prefix of output file. Default: None. + out (str, optional): The prefix of output file. + Default: None. Returns: list[dict]: A list of dictionaries with the kitti 2D format. @@ -221,7 +222,7 @@ def evaluate(self, result_files, eval_types=None): Args: result_files (str): Path of result files. - eval_types (str): Types of evaluation. Default: None. + eval_types (str, optional): Types of evaluation. Default: None. KITTI dataset only support 'bbox' evaluation type. Returns: diff --git a/mmdet3d/datasets/kitti_dataset.py b/mmdet3d/datasets/kitti_dataset.py index 23dc207f89..97d6917032 100644 --- a/mmdet3d/datasets/kitti_dataset.py +++ b/mmdet3d/datasets/kitti_dataset.py @@ -46,8 +46,9 @@ class KittiDataset(Custom3DDataset): Defaults to True. test_mode (bool, optional): Whether the dataset is in test mode. Defaults to False. - pcd_limit_range (list): The range of point cloud used to filter - invalid predicted boxes. Default: [0, -40, -3, 70.4, 40, 0.0]. + pcd_limit_range (list, optional): The range of point cloud used to + filter invalid predicted boxes. + Default: [0, -40, -3, 70.4, 40, 0.0]. """ CLASSES = ('car', 'pedestrian', 'cyclist') @@ -99,14 +100,14 @@ def get_data_info(self, index): index (int): Index of the sample data to get. Returns: - dict: Data information that will be passed to the data \ + dict: Data information that will be passed to the data preprocessing pipelines. It includes the following keys: - sample_idx (str): Sample index. - pts_filename (str): Filename of point clouds. - - img_prefix (str | None): Prefix of image files. + - img_prefix (str): Prefix of image files. - img_info (dict): Image info. - - lidar2img (list[np.ndarray], optional): Transformations \ + - lidar2img (list[np.ndarray], optional): Transformations from lidar to different cameras. - ann_info (dict): Annotation info. """ @@ -144,7 +145,7 @@ def get_ann_info(self, index): Returns: dict: annotation information consists of the following keys: - - gt_bboxes_3d (:obj:`LiDARInstance3DBoxes`): \ + - gt_bboxes_3d (:obj:`LiDARInstance3DBoxes`): 3D ground truth bboxes. - gt_labels_3d (np.ndarray): Labels of ground truths. - gt_bboxes (np.ndarray): 2D ground truth bboxes. @@ -247,17 +248,17 @@ def format_results(self, Args: outputs (list[dict]): Testing results of the dataset. - pklfile_prefix (str | None): The prefix of pkl files. It includes + pklfile_prefix (str): The prefix of pkl files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. - submission_prefix (str | None): The prefix of submitted files. It + submission_prefix (str): The prefix of submitted files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. Returns: - tuple: (result_files, tmp_dir), result_files is a dict containing \ - the json filepaths, tmp_dir is the temporal directory created \ + tuple: (result_files, tmp_dir), result_files is a dict containing + the json filepaths, tmp_dir is the temporal directory created for saving json files when jsonfile_prefix is not specified. """ if pklfile_prefix is None: @@ -307,17 +308,18 @@ def evaluate(self, Args: results (list[dict]): Testing results of the dataset. - metric (str | list[str]): Metrics to be evaluated. - logger (logging.Logger | str | None): Logger used for printing + metric (str | list[str], optional): Metrics to be evaluated. + Default: None. + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. - pklfile_prefix (str | None): The prefix of pkl files. It includes + pklfile_prefix (str, optional): The prefix of pkl files, including the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. - submission_prefix (str | None): The prefix of submission datas. + submission_prefix (str, optional): The prefix of submission datas. If not specified, the submission data will not be generated. - show (bool): Whether to visualize. + show (bool, optional): Whether to visualize. Default: False. - out_dir (str): Path to save the visualization results. + out_dir (str, optional): Path to save the visualization results. Default: None. pipeline (list[dict], optional): raw data loading for showing. Default: None. @@ -370,11 +372,11 @@ def bbox2result_kitti(self, submission. Args: - net_outputs (list[np.ndarray]): List of array storing the \ + net_outputs (list[np.ndarray]): List of array storing the inferenced bounding boxes and scores. class_names (list[String]): A list of class names. - pklfile_prefix (str | None): The prefix of pkl file. - submission_prefix (str | None): The prefix of submission file. + pklfile_prefix (str): The prefix of pkl file. + submission_prefix (str): The prefix of submission file. Returns: list[dict]: A list of dictionaries with the kitti format. @@ -485,11 +487,11 @@ def bbox2result_kitti2d(self, submission. Args: - net_outputs (list[np.ndarray]): List of array storing the \ + net_outputs (list[np.ndarray]): List of array storing the inferenced bounding boxes and scores. class_names (list[String]): A list of class names. - pklfile_prefix (str | None): The prefix of pkl file. - submission_prefix (str | None): The prefix of submission file. + pklfile_prefix (str): The prefix of pkl file. + submission_prefix (str): The prefix of submission file. Returns: list[dict]: A list of dictionaries have the kitti format @@ -603,9 +605,9 @@ def convert_valid_bboxes(self, box_dict, info): dict: Valid predicted boxes. - bbox (np.ndarray): 2D bounding boxes. - - box3d_camera (np.ndarray): 3D bounding boxes in \ + - box3d_camera (np.ndarray): 3D bounding boxes in camera coordinate. - - box3d_lidar (np.ndarray): 3D bounding boxes in \ + - box3d_lidar (np.ndarray): 3D bounding boxes in LiDAR coordinate. - scores (np.ndarray): Scores of boxes. - label_preds (np.ndarray): Class label predictions. diff --git a/mmdet3d/datasets/kitti_mono_dataset.py b/mmdet3d/datasets/kitti_mono_dataset.py index 5b9d31b12d..a3a17215e7 100644 --- a/mmdet3d/datasets/kitti_mono_dataset.py +++ b/mmdet3d/datasets/kitti_mono_dataset.py @@ -56,8 +56,8 @@ def _parse_ann_info(self, img_info, ann_info): with_mask (bool): Whether to parse mask annotations. Returns: - dict: A dict containing the following keys: bboxes, bboxes_ignore,\ - labels, masks, seg_map. "masks" are raw annotations and not \ + dict: A dict containing the following keys: bboxes, bboxes_ignore, + labels, masks, seg_map. "masks" are raw annotations and not decoded into binary masks. """ gt_bboxes = [] @@ -146,17 +146,17 @@ def format_results(self, Args: outputs (list[dict]): Testing results of the dataset. - pklfile_prefix (str | None): The prefix of pkl files. It includes + pklfile_prefix (str): The prefix of pkl files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. - submission_prefix (str | None): The prefix of submitted files. It + submission_prefix (str): The prefix of submitted files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. Returns: - tuple: (result_files, tmp_dir), result_files is a dict containing \ - the json filepaths, tmp_dir is the temporal directory created \ + tuple: (result_files, tmp_dir), result_files is a dict containing + the json filepaths, tmp_dir is the temporal directory created for saving json files when jsonfile_prefix is not specified. """ if pklfile_prefix is None: @@ -206,17 +206,18 @@ def evaluate(self, Args: results (list[dict]): Testing results of the dataset. - metric (str | list[str]): Metrics to be evaluated. - logger (logging.Logger | str | None): Logger used for printing + metric (str | list[str], optional): Metrics to be evaluated. + Defaults to None. + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. - pklfile_prefix (str | None): The prefix of pkl files. It includes + pklfile_prefix (str, optional): The prefix of pkl files, including the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. - submission_prefix (str | None): The prefix of submission datas. + submission_prefix (str, optional): The prefix of submission datas. If not specified, the submission data will not be generated. - show (bool): Whether to visualize. + show (bool, optional): Whether to visualize. Default: False. - out_dir (str): Path to save the visualization results. + out_dir (str, optional): Path to save the visualization results. Default: None. Returns: @@ -267,11 +268,11 @@ def bbox2result_kitti(self, submission. Args: - net_outputs (list[np.ndarray]): List of array storing the \ + net_outputs (list[np.ndarray]): List of array storing the inferenced bounding boxes and scores. class_names (list[String]): A list of class names. - pklfile_prefix (str | None): The prefix of pkl file. - submission_prefix (str | None): The prefix of submission file. + pklfile_prefix (str): The prefix of pkl file. + submission_prefix (str): The prefix of submission file. Returns: list[dict]: A list of dictionaries with the kitti format. @@ -382,11 +383,11 @@ def bbox2result_kitti2d(self, submission. Args: - net_outputs (list[np.ndarray]): List of array storing the \ + net_outputs (list[np.ndarray]): List of array storing the inferenced bounding boxes and scores. class_names (list[String]): A list of class names. - pklfile_prefix (str | None): The prefix of pkl file. - submission_prefix (str | None): The prefix of submission file. + pklfile_prefix (str): The prefix of pkl file. + submission_prefix (str): The prefix of submission file. Returns: list[dict]: A list of dictionaries have the kitti format @@ -497,7 +498,7 @@ def convert_valid_bboxes(self, box_dict, info): Returns: dict: Valid predicted boxes. - bbox (np.ndarray): 2D bounding boxes. - - box3d_camera (np.ndarray): 3D bounding boxes in \ + - box3d_camera (np.ndarray): 3D bounding boxes in camera coordinate. - scores (np.ndarray): Scores of boxes. - label_preds (np.ndarray): Class label predictions. diff --git a/mmdet3d/datasets/lyft_dataset.py b/mmdet3d/datasets/lyft_dataset.py index d701fc6a98..f56450e53b 100644 --- a/mmdet3d/datasets/lyft_dataset.py +++ b/mmdet3d/datasets/lyft_dataset.py @@ -128,7 +128,7 @@ def get_data_info(self, index): index (int): Index of the sample data to get. Returns: - dict: Data information that will be passed to the data \ + dict: Data information that will be passed to the data preprocessing pipelines. It includes the following keys: - sample_idx (str): sample index @@ -136,7 +136,7 @@ def get_data_info(self, index): - sweeps (list[dict]): infos of sweeps - timestamp (float): sample timestamp - img_filename (str, optional): image filename - - lidar2img (list[np.ndarray], optional): transformations \ + - lidar2img (list[np.ndarray], optional): transformations from lidar to different cameras - ann_info (dict): annotation info """ @@ -189,7 +189,7 @@ def get_ann_info(self, index): Returns: dict: Annotation information consists of the following keys: - - gt_bboxes_3d (:obj:`LiDARInstance3DBoxes`): \ + - gt_bboxes_3d (:obj:`LiDARInstance3DBoxes`): 3D ground truth bboxes. - gt_labels_3d (np.ndarray): Labels of ground truths. - gt_names (list[str]): Class names of ground truths. @@ -274,10 +274,11 @@ def _evaluate_single(self, Args: result_path (str): Path of the result file. - logger (logging.Logger | str | None): Logger used for printing + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. - metric (str): Metric name used for evaluation. Default: 'bbox'. - result_name (str): Result name in the metric prefix. + metric (str, optional): Metric name used for evaluation. + Default: 'bbox'. + result_name (str, optional): Result name in the metric prefix. Default: 'pts_bbox'. Returns: @@ -311,18 +312,18 @@ def format_results(self, results, jsonfile_prefix=None, csv_savepath=None): Args: results (list[dict]): Testing results of the dataset. - jsonfile_prefix (str | None): The prefix of json files. It includes + jsonfile_prefix (str): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. - csv_savepath (str | None): The path for saving csv files. + csv_savepath (str): The path for saving csv files. It includes the file path and the csv filename, e.g., "a/b/filename.csv". If not specified, the result will not be converted to csv file. Returns: - tuple: Returns (result_files, tmp_dir), where `result_files` is a \ - dict containing the json filepaths, `tmp_dir` is the temporal \ - directory created for saving json files when \ + tuple: Returns (result_files, tmp_dir), where `result_files` is a + dict containing the json filepaths, `tmp_dir` is the temporal + directory created for saving json files when `jsonfile_prefix` is not specified. """ assert isinstance(results, list), 'results must be a list' @@ -371,19 +372,22 @@ def evaluate(self, Args: results (list[dict]): Testing results of the dataset. - metric (str | list[str]): Metrics to be evaluated. - logger (logging.Logger | str | None): Logger used for printing + metric (str | list[str], optional): Metrics to be evaluated. + Default: 'bbox'. + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. - jsonfile_prefix (str | None): The prefix of json files. It includes + jsonfile_prefix (str, optional): The prefix of json files including the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. - csv_savepath (str | None): The path for saving csv files. + csv_savepath (str, optional): The path for saving csv files. It includes the file path and the csv filename, e.g., "a/b/filename.csv". If not specified, the result will not be converted to csv file. - show (bool): Whether to visualize. + result_names (list[str], optional): Result names in the + metric prefix. Default: ['pts_bbox']. + show (bool, optional): Whether to visualize. Default: False. - out_dir (str): Path to save the visualization results. + out_dir (str, optional): Path to save the visualization results. Default: None. pipeline (list[dict], optional): raw data loading for showing. Default: None. diff --git a/mmdet3d/datasets/nuscenes_dataset.py b/mmdet3d/datasets/nuscenes_dataset.py index 4d07a576f2..ac05a984fc 100644 --- a/mmdet3d/datasets/nuscenes_dataset.py +++ b/mmdet3d/datasets/nuscenes_dataset.py @@ -47,8 +47,9 @@ class NuScenesDataset(Custom3DDataset): Defaults to False. eval_version (bool, optional): Configuration version of evaluation. Defaults to 'detection_cvpr_2019'. - use_valid_flag (bool): Whether to use `use_valid_flag` key in the info - file as mask to filter gt_boxes and gt_names. Defaults to False. + use_valid_flag (bool, optional): Whether to use `use_valid_flag` key + in the info file as mask to filter gt_boxes and gt_names. + Defaults to False. """ NameMapping = { 'movable_object.barrier': 'barrier', @@ -195,7 +196,7 @@ def get_data_info(self, index): index (int): Index of the sample data to get. Returns: - dict: Data information that will be passed to the data \ + dict: Data information that will be passed to the data preprocessing pipelines. It includes the following keys: - sample_idx (str): Sample index. @@ -203,7 +204,7 @@ def get_data_info(self, index): - sweeps (list[dict]): Infos of sweeps. - timestamp (float): Sample timestamp. - img_filename (str, optional): Image filename. - - lidar2img (list[np.ndarray], optional): Transformations \ + - lidar2img (list[np.ndarray], optional): Transformations from lidar to different cameras. - ann_info (dict): Annotation info. """ @@ -255,7 +256,7 @@ def get_ann_info(self, index): Returns: dict: Annotation information consists of the following keys: - - gt_bboxes_3d (:obj:`LiDARInstance3DBoxes`): \ + - gt_bboxes_3d (:obj:`LiDARInstance3DBoxes`): 3D ground truth bboxes - gt_labels_3d (np.ndarray): Labels of ground truths. - gt_names (list[str]): Class names of ground truths. @@ -373,10 +374,11 @@ def _evaluate_single(self, Args: result_path (str): Path of the result file. - logger (logging.Logger | str | None): Logger used for printing + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. - metric (str): Metric name used for evaluation. Default: 'bbox'. - result_name (str): Result name in the metric prefix. + metric (str, optional): Metric name used for evaluation. + Default: 'bbox'. + result_name (str, optional): Result name in the metric prefix. Default: 'pts_bbox'. Returns: @@ -426,14 +428,14 @@ def format_results(self, results, jsonfile_prefix=None): Args: results (list[dict]): Testing results of the dataset. - jsonfile_prefix (str | None): The prefix of json files. It includes + jsonfile_prefix (str): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. Returns: - tuple: Returns (result_files, tmp_dir), where `result_files` is a \ - dict containing the json filepaths, `tmp_dir` is the temporal \ - directory created for saving json files when \ + tuple: Returns (result_files, tmp_dir), where `result_files` is a + dict containing the json filepaths, `tmp_dir` is the temporal + directory created for saving json files when `jsonfile_prefix` is not specified. """ assert isinstance(results, list), 'results must be a list' @@ -479,15 +481,16 @@ def evaluate(self, Args: results (list[dict]): Testing results of the dataset. - metric (str | list[str]): Metrics to be evaluated. - logger (logging.Logger | str | None): Logger used for printing + metric (str | list[str], optional): Metrics to be evaluated. + Default: 'bbox'. + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. - jsonfile_prefix (str | None): The prefix of json files. It includes + jsonfile_prefix (str, optional): The prefix of json files including the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. - show (bool): Whether to visualize. + show (bool, optional): Whether to visualize. Default: False. - out_dir (str): Path to save the visualization results. + out_dir (str, optional): Path to save the visualization results. Default: None. pipeline (list[dict], optional): raw data loading for showing. Default: None. @@ -623,7 +626,7 @@ def lidar_nusc_box_to_global(info, boxes (list[:obj:`NuScenesBox`]): List of predicted NuScenesBoxes. classes (list[str]): Mapped classes in the evaluation. eval_configs (object): Evaluation configuration object. - eval_version (str): Evaluation version. + eval_version (str, optional): Evaluation version. Default: 'detection_cvpr_2019' Returns: diff --git a/mmdet3d/datasets/nuscenes_mono_dataset.py b/mmdet3d/datasets/nuscenes_mono_dataset.py index f3f1b0b1aa..c39c68b648 100644 --- a/mmdet3d/datasets/nuscenes_mono_dataset.py +++ b/mmdet3d/datasets/nuscenes_mono_dataset.py @@ -43,8 +43,9 @@ class NuScenesMonoDataset(CocoDataset): - 'Camera': Box in camera coordinates. eval_version (str, optional): Configuration version of evaluation. Defaults to 'detection_cvpr_2019'. - use_valid_flag (bool): Whether to use `use_valid_flag` key in the info - file as mask to filter gt_boxes and gt_names. Defaults to False. + use_valid_flag (bool, optional): Whether to use `use_valid_flag` key + in the info file as mask to filter gt_boxes and gt_names. + Defaults to False. version (str, optional): Dataset version. Defaults to 'v1.0-trainval'. """ CLASSES = ('car', 'truck', 'trailer', 'bus', 'construction_vehicle', @@ -139,8 +140,8 @@ def _parse_ann_info(self, img_info, ann_info): ann_info (list[dict]): Annotation info of an image. Returns: - dict: A dict containing the following keys: bboxes, labels, \ - gt_bboxes_3d, gt_labels_3d, attr_labels, centers2d, \ + dict: A dict containing the following keys: bboxes, labels, + gt_bboxes_3d, gt_labels_3d, attr_labels, centers2d, depths, bboxes_ignore, masks, seg_map """ gt_bboxes = [] @@ -393,10 +394,11 @@ def _evaluate_single(self, Args: result_path (str): Path of the result file. - logger (logging.Logger | str | None): Logger used for printing + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. - metric (str): Metric name used for evaluation. Default: 'bbox'. - result_name (str): Result name in the metric prefix. + metric (str, optional): Metric name used for evaluation. + Default: 'bbox'. + result_name (str, optional): Result name in the metric prefix. Default: 'img_bbox'. Returns: @@ -447,13 +449,13 @@ def format_results(self, results, jsonfile_prefix=None, **kwargs): Args: results (list[tuple | numpy.ndarray]): Testing results of the dataset. - jsonfile_prefix (str | None): The prefix of json files. It includes + jsonfile_prefix (str): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. Returns: - tuple: (result_files, tmp_dir), result_files is a dict containing \ - the json filepaths, tmp_dir is the temporal directory created \ + tuple: (result_files, tmp_dir), result_files is a dict containing + the json filepaths, tmp_dir is the temporal directory created for saving json files when jsonfile_prefix is not specified. """ assert isinstance(results, list), 'results must be a list' @@ -503,15 +505,18 @@ def evaluate(self, Args: results (list[dict]): Testing results of the dataset. - metric (str | list[str]): Metrics to be evaluated. - logger (logging.Logger | str | None): Logger used for printing + metric (str | list[str], optional): Metrics to be evaluated. + Default: 'bbox'. + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. - jsonfile_prefix (str | None): The prefix of json files. It includes + jsonfile_prefix (str): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. - show (bool): Whether to visualize. + result_names (list[str], optional): Result names in the + metric prefix. Default: ['img_bbox']. + show (bool, optional): Whether to visualize. Default: False. - out_dir (str): Path to save the visualization results. + out_dir (str, optional): Path to save the visualization results. Default: None. pipeline (list[dict], optional): raw data loading for showing. Default: None. @@ -575,7 +580,7 @@ def _get_pipeline(self, pipeline): """Get data loading pipeline in self.show/evaluate function. Args: - pipeline (list[dict] | None): Input pipeline. If None is given, \ + pipeline (list[dict]): Input pipeline. If None is given, get from self.pipeline. """ if pipeline is None: @@ -695,7 +700,7 @@ def cam_nusc_box_to_global(info, boxes (list[:obj:`NuScenesBox`]): List of predicted NuScenesBoxes. classes (list[str]): Mapped classes in the evaluation. eval_configs (object): Evaluation configuration object. - eval_version (str): Evaluation version. + eval_version (str, optional): Evaluation version. Default: 'detection_cvpr_2019' Returns: @@ -735,7 +740,7 @@ def global_nusc_box_to_cam(info, boxes (list[:obj:`NuScenesBox`]): List of predicted NuScenesBoxes. classes (list[str]): Mapped classes in the evaluation. eval_configs (object): Evaluation configuration object. - eval_version (str): Evaluation version. + eval_version (str, optional): Evaluation version. Default: 'detection_cvpr_2019' Returns: @@ -768,7 +773,7 @@ def nusc_box_to_cam_box3d(boxes): boxes (list[:obj:`NuScenesBox`]): List of predicted NuScenesBoxes. Returns: - tuple (:obj:`CameraInstance3DBoxes` | torch.Tensor | torch.Tensor): \ + tuple (:obj:`CameraInstance3DBoxes` | torch.Tensor | torch.Tensor): Converted 3D bounding boxes, scores and labels. """ locs = torch.Tensor([b.center for b in boxes]).view(-1, 3) diff --git a/mmdet3d/datasets/pipelines/data_augment_utils.py b/mmdet3d/datasets/pipelines/data_augment_utils.py index 2cb3925048..269b5e40b2 100644 --- a/mmdet3d/datasets/pipelines/data_augment_utils.py +++ b/mmdet3d/datasets/pipelines/data_augment_utils.py @@ -33,8 +33,8 @@ def box_collision_test(boxes, qboxes, clockwise=True): Args: boxes (np.ndarray): Corners of current boxes. qboxes (np.ndarray): Boxes to be avoid colliding. - clockwise (bool): Whether the corners are in clockwise order. - Default: True. + clockwise (bool, optional): Whether the corners are in + clockwise order. Default: True. """ N = boxes.shape[0] K = qboxes.shape[0] @@ -316,7 +316,7 @@ def box3d_transform_(boxes, loc_transform, rot_transform, valid_mask): boxes (np.ndarray): 3D boxes to be transformed. loc_transform (np.ndarray): Location transform to be applied. rot_transform (np.ndarray): Rotation transform to be applied. - valid_mask (np.ndarray | None): Mask to indicate which boxes are valid. + valid_mask (np.ndarray): Mask to indicate which boxes are valid. """ num_box = boxes.shape[0] for i in range(num_box): @@ -337,16 +337,17 @@ def noise_per_object_v3_(gt_boxes, Args: gt_boxes (np.ndarray): Ground truth boxes with shape (N, 7). - points (np.ndarray | None): Input point cloud with shape (M, 4). - Default: None. - valid_mask (np.ndarray | None): Mask to indicate which boxes are valid. - Default: None. - rotation_perturb (float): Rotation perturbation. Default: pi / 4. - center_noise_std (float): Center noise standard deviation. + points (np.ndarray, optional): Input point cloud with + shape (M, 4). Default: None. + valid_mask (np.ndarray, optional): Mask to indicate which + boxes are valid. Default: None. + rotation_perturb (float, optional): Rotation perturbation. + Default: pi / 4. + center_noise_std (float, optional): Center noise standard deviation. Default: 1.0. - global_random_rot_range (float): Global random rotation range. - Default: pi/4. - num_try (int): Number of try. Default: 100. + global_random_rot_range (float, optional): Global random rotation + range. Default: pi/4. + num_try (int, optional): Number of try. Default: 100. """ num_boxes = gt_boxes.shape[0] if not isinstance(rotation_perturb, (list, tuple, np.ndarray)): diff --git a/mmdet3d/datasets/pipelines/dbsampler.py b/mmdet3d/datasets/pipelines/dbsampler.py index b5ccbbeb58..287da5ce26 100644 --- a/mmdet3d/datasets/pipelines/dbsampler.py +++ b/mmdet3d/datasets/pipelines/dbsampler.py @@ -14,10 +14,10 @@ class BatchSampler: Args: sample_list (list[dict]): List of samples. - name (str | None): The category of samples. Default: None. - epoch (int | None): Sampling epoch. Default: None. - shuffle (bool): Whether to shuffle indices. Default: False. - drop_reminder (bool): Drop reminder. Default: False. + name (str, optional): The category of samples. Default: None. + epoch (int, optional): Sampling epoch. Default: None. + shuffle (bool, optional): Whether to shuffle indices. Default: False. + drop_reminder (bool, optional): Drop reminder. Default: False. """ def __init__(self, @@ -86,9 +86,9 @@ class DataBaseSampler(object): rate (float): Rate of actual sampled over maximum sampled number. prepare (dict): Name of preparation functions and the input value. sample_groups (dict): Sampled classes and numbers. - classes (list[str]): List of classes. Default: None. - points_loader(dict): Config of points loader. Default: dict( - type='LoadPointsFromFile', load_dim=4, use_dim=[0,1,2,3]) + classes (list[str], optional): List of classes. Default: None. + points_loader(dict, optional): Config of points loader. Default: + dict(type='LoadPointsFromFile', load_dim=4, use_dim=[0,1,2,3]) """ def __init__(self, @@ -197,9 +197,9 @@ def sample_all(self, gt_bboxes, gt_labels, img=None): Returns: dict: Dict of sampled 'pseudo ground truths'. - - gt_labels_3d (np.ndarray): ground truths labels \ + - gt_labels_3d (np.ndarray): ground truths labels of sampled objects. - - gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): \ + - gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): sampled ground truth 3D bounding boxes - points (np.ndarray): sampled points - group_ids (np.ndarray): ids of sampled ground truths diff --git a/mmdet3d/datasets/pipelines/formating.py b/mmdet3d/datasets/pipelines/formating.py index e7eb7e9d2d..06ec48fb30 100644 --- a/mmdet3d/datasets/pipelines/formating.py +++ b/mmdet3d/datasets/pipelines/formating.py @@ -23,7 +23,7 @@ class DefaultFormatBundle(object): - gt_bboxes_ignore: (1)to tensor, (2)to DataContainer - gt_labels: (1)to tensor, (2)to DataContainer - gt_masks: (1)to tensor, (2)to DataContainer (cpu_only=True) - - gt_semantic_seg: (1)unsqueeze dim-0 (2)to tensor, \ + - gt_semantic_seg: (1)unsqueeze dim-0 (2)to tensor, (3)to DataContainer (stack=True) """ @@ -91,8 +91,8 @@ class Collect3D(object): The "img_meta" item is always populated. The contents of the "img_meta" dictionary depends on "meta_keys". By default this includes: - - 'img_shape': shape of the image input to the network as a tuple \ - (h, w, c). Note that images may be zero padded on the \ + - 'img_shape': shape of the image input to the network as a tuple + (h, w, c). Note that images may be zero padded on the bottom/right if the batch tensor is larger than this shape. - 'scale_factor': a float indicating the preprocessing scale - 'flip': a boolean indicating if image flip transform was used @@ -102,9 +102,9 @@ class Collect3D(object): - 'lidar2img': transform from lidar to image - 'depth2img': transform from depth to image - 'cam2img': transform from camera to image - - 'pcd_horizontal_flip': a boolean indicating if point cloud is \ + - 'pcd_horizontal_flip': a boolean indicating if point cloud is flipped horizontally - - 'pcd_vertical_flip': a boolean indicating if point cloud is \ + - 'pcd_vertical_flip': a boolean indicating if point cloud is flipped vertically - 'box_mode_3d': 3D box mode - 'box_type_3d': 3D box type @@ -129,16 +129,15 @@ class Collect3D(object): 'sample_idx', 'pcd_scale_factor', 'pcd_rotation', 'pts_filename') """ - def __init__(self, - keys, - meta_keys=('filename', 'ori_shape', 'img_shape', 'lidar2img', - 'depth2img', 'cam2img', 'pad_shape', - 'scale_factor', 'flip', 'pcd_horizontal_flip', - 'pcd_vertical_flip', 'box_mode_3d', 'box_type_3d', - 'img_norm_cfg', 'pcd_trans', 'sample_idx', - 'pcd_scale_factor', 'pcd_rotation', - 'pcd_rotation_angle', 'pts_filename', - 'transformation_3d_flow')): + def __init__( + self, + keys, + meta_keys=('filename', 'ori_shape', 'img_shape', 'lidar2img', + 'depth2img', 'cam2img', 'pad_shape', 'scale_factor', 'flip', + 'pcd_horizontal_flip', 'pcd_vertical_flip', 'box_mode_3d', + 'box_type_3d', 'img_norm_cfg', 'pcd_trans', 'sample_idx', + 'pcd_scale_factor', 'pcd_rotation', 'pcd_rotation_angle', + 'pts_filename', 'transformation_3d_flow')): self.keys = keys self.meta_keys = meta_keys diff --git a/mmdet3d/datasets/pipelines/loading.py b/mmdet3d/datasets/pipelines/loading.py index d78b30599f..21c9a6d226 100644 --- a/mmdet3d/datasets/pipelines/loading.py +++ b/mmdet3d/datasets/pipelines/loading.py @@ -13,9 +13,10 @@ class LoadMultiViewImageFromFiles(object): Expects results['img_filename'] to be a list of filenames. Args: - to_float32 (bool): Whether to convert the img to float32. + to_float32 (bool, optional): Whether to convert the img to float32. Defaults to False. - color_type (str): Color type of the file. Defaults to 'unchanged'. + color_type (str, optional): Color type of the file. + Defaults to 'unchanged'. """ def __init__(self, to_float32=False, color_type='unchanged'): @@ -29,7 +30,7 @@ def __call__(self, results): results (dict): Result dict containing multi-view image filenames. Returns: - dict: The result dict containing the multi-view image data. \ + dict: The result dict containing the multi-view image data. Added keys and values are described below. - filename (str): Multi-view image filenames. @@ -76,7 +77,7 @@ class LoadImageFromFileMono3D(LoadImageFromFile): detection, additional camera parameters need to be loaded. Args: - kwargs (dict): Arguments are the same as those in \ + kwargs (dict): Arguments are the same as those in :class:`LoadImageFromFile`. """ @@ -101,17 +102,20 @@ class LoadPointsFromMultiSweeps(object): This is usually used for nuScenes dataset to utilize previous sweeps. Args: - sweeps_num (int): Number of sweeps. Defaults to 10. - load_dim (int): Dimension number of the loaded points. Defaults to 5. - use_dim (list[int]): Which dimension to use. Defaults to [0, 1, 2, 4]. - file_client_args (dict): Config dict of file clients, refer to + sweeps_num (int, optional): Number of sweeps. Defaults to 10. + load_dim (int, optional): Dimension number of the loaded points. + Defaults to 5. + use_dim (list[int], optional): Which dimension to use. + Defaults to [0, 1, 2, 4]. + file_client_args (dict, optional): Config dict of file clients, + refer to https://github.com/open-mmlab/mmcv/blob/master/mmcv/fileio/file_client.py for more details. Defaults to dict(backend='disk'). - pad_empty_sweeps (bool): Whether to repeat keyframe when + pad_empty_sweeps (bool, optional): Whether to repeat keyframe when sweeps is empty. Defaults to False. - remove_close (bool): Whether to remove close points. + remove_close (bool, optional): Whether to remove close points. Defaults to False. - test_mode (bool): If test_model=True used for testing, it will not + test_mode (bool, optional): If `test_mode=True`, it will not randomly sample sweeps but select the nearest N frames. Defaults to False. """ @@ -160,7 +164,7 @@ def _remove_close(self, points, radius=1.0): Args: points (np.ndarray | :obj:`BasePoints`): Sweep points. - radius (float): Radius below which points are removed. + radius (float, optional): Radius below which points are removed. Defaults to 1.0. Returns: @@ -181,14 +185,14 @@ def __call__(self, results): """Call function to load multi-sweep point clouds from files. Args: - results (dict): Result dict containing multi-sweep point cloud \ + results (dict): Result dict containing multi-sweep point cloud filenames. Returns: - dict: The result dict containing the multi-sweep points data. \ + dict: The result dict containing the multi-sweep points data. Added key and value are described below. - - points (np.ndarray | :obj:`BasePoints`): Multi-sweep point \ + - points (np.ndarray | :obj:`BasePoints`): Multi-sweep point cloud arrays. """ points = results['points'] @@ -242,8 +246,8 @@ class PointSegClassMapping(object): Args: valid_cat_ids (tuple[int]): A tuple of valid category. - max_cat_id (int): The max possible cat_id in input segmentation mask. - Defaults to 40. + max_cat_id (int, optional): The max possible cat_id in input + segmentation mask. Defaults to 40. """ def __init__(self, valid_cat_ids, max_cat_id=40): @@ -267,7 +271,7 @@ def __call__(self, results): results (dict): Result dict containing point semantic masks. Returns: - dict: The result dict containing the mapped category ids. \ + dict: The result dict containing the mapped category ids. Updated key and value are described below. - pts_semantic_mask (np.ndarray): Mapped semantic masks. @@ -306,7 +310,7 @@ def __call__(self, results): results (dict): Result dict containing point clouds data. Returns: - dict: The result dict containing the normalized points. \ + dict: The result dict containing the normalized points. Updated key and value are described below. - points (:obj:`BasePoints`): Points after color normalization. @@ -341,14 +345,17 @@ class LoadPointsFromFile(object): - 'LIDAR': Points in LiDAR coordinates. - 'DEPTH': Points in depth coordinates, usually for indoor dataset. - 'CAMERA': Points in camera coordinates. - load_dim (int): The dimension of the loaded points. + load_dim (int, optional): The dimension of the loaded points. Defaults to 6. - use_dim (list[int]): Which dimensions of the points to be used. + use_dim (list[int], optional): Which dimensions of the points to use. Defaults to [0, 1, 2]. For KITTI dataset, set use_dim=4 or use_dim=[0, 1, 2, 3] to use the intensity dimension. - shift_height (bool): Whether to use shifted height. Defaults to False. - use_color (bool): Whether to use color features. Defaults to False. - file_client_args (dict): Config dict of file clients, refer to + shift_height (bool, optional): Whether to use shifted height. + Defaults to False. + use_color (bool, optional): Whether to use color features. + Defaults to False. + file_client_args (dict, optional): Config dict of file clients, + refer to https://github.com/open-mmlab/mmcv/blob/master/mmcv/fileio/file_client.py for more details. Defaults to dict(backend='disk'). """ @@ -404,7 +411,7 @@ def __call__(self, results): results (dict): Result dict containing point clouds data. Returns: - dict: The result dict containing the point clouds data. \ + dict: The result dict containing the point clouds data. Added key and value are described below. - points (:obj:`BasePoints`): Point clouds data. diff --git a/mmdet3d/datasets/pipelines/test_time_aug.py b/mmdet3d/datasets/pipelines/test_time_aug.py index f0de62b284..9965ac23e4 100644 --- a/mmdet3d/datasets/pipelines/test_time_aug.py +++ b/mmdet3d/datasets/pipelines/test_time_aug.py @@ -15,18 +15,19 @@ class MultiScaleFlipAug3D(object): img_scale (tuple | list[tuple]: Images scales for resizing. pts_scale_ratio (float | list[float]): Points scale ratios for resizing. - flip (bool): Whether apply flip augmentation. Defaults to False. - flip_direction (str | list[str]): Flip augmentation directions - for images, options are "horizontal" and "vertical". + flip (bool, optional): Whether apply flip augmentation. + Defaults to False. + flip_direction (str | list[str], optional): Flip augmentation + directions for images, options are "horizontal" and "vertical". If flip_direction is list, multiple flip augmentations will be applied. It has no effect when ``flip == False``. Defaults to "horizontal". - pcd_horizontal_flip (bool): Whether apply horizontal flip augmentation - to point cloud. Defaults to True. Note that it works only when - 'flip' is turned on. - pcd_vertical_flip (bool): Whether apply vertical flip augmentation - to point cloud. Defaults to True. Note that it works only when - 'flip' is turned on. + pcd_horizontal_flip (bool, optional): Whether apply horizontal + flip augmentation to point cloud. Defaults to True. + Note that it works only when 'flip' is turned on. + pcd_vertical_flip (bool, optional): Whether apply vertical flip + augmentation to point cloud. Defaults to True. + Note that it works only when 'flip' is turned on. """ def __init__(self, @@ -69,7 +70,7 @@ def __call__(self, results): results (dict): Result dict contains the data to augment. Returns: - dict: The result dict contains the data that is augmented with \ + dict: The result dict contains the data that is augmented with different scales and flips. """ aug_data = [] diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index bb82cd3351..88706210d2 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -21,7 +21,7 @@ class RandomDropPointsColor(object): util/transform.py#L223>`_ for more details. Args: - drop_ratio (float): The probability of dropping point colors. + drop_ratio (float, optional): The probability of dropping point colors. Defaults to 0.2. """ @@ -37,7 +37,7 @@ def __call__(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after color dropping, \ + dict: Results after color dropping, 'points' key is updated in the result dict. """ points = input_dict['points'] @@ -104,10 +104,11 @@ def random_flip_data_3d(self, input_dict, direction='horizontal'): Args: input_dict (dict): Result dict from loading pipeline. - direction (str): Flip direction. Default: horizontal. + direction (str, optional): Flip direction. + Default: 'horizontal'. Returns: - dict: Flipped results, 'points', 'bbox3d_fields' keys are \ + dict: Flipped results, 'points', 'bbox3d_fields' keys are updated in the result dict. """ assert direction in ['horizontal', 'vertical'] @@ -136,15 +137,15 @@ def random_flip_data_3d(self, input_dict, direction='horizontal'): input_dict['cam2img'][0][2] = w - input_dict['cam2img'][0][2] def __call__(self, input_dict): - """Call function to flip points, values in the ``bbox3d_fields`` and \ + """Call function to flip points, values in the ``bbox3d_fields`` and also flip 2D image and its annotations. Args: input_dict (dict): Result dict from loading pipeline. Returns: - dict: Flipped results, 'flip', 'flip_direction', \ - 'pcd_horizontal_flip' and 'pcd_vertical_flip' keys are added \ + dict: Flipped results, 'flip', 'flip_direction', + 'pcd_horizontal_flip' and 'pcd_vertical_flip' keys are added into result dict. """ # filp 2D image and its annotations @@ -186,20 +187,20 @@ def __repr__(self): class RandomJitterPoints(object): """Randomly jitter point coordinates. - Different from the global translation in ``GlobalRotScaleTrans``, here we \ + Different from the global translation in ``GlobalRotScaleTrans``, here we apply different noises to each point in a scene. Args: jitter_std (list[float]): The standard deviation of jittering noise. - This applies random noise to all points in a 3D scene, which is \ - sampled from a gaussian distribution whose standard deviation is \ + This applies random noise to all points in a 3D scene, which is + sampled from a gaussian distribution whose standard deviation is set by ``jitter_std``. Defaults to [0.01, 0.01, 0.01] - clip_range (list[float] | None): Clip the randomly generated jitter \ + clip_range (list[float]): Clip the randomly generated jitter noise into this range. If None is given, don't perform clipping. Defaults to [-0.05, 0.05] Note: - This transform should only be used in point cloud segmentation tasks \ + This transform should only be used in point cloud segmentation tasks because we don't transform ground-truth bboxes accordingly. For similar transform in detection task, please refer to `ObjectNoise`. """ @@ -228,7 +229,7 @@ def __call__(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after adding noise to each point, \ + dict: Results after adding noise to each point, 'points' key is updated in the result dict. """ points = input_dict['points'] @@ -290,8 +291,8 @@ def __call__(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after object sampling augmentation, \ - 'points', 'gt_bboxes_3d', 'gt_labels_3d' keys are updated \ + dict: Results after object sampling augmentation, + 'points', 'gt_bboxes_3d', 'gt_labels_3d' keys are updated in the result dict. """ gt_bboxes_3d = input_dict['gt_bboxes_3d'] @@ -387,7 +388,7 @@ def __call__(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after adding noise to each object, \ + dict: Results after adding noise to each object, 'points', 'gt_bboxes_3d' keys are updated in the result dict. """ gt_bboxes_3d = input_dict['gt_bboxes_3d'] @@ -427,10 +428,10 @@ class GlobalAlignment(object): rotation_axis (int): Rotation axis for points and bboxes rotation. Note: - We do not record the applied rotation and translation as in \ - GlobalRotScaleTrans. Because usually, we do not need to reverse \ + We do not record the applied rotation and translation as in + GlobalRotScaleTrans. Because usually, we do not need to reverse the alignment step. - For example, ScanNet 3D detection task uses aligned ground-truth \ + For example, ScanNet 3D detection task uses aligned ground-truth bounding boxes for evaluation. """ @@ -482,7 +483,7 @@ def __call__(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after global alignment, 'points' and keys in \ + dict: Results after global alignment, 'points' and keys in input_dict['bbox3d_fields'] are updated in the result dict. """ assert 'axis_align_matrix' in input_dict['ann_info'].keys(), \ @@ -511,15 +512,15 @@ class GlobalRotScaleTrans(object): """Apply global rotation, scaling and translation to a 3D scene. Args: - rot_range (list[float]): Range of rotation angle. + rot_range (list[float], optional): Range of rotation angle. Defaults to [-0.78539816, 0.78539816] (close to [-pi/4, pi/4]). - scale_ratio_range (list[float]): Range of scale ratio. + scale_ratio_range (list[float], optional): Range of scale ratio. Defaults to [0.95, 1.05]. - translation_std (list[float]): The standard deviation of translation - noise. This applies random translation to a scene by a noise, which + translation_std (list[float], optional): The standard deviation of + translation noise applied to a scene, which is sampled from a gaussian distribution whose standard deviation is set by ``translation_std``. Defaults to [0, 0, 0] - shift_height (bool): Whether to shift height. + shift_height (bool, optional): Whether to shift height. (the fourth dimension of indoor points) when scaling. Defaults to False. """ @@ -558,8 +559,8 @@ def _trans_bbox_points(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after translation, 'points', 'pcd_trans' \ - and keys in input_dict['bbox3d_fields'] are updated \ + dict: Results after translation, 'points', 'pcd_trans' + and keys in input_dict['bbox3d_fields'] are updated in the result dict. """ translation_std = np.array(self.translation_std, dtype=np.float32) @@ -577,8 +578,8 @@ def _rot_bbox_points(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after rotation, 'points', 'pcd_rotation' \ - and keys in input_dict['bbox3d_fields'] are updated \ + dict: Results after rotation, 'points', 'pcd_rotation' + and keys in input_dict['bbox3d_fields'] are updated in the result dict. """ rotation = self.rot_range @@ -607,7 +608,7 @@ def _scale_bbox_points(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after scaling, 'points'and keys in \ + dict: Results after scaling, 'points'and keys in input_dict['bbox3d_fields'] are updated in the result dict. """ scale = input_dict['pcd_scale_factor'] @@ -629,7 +630,7 @@ def _random_scale(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after scaling, 'pcd_scale_factor' are updated \ + dict: Results after scaling, 'pcd_scale_factor' are updated in the result dict. """ scale_factor = np.random.uniform(self.scale_ratio_range[0], @@ -637,7 +638,7 @@ def _random_scale(self, input_dict): input_dict['pcd_scale_factor'] = scale_factor def __call__(self, input_dict): - """Private function to rotate, scale and translate bounding boxes and \ + """Private function to rotate, scale and translate bounding boxes and points. Args: @@ -645,7 +646,7 @@ def __call__(self, input_dict): Returns: dict: Results after scaling, 'points', 'pcd_rotation', - 'pcd_scale_factor', 'pcd_trans' and keys in \ + 'pcd_scale_factor', 'pcd_trans' and keys in input_dict['bbox3d_fields'] are updated in the result dict. """ if 'transformation_3d_flow' not in input_dict: @@ -683,7 +684,7 @@ def __call__(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after filtering, 'points', 'pts_instance_mask' \ + dict: Results after filtering, 'points', 'pts_instance_mask' and 'pts_semantic_mask' keys are updated in the result dict. """ idx = input_dict['points'].shuffle() @@ -722,7 +723,7 @@ def __call__(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after filtering, 'gt_bboxes_3d', 'gt_labels_3d' \ + dict: Results after filtering, 'gt_bboxes_3d', 'gt_labels_3d' keys are updated in the result dict. """ # Check points instance type and initialise bev_range @@ -774,7 +775,7 @@ def __call__(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after filtering, 'points', 'pts_instance_mask' \ + dict: Results after filtering, 'points', 'pts_instance_mask' and 'pts_semantic_mask' keys are updated in the result dict. """ points = input_dict['points'] @@ -820,7 +821,7 @@ def __call__(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after filtering, 'gt_bboxes_3d', 'gt_labels_3d' \ + dict: Results after filtering, 'gt_bboxes_3d', 'gt_labels_3d' keys are updated in the result dict. """ gt_labels_3d = input_dict['gt_labels_3d'] @@ -912,7 +913,7 @@ def __call__(self, results): Args: input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after sampling, 'points', 'pts_instance_mask' \ + dict: Results after sampling, 'points', 'pts_instance_mask' and 'pts_semantic_mask' keys are updated in the result dict. """ points = results['points'] @@ -993,10 +994,10 @@ class IndoorPatchPointSample(object): additional features. Defaults to False. num_try (int, optional): Number of times to try if the patch selected is invalid. Defaults to 10. - enlarge_size (float | None, optional): Enlarge the sampled patch to + enlarge_size (float, optional): Enlarge the sampled patch to [-block_size / 2 - enlarge_size, block_size / 2 + enlarge_size] as an augmentation. If None, set it as 0. Defaults to 0.2. - min_unique_num (int | None, optional): Minimum number of unique points + min_unique_num (int, optional): Minimum number of unique points the sampled patch should contain. If None, use PointNet++'s method to judge uniqueness. Defaults to None. eps (float, optional): A value added to patch boundary to guarantee @@ -1037,7 +1038,7 @@ def _input_generation(self, coords, patch_center, coord_max, attributes, attribute_dims, point_type): """Generating model input. - Generate input by subtracting patch center and adding additional \ + Generate input by subtracting patch center and adding additional features. Currently support colors and normalized xyz as features. Args: @@ -1181,7 +1182,7 @@ def __call__(self, results): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after sampling, 'points', 'pts_instance_mask' \ + dict: Results after sampling, 'points', 'pts_instance_mask' and 'pts_semantic_mask' keys are updated in the result dict. """ points = results['points'] @@ -1241,7 +1242,7 @@ def __call__(self, input_dict): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after filtering, 'points', 'pts_instance_mask' \ + dict: Results after filtering, 'points', 'pts_instance_mask' and 'pts_semantic_mask' keys are updated in the result dict. """ points = input_dict['points'] @@ -1339,7 +1340,7 @@ def __call__(self, results): input_dict (dict): Result dict from loading pipeline. Returns: - dict: Results after sampling, 'points', 'pts_instance_mask' \ + dict: Results after sampling, 'points', 'pts_instance_mask' and 'pts_semantic_mask' keys are updated in the result dict. """ points = results['points'] diff --git a/mmdet3d/datasets/scannet_dataset.py b/mmdet3d/datasets/scannet_dataset.py index 8a109d835e..6854f72f7e 100644 --- a/mmdet3d/datasets/scannet_dataset.py +++ b/mmdet3d/datasets/scannet_dataset.py @@ -77,13 +77,13 @@ def get_data_info(self, index): index (int): Index of the sample data to get. Returns: - dict: Data information that will be passed to the data \ + dict: Data information that will be passed to the data preprocessing pipelines. It includes the following keys: - sample_idx (str): Sample index. - pts_filename (str): Filename of point clouds. - file_name (str): Filename of point clouds. - - img_prefix (str | None, optional): Prefix of image files. + - img_prefix (str, optional): Prefix of image files. - img_info (dict, optional): Image info. - ann_info (dict): Annotation info. """ @@ -128,12 +128,12 @@ def get_ann_info(self, index): Returns: dict: annotation information consists of the following keys: - - gt_bboxes_3d (:obj:`DepthInstance3DBoxes`): \ + - gt_bboxes_3d (:obj:`DepthInstance3DBoxes`): 3D ground truth bboxes - gt_labels_3d (np.ndarray): Labels of ground truths. - pts_instance_mask_path (str): Path of instance masks. - pts_semantic_mask_path (str): Path of semantic masks. - - axis_align_matrix (np.ndarray): Transformation matrix for \ + - axis_align_matrix (np.ndarray): Transformation matrix for global scene alignment. """ # Use index to get the annos, thus the evalhook could also use this api @@ -171,7 +171,7 @@ def get_ann_info(self, index): def prepare_test_data(self, index): """Prepare data for testing. - We should take axis_align_matrix from self.data_infos since we need \ + We should take axis_align_matrix from self.data_infos since we need to align point clouds. Args: @@ -271,7 +271,7 @@ class ScanNetSegDataset(Custom3DSegDataset): as input. Defaults to None. test_mode (bool, optional): Whether the dataset is in test mode. Defaults to False. - ignore_index (int, optional): The label index to be ignored, e.g. \ + ignore_index (int, optional): The label index to be ignored, e.g. unannotated points. If None is given, set to len(self.CLASSES). Defaults to None. scene_idxs (np.ndarray | str, optional): Precomputed index to load @@ -423,7 +423,7 @@ def format_results(self, results, txtfile_prefix=None): Args: outputs (list[dict]): Testing results of the dataset. - txtfile_prefix (str | None): The prefix of saved files. It includes + txtfile_prefix (str): The prefix of saved files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. diff --git a/mmdet3d/datasets/sunrgbd_dataset.py b/mmdet3d/datasets/sunrgbd_dataset.py index ce1f0c0304..548a9683cc 100644 --- a/mmdet3d/datasets/sunrgbd_dataset.py +++ b/mmdet3d/datasets/sunrgbd_dataset.py @@ -73,13 +73,13 @@ def get_data_info(self, index): index (int): Index of the sample data to get. Returns: - dict: Data information that will be passed to the data \ + dict: Data information that will be passed to the data preprocessing pipelines. It includes the following keys: - sample_idx (str): Sample index. - pts_filename (str, optional): Filename of point clouds. - file_name (str, optional): Filename of point clouds. - - img_prefix (str | None, optional): Prefix of image files. + - img_prefix (str, optional): Prefix of image files. - img_info (dict, optional): Image info. - calib (dict, optional): Camera calibration info. - ann_info (dict): Annotation info. @@ -124,7 +124,7 @@ def get_ann_info(self, index): Returns: dict: annotation information consists of the following keys: - - gt_bboxes_3d (:obj:`DepthInstance3DBoxes`): \ + - gt_bboxes_3d (:obj:`DepthInstance3DBoxes`): 3D ground truth bboxes - gt_labels_3d (np.ndarray): Labels of ground truths. - pts_instance_mask_path (str): Path of instance masks. @@ -238,12 +238,15 @@ def evaluate(self, Args: results (list[dict]): List of results. - metric (str | list[str]): Metrics to be evaluated. - iou_thr (list[float]): AP IoU thresholds. - iou_thr_2d (list[float]): AP IoU thresholds for 2d evaluation. - show (bool): Whether to visualize. + metric (str | list[str], optional): Metrics to be evaluated. + Default: None. + iou_thr (list[float], optional): AP IoU thresholds for 3D + evaluation. Default: (0.25, 0.5). + iou_thr_2d (list[float], optional): AP IoU thresholds for 2D + evaluation. Default: (0.5, ). + show (bool, optional): Whether to visualize. Default: False. - out_dir (str): Path to save the visualization results. + out_dir (str, optional): Path to save the visualization results. Default: None. pipeline (list[dict], optional): raw data loading for showing. Default: None. diff --git a/mmdet3d/datasets/utils.py b/mmdet3d/datasets/utils.py index 893529dae9..3264b725a9 100644 --- a/mmdet3d/datasets/utils.py +++ b/mmdet3d/datasets/utils.py @@ -24,7 +24,7 @@ def is_loading_function(transform): transform (dict | :obj:`Pipeline`): A transform config or a function. Returns: - bool | None: Whether it is a loading function. None means can't judge. + bool: Whether it is a loading function. None means can't judge. When transform is `MultiScaleFlipAug3D`, we return None. """ # TODO: use more elegant way to distinguish loading modules @@ -91,7 +91,7 @@ def get_loading_pipeline(pipeline): ... dict(type='Collect3D', ... keys=['points', 'img', 'gt_bboxes_3d', 'gt_labels_3d']) ... ] - >>> assert expected_pipelines ==\ + >>> assert expected_pipelines == \ ... get_loading_pipeline(pipelines) """ loading_pipeline = [] @@ -125,7 +125,7 @@ def extract_result_dict(results, key): key (str): Key of the desired data. Returns: - np.ndarray | torch.Tensor | None: Data term. + np.ndarray | torch.Tensor: Data term. """ if key not in results.keys(): return None diff --git a/mmdet3d/datasets/waymo_dataset.py b/mmdet3d/datasets/waymo_dataset.py index 27eb3af720..842aad5c9f 100644 --- a/mmdet3d/datasets/waymo_dataset.py +++ b/mmdet3d/datasets/waymo_dataset.py @@ -45,8 +45,9 @@ class WaymoDataset(KittiDataset): Defaults to True. test_mode (bool, optional): Whether the dataset is in test mode. Defaults to False. - pcd_limit_range (list): The range of point cloud used to filter - invalid predicted boxes. Default: [-85, -85, -5, 85, 85, 5]. + pcd_limit_range (list(float), optional): The range of point cloud used + to filter invalid predicted boxes. + Default: [-85, -85, -5, 85, 85, 5]. """ CLASSES = ('Car', 'Cyclist', 'Pedestrian') @@ -99,7 +100,7 @@ def get_data_info(self, index): - sample_idx (str): sample index - pts_filename (str): filename of point clouds - - img_prefix (str | None): prefix of image files + - img_prefix (str): prefix of image files - img_info (dict): image info - lidar2img (list[np.ndarray], optional): transformations from lidar to different cameras @@ -139,15 +140,15 @@ def format_results(self, Args: outputs (list[dict]): Testing results of the dataset. - pklfile_prefix (str | None): The prefix of pkl files. It includes + pklfile_prefix (str): The prefix of pkl files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. - submission_prefix (str | None): The prefix of submitted files. It + submission_prefix (str): The prefix of submitted files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. - data_format (str | None): Output data format. Default: 'waymo'. - Another supported choice is 'kitti'. + data_format (str, optional): Output data format. + Default: 'waymo'. Another supported choice is 'kitti'. Returns: tuple: (result_files, tmp_dir), result_files is a dict containing @@ -225,18 +226,18 @@ def evaluate(self, Args: results (list[dict]): Testing results of the dataset. - metric (str | list[str]): Metrics to be evaluated. + metric (str | list[str], optional): Metrics to be evaluated. Default: 'waymo'. Another supported metric is 'kitti'. - logger (logging.Logger | str | None): Logger used for printing + logger (logging.Logger | str, optional): Logger used for printing related information during evaluation. Default: None. - pklfile_prefix (str | None): The prefix of pkl files. It includes + pklfile_prefix (str, optional): The prefix of pkl files including the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Default: None. - submission_prefix (str | None): The prefix of submission datas. + submission_prefix (str, optional): The prefix of submission datas. If not specified, the submission data will not be generated. - show (bool): Whether to visualize. + show (bool, optional): Whether to visualize. Default: False. - out_dir (str): Path to save the visualization results. + out_dir (str, optional): Path to save the visualization results. Default: None. pipeline (list[dict], optional): raw data loading for showing. Default: None. @@ -363,8 +364,8 @@ def bbox2result_kitti(self, net_outputs (List[np.ndarray]): list of array storing the bbox and score class_nanes (List[String]): A list of class names - pklfile_prefix (str | None): The prefix of pkl file. - submission_prefix (str | None): The prefix of submission file. + pklfile_prefix (str): The prefix of pkl file. + submission_prefix (str): The prefix of submission file. Returns: List[dict]: A list of dict have the kitti 3d format diff --git a/mmdet3d/models/backbones/pointnet2_sa_msg.py b/mmdet3d/models/backbones/pointnet2_sa_msg.py index 8e1f083ab8..c641c5aaca 100644 --- a/mmdet3d/models/backbones/pointnet2_sa_msg.py +++ b/mmdet3d/models/backbones/pointnet2_sa_msg.py @@ -133,7 +133,7 @@ def forward(self, points): - sa_xyz (torch.Tensor): The coordinates of sa features. - sa_features (torch.Tensor): The features from the last Set Aggregation Layers. - - sa_indices (torch.Tensor): Indices of the \ + - sa_indices (torch.Tensor): Indices of the input points. """ xyz, features = self._split_point_feats(points) diff --git a/mmdet3d/models/backbones/pointnet2_sa_ssg.py b/mmdet3d/models/backbones/pointnet2_sa_ssg.py index fea5e3d403..7cacf4a9df 100644 --- a/mmdet3d/models/backbones/pointnet2_sa_ssg.py +++ b/mmdet3d/models/backbones/pointnet2_sa_ssg.py @@ -96,11 +96,11 @@ def forward(self, points): Returns: dict[str, list[torch.Tensor]]: Outputs after SA and FP modules. - - fp_xyz (list[torch.Tensor]): The coordinates of \ + - fp_xyz (list[torch.Tensor]): The coordinates of each fp features. - - fp_features (list[torch.Tensor]): The features \ + - fp_features (list[torch.Tensor]): The features from each Feature Propagate Layers. - - fp_indices (list[torch.Tensor]): Indices of the \ + - fp_indices (list[torch.Tensor]): Indices of the input points. """ xyz, features = self._split_point_feats(points) diff --git a/mmdet3d/models/decode_heads/decode_head.py b/mmdet3d/models/decode_heads/decode_head.py index f339f574c3..d3b4a695e5 100644 --- a/mmdet3d/models/decode_heads/decode_head.py +++ b/mmdet3d/models/decode_heads/decode_head.py @@ -12,17 +12,18 @@ class Base3DDecodeHead(BaseModule, metaclass=ABCMeta): Args: channels (int): Channels after modules, before conv_seg. num_classes (int): Number of classes. - dropout_ratio (float): Ratio of dropout layer. Default: 0.5. - conv_cfg (dict|None): Config of conv layers. + dropout_ratio (float, optional): Ratio of dropout layer. Default: 0.5. + conv_cfg (dict, optional): Config of conv layers. Default: dict(type='Conv1d'). - norm_cfg (dict|None): Config of norm layers. + norm_cfg (dict, optional): Config of norm layers. Default: dict(type='BN1d'). - act_cfg (dict): Config of activation layers. + act_cfg (dict, optional): Config of activation layers. Default: dict(type='ReLU'). - loss_decode (dict): Config of decode loss. + loss_decode (dict, optional): Config of decode loss. Default: dict(type='CrossEntropyLoss'). - ignore_index (int | None): The label index to be ignored. When using - masked BCE loss, ignore_index should be set to None. Default: 255. + ignore_index (int, optional): The label index to be ignored. + When using masked BCE loss, ignore_index should be set to None. + Default: 255. """ def __init__(self, @@ -109,9 +110,9 @@ def losses(self, seg_logit, seg_label): """Compute semantic segmentation loss. Args: - seg_logit (torch.Tensor): Predicted per-point segmentation logits \ + seg_logit (torch.Tensor): Predicted per-point segmentation logits of shape [B, num_classes, N]. - seg_label (torch.Tensor): Ground-truth segmentation label of \ + seg_label (torch.Tensor): Ground-truth segmentation label of shape [B, N]. """ loss = dict() diff --git a/mmdet3d/models/decode_heads/paconv_head.py b/mmdet3d/models/decode_heads/paconv_head.py index cf24ba510a..62f1b9fcca 100644 --- a/mmdet3d/models/decode_heads/paconv_head.py +++ b/mmdet3d/models/decode_heads/paconv_head.py @@ -13,7 +13,7 @@ class PAConvHead(PointNet2Head): Args: fp_channels (tuple[tuple[int]]): Tuple of mlp channels in FP modules. - fp_norm_cfg (dict|None): Config of norm layers used in FP modules. + fp_norm_cfg (dict): Config of norm layers used in FP modules. Default: dict(type='BN2d'). """ diff --git a/mmdet3d/models/decode_heads/pointnet2_head.py b/mmdet3d/models/decode_heads/pointnet2_head.py index 271b3d2c83..cc3e570b5f 100644 --- a/mmdet3d/models/decode_heads/pointnet2_head.py +++ b/mmdet3d/models/decode_heads/pointnet2_head.py @@ -15,7 +15,7 @@ class PointNet2Head(Base3DDecodeHead): Args: fp_channels (tuple[tuple[int]]): Tuple of mlp channels in FP modules. - fp_norm_cfg (dict|None): Config of norm layers used in FP modules. + fp_norm_cfg (dict): Config of norm layers used in FP modules. Default: dict(type='BN2d'). """ diff --git a/mmdet3d/models/dense_heads/anchor3d_head.py b/mmdet3d/models/dense_heads/anchor3d_head.py index d5be4735c7..bb7681b6d3 100644 --- a/mmdet3d/models/dense_heads/anchor3d_head.py +++ b/mmdet3d/models/dense_heads/anchor3d_head.py @@ -144,7 +144,7 @@ def forward_single(self, x): x (torch.Tensor): Input features. Returns: - tuple[torch.Tensor]: Contain score of each class, bbox \ + tuple[torch.Tensor]: Contain score of each class, bbox regression and direction classification predictions. """ cls_score = self.conv_cls(x) @@ -162,7 +162,7 @@ def forward(self, feats): features produced by FPN. Returns: - tuple[list[torch.Tensor]]: Multi-level class score, bbox \ + tuple[list[torch.Tensor]]: Multi-level class score, bbox and direction predictions. """ return multi_apply(self.forward_single, feats) @@ -176,7 +176,7 @@ def get_anchors(self, featmap_sizes, input_metas, device='cuda'): device (str): device of current module. Returns: - list[list[torch.Tensor]]: Anchors of each image, valid flags \ + list[list[torch.Tensor]]: Anchors of each image, valid flags of each image. """ num_imgs = len(input_metas) @@ -206,7 +206,7 @@ def loss_single(self, cls_score, bbox_pred, dir_cls_preds, labels, num_total_samples (int): The number of valid samples. Returns: - tuple[torch.Tensor]: Losses of class, bbox \ + tuple[torch.Tensor]: Losses of class, bbox and direction, respectively. """ # classification loss @@ -284,7 +284,7 @@ def add_sin_difference(boxes1, boxes2): the 7th dimension is rotation dimension. Returns: - tuple[torch.Tensor]: ``boxes1`` and ``boxes2`` whose 7th \ + tuple[torch.Tensor]: ``boxes1`` and ``boxes2`` whose 7th dimensions are changed. """ rad_pred_encoding = torch.sin(boxes1[..., 6:7]) * torch.cos( @@ -317,16 +317,16 @@ class predictions. of each sample. gt_labels (list[torch.Tensor]): Gt labels of each sample. input_metas (list[dict]): Contain pcd and img's meta info. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify - which bounding. + gt_bboxes_ignore (list[torch.Tensor]): Specify + which bounding boxes to ignore. Returns: - dict[str, list[torch.Tensor]]: Classification, bbox, and \ + dict[str, list[torch.Tensor]]: Classification, bbox, and direction losses of each level. - loss_cls (list[torch.Tensor]): Classification losses. - loss_bbox (list[torch.Tensor]): Box regression losses. - - loss_dir (list[torch.Tensor]): Direction classification \ + - loss_dir (list[torch.Tensor]): Direction classification losses. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] @@ -384,7 +384,7 @@ def get_bboxes(self, dir_cls_preds (list[torch.Tensor]): Multi-level direction class predictions. input_metas (list[dict]): Contain pcd and img's meta info. - cfg (None | :obj:`ConfigDict`): Training or testing config. + cfg (:obj:`ConfigDict`): Training or testing config. rescale (list[torch.Tensor]): Whether th rescale bbox. Returns: @@ -438,7 +438,7 @@ def get_bboxes_single(self, mlvl_anchors (List[torch.Tensor]): Multi-level anchors in single batch. input_meta (list[dict]): Contain pcd and img's meta info. - cfg (None | :obj:`ConfigDict`): Training or testing config. + cfg (:obj:`ConfigDict`): Training or testing config. rescale (list[torch.Tensor]): whether th rescale bbox. Returns: diff --git a/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py b/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py index 65c4e66e69..90541118bb 100644 --- a/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py +++ b/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py @@ -17,35 +17,45 @@ class AnchorFreeMono3DHead(BaseMono3DDenseHead): num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. - feat_channels (int): Number of hidden channels. Used in child classes. - stacked_convs (int): Number of stacking convs of the head. - strides (tuple): Downsample factor of each feature map. - dcn_on_last_conv (bool): If true, use dcn in the last layer of - towers. Default: False. - conv_bias (bool | str): If specified as `auto`, it will be decided by - the norm_cfg. Bias of conv will be set as True if `norm_cfg` is - None, otherwise False. Default: "auto". - background_label (int | None): Label ID of background, set as 0 for - RPN and num_classes for other heads. It will automatically set as - num_classes if None is given. - use_direction_classifier (bool): Whether to add a direction classifier. - diff_rad_by_sin (bool): Whether to change the difference into sin - difference for box regression loss. - loss_cls (dict): Config of classification loss. - loss_bbox (dict): Config of localization loss. - loss_dir (dict): Config of direction classifier loss. - loss_attr (dict): Config of attribute classifier loss, which is only - active when pred_attrs=True. - bbox_code_size (int): Dimensions of predicted bounding boxes. - pred_attrs (bool): Whether to predict attributes. Default to False. - num_attrs (int): The number of attributes to be predicted. Default: 9. - pred_velo (bool): Whether to predict velocity. Default to False. - pred_bbox2d (bool): Whether to predict 2D boxes. Default to False. - group_reg_dims (tuple[int]): The dimension of each regression target - group. Default: (2, 1, 3, 1, 2). - cls_branch (tuple[int]): Channels for classification branch. + feat_channels (int, optional): Number of hidden channels. + Used in child classes. Defaults to 256. + stacked_convs (int, optional): Number of stacking convs of the head. + strides (tuple, optional): Downsample factor of each feature map. + dcn_on_last_conv (bool, optional): If true, use dcn in the last + layer of towers. Default: False. + conv_bias (bool | str, optional): If specified as `auto`, it will be + decided by the norm_cfg. Bias of conv will be set as True + if `norm_cfg` is None, otherwise False. Default: 'auto'. + background_label (int, optional): Label ID of background, + set as 0 for RPN and num_classes for other heads. + It will automatically set as `num_classes` if None is given. + use_direction_classifier (bool, optional): + Whether to add a direction classifier. + diff_rad_by_sin (bool, optional): Whether to change the difference + into sin difference for box regression loss. Defaults to True. + dir_offset (float, optional): Parameter used in direction + classification. Defaults to 0. + dir_limit_offset (float, optional): Parameter used in direction + classification. Defaults to 0. + loss_cls (dict, optional): Config of classification loss. + loss_bbox (dict, optional): Config of localization loss. + loss_dir (dict, optional): Config of direction classifier loss. + loss_attr (dict, optional): Config of attribute classifier loss, + which is only active when `pred_attrs=True`. + bbox_code_size (int, optional): Dimensions of predicted bounding boxes. + pred_attrs (bool, optional): Whether to predict attributes. + Defaults to False. + num_attrs (int, optional): The number of attributes to be predicted. + Default: 9. + pred_velo (bool, optional): Whether to predict velocity. + Defaults to False. + pred_bbox2d (bool, optional): Whether to predict 2D boxes. + Defaults to False. + group_reg_dims (tuple[int], optional): The dimension of each regression + target group. Default: (2, 1, 3, 1, 2). + cls_branch (tuple[int], optional): Channels for classification branch. Default: (128, 64). - reg_branch (tuple[tuple]): Channels for regression branch. + reg_branch (tuple[tuple], optional): Channels for regression branch. Default: ( (128, 64), # offset (128, 64), # depth @@ -53,14 +63,16 @@ class AnchorFreeMono3DHead(BaseMono3DDenseHead): (64, ), # rot () # velo ), - dir_branch (tuple[int]): Channels for direction classification branch. + dir_branch (tuple[int], optional): Channels for direction + classification branch. Default: (64, ). + attr_branch (tuple[int], optional): Channels for classification branch. Default: (64, ). - attr_branch (tuple[int]): Channels for classification branch. - Default: (64, ). - conv_cfg (dict): Config dict for convolution layer. Default: None. - norm_cfg (dict): Config dict for normalization layer. Default: None. - train_cfg (dict): Training config of anchor head. - test_cfg (dict): Testing config of anchor head. + conv_cfg (dict, optional): Config dict for convolution layer. + Default: None. + norm_cfg (dict, optional): Config dict for normalization layer. + Default: None. + train_cfg (dict, optional): Training config of anchor head. + test_cfg (dict, optional): Testing config of anchor head. """ # noqa: W605 _version = 1 @@ -125,6 +137,7 @@ def __init__( self.use_direction_classifier = use_direction_classifier self.diff_rad_by_sin = diff_rad_by_sin self.dir_offset = dir_offset + self.dir_limit_offset = dir_limit_offset self.loss_cls = build_loss(loss_cls) self.loss_bbox = build_loss(loss_bbox) self.loss_dir = build_loss(loss_dir) @@ -289,7 +302,7 @@ def forward(self, feats): a 4D-tensor. Returns: - tuple: Usually contain classification scores, bbox predictions, \ + tuple: Usually contain classification scores, bbox predictions, and direction class predictions. cls_scores (list[Tensor]): Box scores for each scale level, each is a 4D-tensor, the channel number is @@ -401,7 +414,7 @@ def loss(self, corresponding to each box img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. - gt_bboxes_ignore (None | list[Tensor]): specify which bounding + gt_bboxes_ignore (list[Tensor]): specify which bounding boxes can be ignored when computing the loss. """ diff --git a/mmdet3d/models/dense_heads/centerpoint_head.py b/mmdet3d/models/dense_heads/centerpoint_head.py index 98ab23a588..e748e4e92a 100644 --- a/mmdet3d/models/dense_heads/centerpoint_head.py +++ b/mmdet3d/models/dense_heads/centerpoint_head.py @@ -21,16 +21,16 @@ class SeparateHead(BaseModule): Args: in_channels (int): Input channels for conv_layer. heads (dict): Conv information. - head_conv (int): Output channels. + head_conv (int, optional): Output channels. Default: 64. - final_kernal (int): Kernal size for the last conv layer. + final_kernal (int, optional): Kernal size for the last conv layer. Deafult: 1. - init_bias (float): Initial bias. Default: -2.19. - conv_cfg (dict): Config of conv layer. + init_bias (float, optional): Initial bias. Default: -2.19. + conv_cfg (dict, optional): Config of conv layer. Default: dict(type='Conv2d') - norm_cfg (dict): Config of norm layer. + norm_cfg (dict, optional): Config of norm layer. Default: dict(type='BN2d'). - bias (str): Type of bias. Default: 'auto'. + bias (str, optional): Type of bias. Default: 'auto'. """ def __init__(self, @@ -100,17 +100,17 @@ def forward(self, x): Returns: dict[str: torch.Tensor]: contains the following keys: - -reg （torch.Tensor): 2D regression value with the \ + -reg （torch.Tensor): 2D regression value with the shape of [B, 2, H, W]. - -height (torch.Tensor): Height value with the \ + -height (torch.Tensor): Height value with the shape of [B, 1, H, W]. - -dim (torch.Tensor): Size value with the shape \ + -dim (torch.Tensor): Size value with the shape of [B, 3, H, W]. - -rot (torch.Tensor): Rotation value with the \ + -rot (torch.Tensor): Rotation value with the shape of [B, 2, H, W]. - -vel (torch.Tensor): Velocity value with the \ + -vel (torch.Tensor): Velocity value with the shape of [B, 2, H, W]. - -heatmap (torch.Tensor): Heatmap with the shape of \ + -heatmap (torch.Tensor): Heatmap with the shape of [B, N, H, W]. """ ret_dict = dict() @@ -131,18 +131,19 @@ class DCNSeparateHead(BaseModule): Args: in_channels (int): Input channels for conv_layer. + num_cls (int): Number of classes. heads (dict): Conv information. dcn_config (dict): Config of dcn layer. - num_cls (int): Output channels. + head_conv (int, optional): Output channels. Default: 64. - final_kernal (int): Kernal size for the last conv layer. - Deafult: 1. - init_bias (float): Initial bias. Default: -2.19. - conv_cfg (dict): Config of conv layer. + final_kernal (int, optional): Kernal size for the last conv + layer. Deafult: 1. + init_bias (float, optional): Initial bias. Default: -2.19. + conv_cfg (dict, optional): Config of conv layer. Default: dict(type='Conv2d') - norm_cfg (dict): Config of norm layer. + norm_cfg (dict, optional): Config of norm layer. Default: dict(type='BN2d'). - bias (str): Type of bias. Default: 'auto'. + bias (str, optional): Type of bias. Default: 'auto'. """ # noqa: W605 def __init__(self, @@ -215,17 +216,17 @@ def forward(self, x): Returns: dict[str: torch.Tensor]: contains the following keys: - -reg （torch.Tensor): 2D regression value with the \ + -reg （torch.Tensor): 2D regression value with the shape of [B, 2, H, W]. - -height (torch.Tensor): Height value with the \ + -height (torch.Tensor): Height value with the shape of [B, 1, H, W]. - -dim (torch.Tensor): Size value with the shape \ + -dim (torch.Tensor): Size value with the shape of [B, 3, H, W]. - -rot (torch.Tensor): Rotation value with the \ + -rot (torch.Tensor): Rotation value with the shape of [B, 2, H, W]. - -vel (torch.Tensor): Velocity value with the \ + -vel (torch.Tensor): Velocity value with the shape of [B, 2, H, W]. - -heatmap (torch.Tensor): Heatmap with the shape of \ + -heatmap (torch.Tensor): Heatmap with the shape of [B, N, H, W]. """ center_feat = self.feature_adapt_cls(x) @@ -243,31 +244,30 @@ class CenterHead(BaseModule): """CenterHead for CenterPoint. Args: - mode (str): Mode of the head. Default: '3d'. - in_channels (list[int] | int): Channels of the input feature map. - Default: [128]. - tasks (list[dict]): Task information including class number + in_channels (list[int] | int, optional): Channels of the input + feature map. Default: [128]. + tasks (list[dict], optional): Task information including class number and class names. Default: None. - dataset (str): Name of the dataset. Default: 'nuscenes'. - weight (float): Weight for location loss. Default: 0.25. - code_weights (list[int]): Code weights for location loss. Default: []. - common_heads (dict): Conv information for common heads. + train_cfg (dict, optional): Train-time configs. Default: None. + test_cfg (dict, optional): Test-time configs. Default: None. + bbox_coder (dict, optional): Bbox coder configs. Default: None. + common_heads (dict, optional): Conv information for common heads. Default: dict(). - loss_cls (dict): Config of classification loss function. + loss_cls (dict, optional): Config of classification loss function. Default: dict(type='GaussianFocalLoss', reduction='mean'). - loss_bbox (dict): Config of regression loss function. + loss_bbox (dict, optional): Config of regression loss function. Default: dict(type='L1Loss', reduction='none'). - separate_head (dict): Config of separate head. Default: dict( + separate_head (dict, optional): Config of separate head. Default: dict( type='SeparateHead', init_bias=-2.19, final_kernel=3) - share_conv_channel (int): Output channels for share_conv_layer. - Default: 64. - num_heatmap_convs (int): Number of conv layers for heatmap conv layer. - Default: 2. - conv_cfg (dict): Config of conv layer. + share_conv_channel (int, optional): Output channels for share_conv + layer. Default: 64. + num_heatmap_convs (int, optional): Number of conv layers for heatmap + conv layer. Default: 2. + conv_cfg (dict, optional): Config of conv layer. Default: dict(type='Conv2d') - norm_cfg (dict): Config of norm layer. + norm_cfg (dict, optional): Config of norm layer. Default: dict(type='BN2d'). - bias (str): Type of bias. Default: 'auto'. + bias (str, optional): Type of bias. Default: 'auto'. """ def __init__(self, @@ -366,8 +366,8 @@ def _gather_feat(self, feat, ind, mask=None): feat (torch.tensor): Feature map with the shape of [B, H*W, 10]. ind (torch.Tensor): Index of the ground truth boxes with the shape of [B, max_obj]. - mask (torch.Tensor): Mask of the feature map with the shape - of [B, max_obj]. Default: None. + mask (torch.Tensor, optional): Mask of the feature map with the + shape of [B, max_obj]. Default: None. Returns: torch.Tensor: Feature map after gathering with the shape @@ -392,14 +392,14 @@ def get_targets(self, gt_bboxes_3d, gt_labels_3d): Returns: Returns: - tuple[list[torch.Tensor]]: Tuple of target including \ + tuple[list[torch.Tensor]]: Tuple of target including the following results in order. - list[torch.Tensor]: Heatmap scores. - list[torch.Tensor]: Ground truth boxes. - - list[torch.Tensor]: Indexes indicating the \ + - list[torch.Tensor]: Indexes indicating the position of the valid boxes. - - list[torch.Tensor]: Masks indicating which \ + - list[torch.Tensor]: Masks indicating which boxes are valid. """ heatmaps, anno_boxes, inds, masks = multi_apply( @@ -427,14 +427,14 @@ def get_targets_single(self, gt_bboxes_3d, gt_labels_3d): gt_labels_3d (torch.Tensor): Labels of boxes. Returns: - tuple[list[torch.Tensor]]: Tuple of target including \ + tuple[list[torch.Tensor]]: Tuple of target including the following results in order. - list[torch.Tensor]: Heatmap scores. - list[torch.Tensor]: Ground truth boxes. - - list[torch.Tensor]: Indexes indicating the position \ + - list[torch.Tensor]: Indexes indicating the position of the valid boxes. - - list[torch.Tensor]: Masks indicating which boxes \ + - list[torch.Tensor]: Masks indicating which boxes are valid. """ device = gt_labels_3d.device @@ -718,11 +718,11 @@ def get_task_detections(self, num_class_with_bg, batch_cls_preds, Returns: list[dict[str: torch.Tensor]]: contains the following keys: - -bboxes (torch.Tensor): Prediction bboxes after nms with the \ + -bboxes (torch.Tensor): Prediction bboxes after nms with the shape of [N, 9]. - -scores (torch.Tensor): Prediction scores after nms with the \ + -scores (torch.Tensor): Prediction scores after nms with the shape of [N]. - -labels (torch.Tensor): Prediction labels after nms with the \ + -labels (torch.Tensor): Prediction labels after nms with the shape of [N]. """ predictions_dicts = [] @@ -771,7 +771,7 @@ def get_task_detections(self, num_class_with_bg, batch_cls_preds, boxes_for_nms, top_scores, thresh=self.test_cfg['nms_thr'], - pre_maxsize=self.test_cfg['pre_max_size'], + pre_max_size=self.test_cfg['pre_max_size'], post_max_size=self.test_cfg['post_max_size']) else: selected = [] diff --git a/mmdet3d/models/dense_heads/fcos_mono3d_head.py b/mmdet3d/models/dense_heads/fcos_mono3d_head.py index c7faf21f85..a9a794e446 100644 --- a/mmdet3d/models/dense_heads/fcos_mono3d_head.py +++ b/mmdet3d/models/dense_heads/fcos_mono3d_head.py @@ -20,25 +20,25 @@ class FCOSMono3DHead(AnchorFreeMono3DHead): num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. - regress_ranges (tuple[tuple[int, int]]): Regress range of multiple + regress_ranges (tuple[tuple[int, int]], optional): Regress range of multiple level points. - center_sampling (bool): If true, use center sampling. Default: True. - center_sample_radius (float): Radius of center sampling. Default: 1.5. - norm_on_bbox (bool): If true, normalize the regression targets + center_sampling (bool, optional): If true, use center sampling. Default: True. + center_sample_radius (float, optional): Radius of center sampling. Default: 1.5. + norm_on_bbox (bool, optional): If true, normalize the regression targets with FPN strides. Default: True. - centerness_on_reg (bool): If true, position centerness on the + centerness_on_reg (bool, optional): If true, position centerness on the regress branch. Please refer to https://github.com/tianzhi0549/FCOS/issues/89#issuecomment-516877042. Default: True. - centerness_alpha: Parameter used to adjust the intensity attenuation - from the center to the periphery. Default: 2.5. - loss_cls (dict): Config of classification loss. - loss_bbox (dict): Config of localization loss. - loss_dir (dict): Config of direction classification loss. - loss_attr (dict): Config of attribute classification loss. - loss_centerness (dict): Config of centerness loss. - norm_cfg (dict): dictionary to construct and config norm layer. + centerness_alpha (int, optional): Parameter used to adjust the intensity + attenuation from the center to the periphery. Default: 2.5. + loss_cls (dict, optional): Config of classification loss. + loss_bbox (dict, optional): Config of localization loss. + loss_dir (dict, optional): Config of direction classification loss. + loss_attr (dict, optional): Config of attribute classification loss. + loss_centerness (dict, optional): Config of centerness loss. + norm_cfg (dict, optional): dictionary to construct and config norm layer. Default: norm_cfg=dict(type='GN', num_groups=32, requires_grad=True). - centerness_branch (tuple[int]): Channels for centerness branch. + centerness_branch (tuple[int], optional): Channels for centerness branch. Default: (64, ). """ # noqa: E501 @@ -152,7 +152,7 @@ def forward_single(self, x, scale, stride): is True. Returns: - tuple: scores for each class, bbox and direction class \ + tuple: scores for each class, bbox and direction class predictions, centerness predictions of input feature maps. """ cls_score, bbox_pred, dir_cls_pred, attr_pred, cls_feat, reg_feat = \ @@ -200,7 +200,7 @@ def add_sin_difference(boxes1, boxes2): the 7th dimension is rotation dimension. Returns: - tuple[torch.Tensor]: ``boxes1`` and ``boxes2`` whose 7th \ + tuple[torch.Tensor]: ``boxes1`` and ``boxes2`` whose 7th dimensions are changed. """ rad_pred_encoding = torch.sin(boxes1[..., 6:7]) * torch.cos( @@ -294,7 +294,7 @@ def loss(self, attr_labels (list[Tensor]): Attributes indices of each box. img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. - gt_bboxes_ignore (None | list[Tensor]): specify which bounding + gt_bboxes_ignore (list[Tensor]): specify which bounding boxes can be ignored when computing the loss. Returns: @@ -506,11 +506,11 @@ def get_bboxes(self, rescale (bool): If True, return boxes in original image space Returns: - list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. \ - The first item is an (n, 5) tensor, where the first 4 columns \ - are bounding box positions (tl_x, tl_y, br_x, br_y) and the \ - 5-th column is a score between 0 and 1. The second item is a \ - (n,) tensor where each item is the predicted class label of \ + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where the first 4 columns + are bounding box positions (tl_x, tl_y, br_x, br_y) and the + 5-th column is a score between 0 and 1. The second item is a + (n,) tensor where each item is the predicted class label of the corresponding box. """ assert len(cls_scores) == len(bbox_preds) == len(dir_cls_preds) == \ @@ -579,7 +579,7 @@ def _get_bboxes_single(self, bbox_preds (list[Tensor]): Box energies / deltas for a single scale level with shape (num_points * bbox_code_size, H, W). dir_cls_preds (list[Tensor]): Box scores for direction class - predictions on a single scale level with shape \ + predictions on a single scale level with shape (num_points * 2, H, W) attr_preds (list[Tensor]): Attribute scores for each scale level Has shape (N, num_points * num_attrs, H, W) @@ -699,12 +699,12 @@ def _get_bboxes_single(self, def pts2Dto3D(points, view): """ Args: - points (torch.Tensor): points in 2D images, [N, 3], \ + points (torch.Tensor): points in 2D images, [N, 3], 3 corresponds with x, y in the image and depth. view (np.ndarray): camera instrinsic, [3, 3] Returns: - torch.Tensor: points in 3D space. [N, 3], \ + torch.Tensor: points in 3D space. [N, 3], 3 corresponds with x, y, z in 3D space. """ assert view.shape[0] <= 4 @@ -766,8 +766,8 @@ def get_targets(self, points, gt_bboxes_list, gt_labels_list, Returns: tuple: - concat_lvl_labels (list[Tensor]): Labels of each level. \ - concat_lvl_bbox_targets (list[Tensor]): BBox targets of each \ + concat_lvl_labels (list[Tensor]): Labels of each level. + concat_lvl_bbox_targets (list[Tensor]): BBox targets of each level. """ assert len(points) == len(self.regress_ranges) diff --git a/mmdet3d/models/dense_heads/groupfree3d_head.py b/mmdet3d/models/dense_heads/groupfree3d_head.py index fda0f92745..4e37e26efc 100644 --- a/mmdet3d/models/dense_heads/groupfree3d_head.py +++ b/mmdet3d/models/dense_heads/groupfree3d_head.py @@ -24,13 +24,13 @@ class PointsObjClsModule(BaseModule): Args: in_channel (int): number of channels of seed point features. - num_convs (int): number of conv layers. + num_convs (int, optional): number of conv layers. Default: 3. - conv_cfg (dict): Config of convolution. + conv_cfg (dict, optional): Config of convolution. Default: dict(type='Conv1d'). - norm_cfg (dict): Config of normalization. + norm_cfg (dict, optional): Config of normalization. Default: dict(type='BN1d'). - act_cfg (dict): Config of activation. + act_cfg (dict, optional): Config of activation. Default: dict(type='ReLU'). """ @@ -404,15 +404,15 @@ def loss(self, Args: bbox_preds (dict): Predictions from forward of vote head. points (list[torch.Tensor]): Input points. - gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth \ + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each sample. gt_labels_3d (list[torch.Tensor]): Labels of each sample. - pts_semantic_mask (None | list[torch.Tensor]): Point-wise + pts_semantic_mask (list[torch.Tensor]): Point-wise semantic mask. - pts_instance_mask (None | list[torch.Tensor]): Point-wise + pts_instance_mask (list[torch.Tensor]): Point-wise instance mask. img_metas (list[dict]): Contain pcd and img's meta info. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify + gt_bboxes_ignore (list[torch.Tensor]): Specify which bounding. ret_target (Bool): Return targets or not. @@ -544,12 +544,12 @@ def get_targets(self, Args: points (list[torch.Tensor]): Points of each batch. - gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth \ + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each batch. gt_labels_3d (list[torch.Tensor]): Labels of each batch. - pts_semantic_mask (None | list[torch.Tensor]): Point-wise semantic + pts_semantic_mask (list[torch.Tensor]): Point-wise semantic label of each batch. - pts_instance_mask (None | list[torch.Tensor]): Point-wise instance + pts_instance_mask (list[torch.Tensor]): Point-wise instance label of each batch. bbox_preds (torch.Tensor): Bounding box predictions of vote head. max_gt_num (int): Max number of GTs for single batch. @@ -656,12 +656,12 @@ def get_targets_single(self, Args: points (torch.Tensor): Points of each batch. - gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth \ + gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth boxes of each batch. gt_labels_3d (torch.Tensor): Labels of each batch. - pts_semantic_mask (None | torch.Tensor): Point-wise semantic + pts_semantic_mask (torch.Tensor): Point-wise semantic label of each batch. - pts_instance_mask (None | torch.Tensor): Point-wise instance + pts_instance_mask (torch.Tensor): Point-wise instance label of each batch. max_gt_nums (int): Max number of GTs for single batch. seed_points (torch.Tensor): Coordinates of seed points. @@ -709,7 +709,7 @@ def get_targets_single(self, if self.bbox_coder.with_rot: vote_targets = points.new_zeros([num_points, 4 * self.gt_per_seed]) vote_target_idx = points.new_zeros([num_points], dtype=torch.long) - box_indices_all = gt_bboxes_3d.points_in_boxes(points) + box_indices_all = gt_bboxes_3d.points_in_boxes_part(points) for i in range(gt_labels_3d.shape[0]): box_indices = box_indices_all[:, i] indices = torch.nonzero( @@ -950,7 +950,7 @@ def multiclass_nms_single(self, obj_scores, sem_scores, bbox, points, box_dim=bbox.shape[-1], with_yaw=self.bbox_coder.with_rot, origin=(0.5, 0.5, 0.5)) - box_indices = bbox.points_in_boxes_batch(points) + box_indices = bbox.points_in_boxes_all(points) corner3d = bbox.corners minmax_box3d = corner3d.new(torch.Size((corner3d.shape[0], 6))) diff --git a/mmdet3d/models/dense_heads/parta2_rpn_head.py b/mmdet3d/models/dense_heads/parta2_rpn_head.py index 418a161f51..95c8adffb8 100644 --- a/mmdet3d/models/dense_heads/parta2_rpn_head.py +++ b/mmdet3d/models/dense_heads/parta2_rpn_head.py @@ -99,20 +99,20 @@ def loss(self, bbox_preds (list[torch.Tensor]): Multi-level bbox predictions. dir_cls_preds (list[torch.Tensor]): Multi-level direction class predictions. - gt_bboxes (list[:obj:`BaseInstance3DBoxes`]): Ground truth boxes \ + gt_bboxes (list[:obj:`BaseInstance3DBoxes`]): Ground truth boxes of each sample. gt_labels (list[torch.Tensor]): Labels of each sample. input_metas (list[dict]): Point cloud and image's meta info. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify + gt_bboxes_ignore (list[torch.Tensor]): Specify which bounding. Returns: - dict[str, list[torch.Tensor]]: Classification, bbox, and \ + dict[str, list[torch.Tensor]]: Classification, bbox, and direction losses of each level. - loss_rpn_cls (list[torch.Tensor]): Classification losses. - loss_rpn_bbox (list[torch.Tensor]): Box regression losses. - - loss_rpn_dir (list[torch.Tensor]): Direction classification \ + - loss_rpn_dir (list[torch.Tensor]): Direction classification losses. """ loss_dict = super().loss(cls_scores, bbox_preds, dir_cls_preds, @@ -142,7 +142,7 @@ def get_bboxes_single(self, mlvl_anchors (List[torch.Tensor]): Multi-level anchors in single batch. input_meta (list[dict]): Contain pcd and img's meta info. - cfg (None | :obj:`ConfigDict`): Training or testing config. + cfg (:obj:`ConfigDict`): Training or testing config. rescale (list[torch.Tensor]): whether th rescale bbox. Returns: @@ -239,7 +239,7 @@ def class_agnostic_nms(self, mlvl_bboxes, mlvl_bboxes_for_nms, Multi-level bbox. score_thr (int): Score threshold. max_num (int): Max number of bboxes after nms. - cfg (None | :obj:`ConfigDict`): Training or testing config. + cfg (:obj:`ConfigDict`): Training or testing config. input_meta (dict): Contain pcd and img's meta info. Returns: diff --git a/mmdet3d/models/dense_heads/shape_aware_head.py b/mmdet3d/models/dense_heads/shape_aware_head.py index 082f46ab6c..15d22a1f29 100644 --- a/mmdet3d/models/dense_heads/shape_aware_head.py +++ b/mmdet3d/models/dense_heads/shape_aware_head.py @@ -29,15 +29,17 @@ class BaseShapeHead(BaseModule): num_base_anchors (int): Number of anchors per location. box_code_size (int): The dimension of boxes to be encoded. in_channels (int): Input channels for convolutional layers. - shared_conv_channels (tuple): Channels for shared convolutional \ - layers. Default: (64, 64). \ - shared_conv_strides (tuple): Strides for shared convolutional \ - layers. Default: (1, 1). - use_direction_classifier (bool, optional): Whether to use direction \ + shared_conv_channels (tuple, optional): Channels for shared + convolutional layers. Default: (64, 64). + shared_conv_strides (tuple, optional): Strides for shared + convolutional layers. Default: (1, 1). + use_direction_classifier (bool, optional): Whether to use direction classifier. Default: True. - conv_cfg (dict): Config of conv layer. Default: dict(type='Conv2d') - norm_cfg (dict): Config of norm layer. Default: dict(type='BN2d'). - bias (bool|str, optional): Type of bias. Default: False. + conv_cfg (dict, optional): Config of conv layer. + Default: dict(type='Conv2d') + norm_cfg (dict, optional): Config of norm layer. + Default: dict(type='BN2d'). + bias (bool | str, optional): Type of bias. Default: False. """ def __init__(self, @@ -126,11 +128,11 @@ def forward(self, x): [B, C, H, W]. Returns: - dict[torch.Tensor]: Contain score of each class, bbox \ - regression and direction classification predictions. \ - Note that all the returned tensors are reshaped as \ - [bs*num_base_anchors*H*W, num_cls/box_code_size/dir_bins]. \ - It is more convenient to concat anchors for different \ + dict[torch.Tensor]: Contain score of each class, bbox + regression and direction classification predictions. + Note that all the returned tensors are reshaped as + [bs*num_base_anchors*H*W, num_cls/box_code_size/dir_bins]. + It is more convenient to concat anchors for different classes even though they have different feature map sizes. """ x = self.shared_conv(x) @@ -167,9 +169,9 @@ class ShapeAwareHead(Anchor3DHead): Args: tasks (dict): Shape-aware groups of multi-class objects. - assign_per_class (bool, optional): Whether to do assignment for each \ + assign_per_class (bool, optional): Whether to do assignment for each class. Default: True. - kwargs (dict): Other arguments are the same as those in \ + kwargs (dict): Other arguments are the same as those in :class:`Anchor3DHead`. """ @@ -216,7 +218,7 @@ def forward_single(self, x): Args: x (torch.Tensor): Input features. Returns: - tuple[torch.Tensor]: Contain score of each class, bbox \ + tuple[torch.Tensor]: Contain score of each class, bbox regression and direction classification predictions. """ results = [] @@ -262,7 +264,7 @@ def loss_single(self, cls_score, bbox_pred, dir_cls_preds, labels, num_total_samples (int): The number of valid samples. Returns: - tuple[torch.Tensor]: Losses of class, bbox \ + tuple[torch.Tensor]: Losses of class, bbox and direction, respectively. """ # classification loss @@ -324,16 +326,16 @@ class predictions. of each sample. gt_labels (list[torch.Tensor]): Gt labels of each sample. input_metas (list[dict]): Contain pcd and img's meta info. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify + gt_bboxes_ignore (list[torch.Tensor]): Specify which bounding. Returns: - dict[str, list[torch.Tensor]]: Classification, bbox, and \ + dict[str, list[torch.Tensor]]: Classification, bbox, and direction losses of each level. - loss_cls (list[torch.Tensor]): Classification losses. - loss_bbox (list[torch.Tensor]): Box regression losses. - - loss_dir (list[torch.Tensor]): Direction classification \ + - loss_dir (list[torch.Tensor]): Direction classification losses. """ device = cls_scores[0].device @@ -387,7 +389,7 @@ def get_bboxes(self, dir_cls_preds (list[torch.Tensor]): Multi-level direction class predictions. input_metas (list[dict]): Contain pcd and img's meta info. - cfg (None | :obj:`ConfigDict`): Training or testing config. + cfg (:obj:`ConfigDict`, optional): Training or testing config. Default: None. rescale (list[torch.Tensor], optional): Whether to rescale bbox. Default: False. @@ -442,8 +444,8 @@ def get_bboxes_single(self, mlvl_anchors (List[torch.Tensor]): Multi-level anchors in single batch. input_meta (list[dict]): Contain pcd and img's meta info. - cfg (None | :obj:`ConfigDict`): Training or testing config. - rescale (list[torch.Tensor], optional): whether to rescale bbox. \ + cfg (:obj:`ConfigDict`): Training or testing config. + rescale (list[torch.Tensor], optional): whether to rescale bbox. Default: False. Returns: diff --git a/mmdet3d/models/dense_heads/ssd_3d_head.py b/mmdet3d/models/dense_heads/ssd_3d_head.py index c5feee107f..1da038d47d 100644 --- a/mmdet3d/models/dense_heads/ssd_3d_head.py +++ b/mmdet3d/models/dense_heads/ssd_3d_head.py @@ -126,15 +126,15 @@ def loss(self, Args: bbox_preds (dict): Predictions from forward of SSD3DHead. points (list[torch.Tensor]): Input points. - gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth \ + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each sample. gt_labels_3d (list[torch.Tensor]): Labels of each sample. - pts_semantic_mask (None | list[torch.Tensor]): Point-wise + pts_semantic_mask (list[torch.Tensor]): Point-wise semantic mask. - pts_instance_mask (None | list[torch.Tensor]): Point-wise + pts_instance_mask (list[torch.Tensor]): Point-wise instance mask. img_metas (list[dict]): Contain pcd and img's meta info. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify + gt_bboxes_ignore (list[torch.Tensor]): Specify which bounding. Returns: @@ -229,12 +229,12 @@ def get_targets(self, Args: points (list[torch.Tensor]): Points of each batch. - gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth \ + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each batch. gt_labels_3d (list[torch.Tensor]): Labels of each batch. - pts_semantic_mask (None | list[torch.Tensor]): Point-wise semantic + pts_semantic_mask (list[torch.Tensor]): Point-wise semantic label of each batch. - pts_instance_mask (None | list[torch.Tensor]): Point-wise instance + pts_instance_mask (list[torch.Tensor]): Point-wise instance label of each batch. bbox_preds (torch.Tensor): Bounding box predictions of ssd3d head. @@ -318,12 +318,12 @@ def get_targets_single(self, Args: points (torch.Tensor): Points of each batch. - gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth \ + gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth boxes of each batch. gt_labels_3d (torch.Tensor): Labels of each batch. - pts_semantic_mask (None | torch.Tensor): Point-wise semantic + pts_semantic_mask (torch.Tensor): Point-wise semantic label of each batch. - pts_instance_mask (None | torch.Tensor): Point-wise instance + pts_instance_mask (torch.Tensor): Point-wise instance label of each batch. aggregated_points (torch.Tensor): Aggregated points from candidate points layer. @@ -492,7 +492,7 @@ def multiclass_nms_single(self, obj_scores, sem_scores, bbox, points, origin=(0.5, 0.5, 0.5)) if isinstance(bbox, (LiDARInstance3DBoxes, DepthInstance3DBoxes)): - box_indices = bbox.points_in_boxes_batch(points) + box_indices = bbox.points_in_boxes_all(points) nonempty_box_mask = box_indices.T.sum(1) >= 0 else: raise NotImplementedError('Unsupported bbox type!') @@ -548,7 +548,7 @@ def _assign_targets_by_points_inside(self, bboxes_3d, points): inside bbox and the index of box where each point are in. """ if isinstance(bboxes_3d, (LiDARInstance3DBoxes, DepthInstance3DBoxes)): - points_mask = bboxes_3d.points_in_boxes_batch(points) + points_mask = bboxes_3d.points_in_boxes_all(points) assignment = points_mask.argmax(dim=-1) else: raise NotImplementedError('Unsupported bbox type!') diff --git a/mmdet3d/models/dense_heads/train_mixins.py b/mmdet3d/models/dense_heads/train_mixins.py index 21ce738477..6e4a8b3e38 100644 --- a/mmdet3d/models/dense_heads/train_mixins.py +++ b/mmdet3d/models/dense_heads/train_mixins.py @@ -24,7 +24,7 @@ def anchor_target_3d(self, gt_bboxes_list (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each image. input_metas (list[dict]): Meta info of each image. - gt_bboxes_ignore_list (None | list): Ignore list of gt bboxes. + gt_bboxes_ignore_list (list): Ignore list of gt bboxes. gt_labels_list (list[torch.Tensor]): Gt labels of batches. label_channels (int): The channel of labels. num_classes (int): The number of classes. diff --git a/mmdet3d/models/dense_heads/vote_head.py b/mmdet3d/models/dense_heads/vote_head.py index 13371b1bb3..3887db1198 100644 --- a/mmdet3d/models/dense_heads/vote_head.py +++ b/mmdet3d/models/dense_heads/vote_head.py @@ -233,15 +233,15 @@ def loss(self, Args: bbox_preds (dict): Predictions from forward of vote head. points (list[torch.Tensor]): Input points. - gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth \ + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each sample. gt_labels_3d (list[torch.Tensor]): Labels of each sample. - pts_semantic_mask (None | list[torch.Tensor]): Point-wise + pts_semantic_mask (list[torch.Tensor]): Point-wise semantic mask. - pts_instance_mask (None | list[torch.Tensor]): Point-wise + pts_instance_mask (list[torch.Tensor]): Point-wise instance mask. img_metas (list[dict]): Contain pcd and img's meta info. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify + gt_bboxes_ignore (list[torch.Tensor]): Specify which bounding. ret_target (Bool): Return targets or not. @@ -357,12 +357,12 @@ def get_targets(self, Args: points (list[torch.Tensor]): Points of each batch. - gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth \ + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each batch. gt_labels_3d (list[torch.Tensor]): Labels of each batch. - pts_semantic_mask (None | list[torch.Tensor]): Point-wise semantic + pts_semantic_mask (list[torch.Tensor]): Point-wise semantic label of each batch. - pts_instance_mask (None | list[torch.Tensor]): Point-wise instance + pts_instance_mask (list[torch.Tensor]): Point-wise instance label of each batch. bbox_preds (torch.Tensor): Bounding box predictions of vote head. @@ -446,12 +446,12 @@ def get_targets_single(self, Args: points (torch.Tensor): Points of each batch. - gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth \ + gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth boxes of each batch. gt_labels_3d (torch.Tensor): Labels of each batch. - pts_semantic_mask (None | torch.Tensor): Point-wise semantic + pts_semantic_mask (torch.Tensor): Point-wise semantic label of each batch. - pts_instance_mask (None | torch.Tensor): Point-wise instance + pts_instance_mask (torch.Tensor): Point-wise instance label of each batch. aggregated_points (torch.Tensor): Aggregated points from vote aggregation layer. @@ -470,7 +470,7 @@ def get_targets_single(self, vote_target_masks = points.new_zeros([num_points], dtype=torch.long) vote_target_idx = points.new_zeros([num_points], dtype=torch.long) - box_indices_all = gt_bboxes_3d.points_in_boxes_batch(points) + box_indices_all = gt_bboxes_3d.points_in_boxes_all(points) for i in range(gt_labels_3d.shape[0]): box_indices = box_indices_all[:, i] indices = torch.nonzero( @@ -620,7 +620,7 @@ def multiclass_nms_single(self, obj_scores, sem_scores, bbox, points, box_dim=bbox.shape[-1], with_yaw=self.bbox_coder.with_rot, origin=(0.5, 0.5, 0.5)) - box_indices = bbox.points_in_boxes_batch(points) + box_indices = bbox.points_in_boxes_all(points) corner3d = bbox.corners minmax_box3d = corner3d.new(torch.Size((corner3d.shape[0], 6))) diff --git a/mmdet3d/models/detectors/centerpoint.py b/mmdet3d/models/detectors/centerpoint.py index a7f0796f27..c33d9dcd97 100644 --- a/mmdet3d/models/detectors/centerpoint.py +++ b/mmdet3d/models/detectors/centerpoint.py @@ -96,7 +96,8 @@ def aug_test_pts(self, feats, img_metas, rescale=False): Args: feats (list[torch.Tensor]): Feature of point cloud. img_metas (list[dict]): Meta information of samples. - rescale (bool): Whether to rescale bboxes. Default: False. + rescale (bool, optional): Whether to rescale bboxes. + Default: False. Returns: dict: Returned bboxes consists of the following keys: diff --git a/mmdet3d/models/detectors/groupfree3dnet.py b/mmdet3d/models/detectors/groupfree3dnet.py index e50aec5ecb..9ebac3284c 100644 --- a/mmdet3d/models/detectors/groupfree3dnet.py +++ b/mmdet3d/models/detectors/groupfree3dnet.py @@ -37,11 +37,11 @@ def forward_train(self, img_metas (list): Image metas. gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): gt bboxes of each batch. gt_labels_3d (list[torch.Tensor]): gt class labels of each batch. - pts_semantic_mask (None | list[torch.Tensor]): point-wise semantic + pts_semantic_mask (list[torch.Tensor]): point-wise semantic label of each batch. - pts_instance_mask (None | list[torch.Tensor]): point-wise instance + pts_instance_mask (list[torch.Tensor]): point-wise instance label of each batch. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify + gt_bboxes_ignore (list[torch.Tensor]): Specify which bounding. Returns: diff --git a/mmdet3d/models/detectors/h3dnet.py b/mmdet3d/models/detectors/h3dnet.py index 36fcb3bc45..281a56d3b6 100644 --- a/mmdet3d/models/detectors/h3dnet.py +++ b/mmdet3d/models/detectors/h3dnet.py @@ -46,11 +46,11 @@ def forward_train(self, img_metas (list): Image metas. gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): gt bboxes of each batch. gt_labels_3d (list[torch.Tensor]): gt class labels of each batch. - pts_semantic_mask (None | list[torch.Tensor]): point-wise semantic + pts_semantic_mask (list[torch.Tensor]): point-wise semantic label of each batch. - pts_instance_mask (None | list[torch.Tensor]): point-wise instance + pts_instance_mask (list[torch.Tensor]): point-wise instance label of each batch. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify + gt_bboxes_ignore (list[torch.Tensor]): Specify which bounding. Returns: diff --git a/mmdet3d/models/detectors/imvotenet.py b/mmdet3d/models/detectors/imvotenet.py index 571e199739..3f375eafc5 100644 --- a/mmdet3d/models/detectors/imvotenet.py +++ b/mmdet3d/models/detectors/imvotenet.py @@ -148,21 +148,21 @@ def __init__(self, if self.with_img_backbone: if img_pretrained is not None: - warnings.warn('DeprecationWarning: pretrained is a deprecated \ - key, please consider using init_cfg') + warnings.warn('DeprecationWarning: pretrained is a deprecated ' + 'key, please consider using init_cfg.') self.img_backbone.init_cfg = dict( type='Pretrained', checkpoint=img_pretrained) if self.with_img_roi_head: if img_pretrained is not None: - warnings.warn('DeprecationWarning: pretrained is a deprecated \ - key, please consider using init_cfg') + warnings.warn('DeprecationWarning: pretrained is a deprecated ' + 'key, please consider using init_cfg.') self.img_roi_head.init_cfg = dict( type='Pretrained', checkpoint=img_pretrained) if self.with_pts_backbone: if img_pretrained is not None: - warnings.warn('DeprecationWarning: pretrained is a deprecated \ - key, please consider using init_cfg') + warnings.warn('DeprecationWarning: pretrained is a deprecated ' + 'key, please consider using init_cfg.') self.pts_backbone.init_cfg = dict( type='Pretrained', checkpoint=pts_pretrained) @@ -392,9 +392,9 @@ def forward_train(self, with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. gt_labels (list[torch.Tensor]): class indices for each 2d bounding box. - gt_bboxes_ignore (None | list[torch.Tensor]): specify which + gt_bboxes_ignore (list[torch.Tensor]): specify which 2d bounding boxes can be ignored when computing the loss. - gt_masks (None | torch.Tensor): true segmentation masks for each + gt_masks (torch.Tensor): true segmentation masks for each 2d bbox, used if the architecture supports a segmentation task. proposals: override rpn proposals (2d) with custom proposals. Use when `with_rpn` is False. @@ -402,9 +402,9 @@ def forward_train(self, not supported yet. gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): 3d gt bboxes. gt_labels_3d (list[torch.Tensor]): gt class labels for 3d bboxes. - pts_semantic_mask (None | list[torch.Tensor]): point-wise semantic + pts_semantic_mask (list[torch.Tensor]): point-wise semantic label of each batch. - pts_instance_mask (None | list[torch.Tensor]): point-wise instance + pts_instance_mask (list[torch.Tensor]): point-wise instance label of each batch. Returns: diff --git a/mmdet3d/models/detectors/mvx_two_stage.py b/mmdet3d/models/detectors/mvx_two_stage.py index fae6d673a5..726a145618 100644 --- a/mmdet3d/models/detectors/mvx_two_stage.py +++ b/mmdet3d/models/detectors/mvx_two_stage.py @@ -83,21 +83,21 @@ def __init__(self, if self.with_img_backbone: if img_pretrained is not None: - warnings.warn('DeprecationWarning: pretrained is a deprecated \ - key, please consider using init_cfg') + warnings.warn('DeprecationWarning: pretrained is a deprecated ' + 'key, please consider using init_cfg.') self.img_backbone.init_cfg = dict( type='Pretrained', checkpoint=img_pretrained) if self.with_img_roi_head: if img_pretrained is not None: - warnings.warn('DeprecationWarning: pretrained is a deprecated \ - key, please consider using init_cfg') + warnings.warn('DeprecationWarning: pretrained is a deprecated ' + 'key, please consider using init_cfg.') self.img_roi_head.init_cfg = dict( type='Pretrained', checkpoint=img_pretrained) if self.with_pts_backbone: if img_pretrained is not None: - warnings.warn('DeprecationWarning: pretrained is a deprecated \ - key, please consider using init_cfg') + warnings.warn('DeprecationWarning: pretrained is a deprecated ' + 'key, please consider using init_cfg.') self.pts_backbone.init_cfg = dict( type='Pretrained', checkpoint=pts_pretrained) @@ -259,7 +259,7 @@ def forward_train(self, of 2D boxes in images. Defaults to None. gt_bboxes (list[torch.Tensor], optional): Ground truth 2D boxes in images. Defaults to None. - img (torch.Tensor optional): Images of each sample with shape + img (torch.Tensor, optional): Images of each sample with shape (N, C, H, W). Defaults to None. proposals ([list[torch.Tensor], optional): Predicted proposals used for training Fast RCNN. Defaults to None. diff --git a/mmdet3d/models/detectors/single_stage_mono3d.py b/mmdet3d/models/detectors/single_stage_mono3d.py index 6217bbe3c8..3219154579 100644 --- a/mmdet3d/models/detectors/single_stage_mono3d.py +++ b/mmdet3d/models/detectors/single_stage_mono3d.py @@ -47,14 +47,15 @@ def forward_train(self, image in [tl_x, tl_y, br_x, br_y] format. gt_labels (list[Tensor]): Class indices corresponding to each box gt_bboxes_3d (list[Tensor]): Each item are the 3D truth boxes for - each image in [x, y, z, w, l, h, theta, vx, vy] format. + each image in [x, y, z, x_size, y_size, z_size, yaw, vx, vy] + format. gt_labels_3d (list[Tensor]): 3D class indices corresponding to each box. centers2d (list[Tensor]): Projected 3D centers onto 2D images. depths (list[Tensor]): Depth of projected centers on 2D images. attr_labels (list[Tensor], optional): Attribute indices corresponding to each box - gt_bboxes_ignore (None | list[Tensor]): Specify which bounding + gt_bboxes_ignore (list[Tensor]): Specify which bounding boxes can be ignored when computing the loss. Returns: diff --git a/mmdet3d/models/detectors/votenet.py b/mmdet3d/models/detectors/votenet.py index 7b3af721cc..384204d4ed 100644 --- a/mmdet3d/models/detectors/votenet.py +++ b/mmdet3d/models/detectors/votenet.py @@ -39,11 +39,11 @@ def forward_train(self, img_metas (list): Image metas. gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): gt bboxes of each batch. gt_labels_3d (list[torch.Tensor]): gt class labels of each batch. - pts_semantic_mask (None | list[torch.Tensor]): point-wise semantic + pts_semantic_mask (list[torch.Tensor]): point-wise semantic label of each batch. - pts_instance_mask (None | list[torch.Tensor]): point-wise instance + pts_instance_mask (list[torch.Tensor]): point-wise instance label of each batch. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify + gt_bboxes_ignore (list[torch.Tensor]): Specify which bounding. Returns: diff --git a/mmdet3d/models/fusion_layers/point_fusion.py b/mmdet3d/models/fusion_layers/point_fusion.py index 0739b6585d..a6f4297d6c 100644 --- a/mmdet3d/models/fusion_layers/point_fusion.py +++ b/mmdet3d/models/fusion_layers/point_fusion.py @@ -29,9 +29,9 @@ def point_sample( img_features (torch.Tensor): 1 x C x H x W image features. points (torch.Tensor): Nx3 point cloud in LiDAR coordinates. lidar2img_rt (torch.Tensor): 4x4 transformation matrix. - img_scale_factor (torch.Tensor): Scale factor with shape of \ + img_scale_factor (torch.Tensor): Scale factor with shape of (w_scale, h_scale). - img_crop_offset (torch.Tensor): Crop offset used to crop \ + img_crop_offset (torch.Tensor): Crop offset used to crop image during data augmentation with shape of (w_offset, h_offset). img_flip (bool): Whether the image is flipped. img_pad_shape (tuple[int]): int tuple indicates the h & w after diff --git a/mmdet3d/models/losses/axis_aligned_iou_loss.py b/mmdet3d/models/losses/axis_aligned_iou_loss.py index d72d67230b..a09acf2a9f 100644 --- a/mmdet3d/models/losses/axis_aligned_iou_loss.py +++ b/mmdet3d/models/losses/axis_aligned_iou_loss.py @@ -53,7 +53,7 @@ def forward(self, Args: pred (torch.Tensor): Bbox predictions with shape [..., 3]. target (torch.Tensor): Bbox targets (gt) with shape [..., 3]. - weight (torch.Tensor|float, optional): Weight of loss. \ + weight (torch.Tensor | float, optional): Weight of loss. Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. diff --git a/mmdet3d/models/losses/chamfer_distance.py b/mmdet3d/models/losses/chamfer_distance.py index 2120a34c83..14e9a35f93 100644 --- a/mmdet3d/models/losses/chamfer_distance.py +++ b/mmdet3d/models/losses/chamfer_distance.py @@ -28,13 +28,13 @@ def chamfer_distance(src, Returns: tuple: Source and Destination loss with the corresponding indices. - - loss_src (torch.Tensor): The min distance \ + - loss_src (torch.Tensor): The min distance from source to destination. - - loss_dst (torch.Tensor): The min distance \ + - loss_dst (torch.Tensor): The min distance from destination to source. - - indices1 (torch.Tensor): Index the min distance point \ + - indices1 (torch.Tensor): Index the min distance point for each point in source to destination. - - indices2 (torch.Tensor): Index the min distance point \ + - indices2 (torch.Tensor): Index the min distance point for each point in destination to source. """ @@ -124,10 +124,10 @@ def forward(self, Defaults to False. Returns: - tuple[torch.Tensor]: If ``return_indices=True``, return losses of \ - source and target with their corresponding indices in the \ - order of ``(loss_source, loss_target, indices1, indices2)``. \ - If ``return_indices=False``, return \ + tuple[torch.Tensor]: If ``return_indices=True``, return losses of + source and target with their corresponding indices in the + order of ``(loss_source, loss_target, indices1, indices2)``. + If ``return_indices=False``, return ``(loss_source, loss_target)``. """ assert reduction_override in (None, 'none', 'mean', 'sum') diff --git a/mmdet3d/models/middle_encoders/sparse_encoder.py b/mmdet3d/models/middle_encoders/sparse_encoder.py index 0fbe48cdfc..b406b0d41e 100644 --- a/mmdet3d/models/middle_encoders/sparse_encoder.py +++ b/mmdet3d/models/middle_encoders/sparse_encoder.py @@ -13,19 +13,21 @@ class SparseEncoder(nn.Module): Args: in_channels (int): The number of input channels. sparse_shape (list[int]): The sparse shape of input tensor. - order (list[str]): Order of conv module. Defaults to ('conv', - 'norm', 'act'). - norm_cfg (dict): Config of normalization layer. Defaults to + order (list[str], optional): Order of conv module. + Defaults to ('conv', 'norm', 'act'). + norm_cfg (dict, optional): Config of normalization layer. Defaults to dict(type='BN1d', eps=1e-3, momentum=0.01). - base_channels (int): Out channels for conv_input layer. + base_channels (int, optional): Out channels for conv_input layer. Defaults to 16. - output_channels (int): Out channels for conv_out layer. + output_channels (int, optional): Out channels for conv_out layer. Defaults to 128. - encoder_channels (tuple[tuple[int]]): + encoder_channels (tuple[tuple[int]], optional): Convolutional channels of each encode block. - encoder_paddings (tuple[tuple[int]]): Paddings of each encode block. + encoder_paddings (tuple[tuple[int]], optional): + Paddings of each encode block. Defaults to ((16, ), (32, 32, 32), (64, 64, 64), (64, 64, 64)). - block_type (str): Type of the block to use. Defaults to 'conv_module'. + block_type (str, optional): Type of the block to use. + Defaults to 'conv_module'. """ def __init__(self, @@ -98,7 +100,7 @@ def forward(self, voxel_features, coors, batch_size): Args: voxel_features (torch.float32): Voxel features in shape (N, C). - coors (torch.int32): Coordinates in shape (N, 4), \ + coors (torch.int32): Coordinates in shape (N, 4), the columns in the order of (batch_idx, z_idx, y_idx, x_idx). batch_size (int): Batch size. @@ -138,9 +140,9 @@ def make_encoder_layers(self, make_block (method): A bounded function to build blocks. norm_cfg (dict[str]): Config of normalization layer. in_channels (int): The number of encoder input channels. - block_type (str): Type of the block to use. Defaults to - 'conv_module'. - conv_cfg (dict): Config of conv layer. Defaults to + block_type (str, optional): Type of the block to use. + Defaults to 'conv_module'. + conv_cfg (dict, optional): Config of conv layer. Defaults to dict(type='SubMConv3d'). Returns: diff --git a/mmdet3d/models/model_utils/transformer.py b/mmdet3d/models/model_utils/transformer.py index 2db8a2859d..c58cd7bf3d 100644 --- a/mmdet3d/models/model_utils/transformer.py +++ b/mmdet3d/models/model_utils/transformer.py @@ -14,15 +14,16 @@ class GroupFree3DMHA(MultiheadAttention): embed_dims (int): The embedding dimension. num_heads (int): Parallel attention heads. Same as `nn.MultiheadAttention`. - attn_drop (float): A Dropout layer on attn_output_weights. Default 0.0. - proj_drop (float): A Dropout layer. Default 0.0. - dropout_layer (obj:`ConfigDict`): The dropout_layer used + attn_drop (float, optional): A Dropout layer on attn_output_weights. + Defaults to 0.0. + proj_drop (float, optional): A Dropout layer. Defaults to 0.0. + dropout_layer (obj:`ConfigDict`, optional): The dropout_layer used when adding the shortcut. - init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. - Default: None. - batch_first (bool): Key, Query and Value are shape of + init_cfg (obj:`mmcv.ConfigDict`, optional): The Config for + initialization. Default: None. + batch_first (bool, optional): Key, Query and Value are shape of (batch, n, embed_dim) - or (n, batch, embed_dim). Default to False. + or (n, batch, embed_dim). Defaults to False. """ def __init__(self, @@ -57,26 +58,26 @@ def forward(self, embed_dims]. Same in `nn.MultiheadAttention.forward`. key (Tensor): The key tensor with shape [num_keys, bs, embed_dims]. Same in `nn.MultiheadAttention.forward`. - If None, the ``query`` will be used. Defaults to None. + If None, the ``query`` will be used. value (Tensor): The value tensor with same shape as `key`. - Same in `nn.MultiheadAttention.forward`. Defaults to None. + Same in `nn.MultiheadAttention.forward`. If None, the `key` will be used. identity (Tensor): This tensor, with the same shape as x, - will be used for the identity link. - If None, `x` will be used. Defaults to None. - query_pos (Tensor): The positional encoding for query, with - the same shape as `x`. If not None, it will - be added to `x` before forward function. Defaults to None. - key_pos (Tensor): The positional encoding for `key`, with the - same shape as `key`. Defaults to None. If not None, it will - be added to `key` before forward function. If None, and - `query_pos` has the same shape as `key`, then `query_pos` + will be used for the identity link. If None, `x` will be used. + query_pos (Tensor, optional): The positional encoding for query, + with the same shape as `x`. Defaults to None. + If not None, it will be added to `x` before forward function. + key_pos (Tensor, optional): The positional encoding for `key`, + with the same shape as `key`. Defaults to None. If not None, + it will be added to `key` before forward function. If None, + and `query_pos` has the same shape as `key`, then `query_pos` will be used for `key_pos`. Defaults to None. - attn_mask (Tensor): ByteTensor mask with shape [num_queries, - num_keys]. Same in `nn.MultiheadAttention.forward`. - Defaults to None. - key_padding_mask (Tensor): ByteTensor with shape [bs, num_keys]. + attn_mask (Tensor, optional): ByteTensor mask with shape + [num_queries, num_keys]. Same in `nn.MultiheadAttention.forward`. Defaults to None. + key_padding_mask (Tensor, optional): ByteTensor with shape + [bs, num_keys]. Same in `nn.MultiheadAttention.forward`. + Defaults to None. Returns: Tensor: forwarded results with shape [num_queries, bs, embed_dims]. @@ -112,7 +113,7 @@ class ConvBNPositionalEncoding(nn.Module): Args: input_channel (int): input features dim. - num_pos_feats (int): output position features dim. + num_pos_feats (int, optional): output position features dim. Defaults to 288 to be consistent with seed features dim. """ diff --git a/mmdet3d/models/model_utils/vote_module.py b/mmdet3d/models/model_utils/vote_module.py index 8862cd7270..f5e0f520d5 100644 --- a/mmdet3d/models/model_utils/vote_module.py +++ b/mmdet3d/models/model_utils/vote_module.py @@ -13,22 +13,25 @@ class VoteModule(nn.Module): Args: in_channels (int): Number of channels of seed point features. - vote_per_seed (int): Number of votes generated from each seed point. - gt_per_seed (int): Number of ground truth votes generated - from each seed point. - num_points (int): Number of points to be used for voting. - conv_channels (tuple[int]): Out channels of vote - generating convolution. - conv_cfg (dict): Config of convolution. + vote_per_seed (int, optional): Number of votes generated from + each seed point. Default: 1. + gt_per_seed (int, optional): Number of ground truth votes generated + from each seed point. Default: 3. + num_points (int, optional): Number of points to be used for voting. + Default: 1. + conv_channels (tuple[int], optional): Out channels of vote + generating convolution. Default: (16, 16). + conv_cfg (dict, optional): Config of convolution. Default: dict(type='Conv1d'). - norm_cfg (dict): Config of normalization. + norm_cfg (dict, optional): Config of normalization. Default: dict(type='BN1d'). - norm_feats (bool): Whether to normalize features. + norm_feats (bool, optional): Whether to normalize features. Default: True. - with_res_feat (bool): Whether to predict residual features. + with_res_feat (bool, optional): Whether to predict residual features. Default: True. - vote_xyz_range (list[float], None): The range of points translation. - vote_loss (dict): Config of vote loss. + vote_xyz_range (list[float], optional): + The range of points translation. Default: None. + vote_loss (dict, optional): Config of vote loss. Default: None. """ def __init__(self, @@ -94,10 +97,10 @@ def forward(self, seed_points, seed_feats): Returns: tuple[torch.Tensor]: - - vote_points: Voted xyz based on the seed points \ + - vote_points: Voted xyz based on the seed points with shape (B, M, 3), ``M=num_seed*vote_per_seed``. - - vote_features: Voted features based on the seed points with \ - shape (B, C, M) where ``M=num_seed*vote_per_seed``, \ + - vote_features: Voted features based on the seed points with + shape (B, C, M) where ``M=num_seed*vote_per_seed``, ``C=vote_feature_dim``. """ if self.num_points != -1: diff --git a/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py b/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py index 75d40af536..f242e2a5aa 100644 --- a/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py +++ b/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py @@ -323,16 +323,16 @@ def loss(self, Args: bbox_preds (dict): Predictions from forward of h3d bbox head. points (list[torch.Tensor]): Input points. - gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth \ + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each sample. gt_labels_3d (list[torch.Tensor]): Labels of each sample. - pts_semantic_mask (None | list[torch.Tensor]): Point-wise + pts_semantic_mask (list[torch.Tensor]): Point-wise semantic mask. - pts_instance_mask (None | list[torch.Tensor]): Point-wise + pts_instance_mask (list[torch.Tensor]): Point-wise instance mask. img_metas (list[dict]): Contain pcd and img's meta info. rpn_targets (Tuple) : Targets generated by rpn head. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify + gt_bboxes_ignore (list[torch.Tensor]): Specify which bounding. Returns: @@ -501,7 +501,7 @@ def multiclass_nms_single(self, obj_scores, sem_scores, bbox, points, box_dim=bbox.shape[-1], with_yaw=self.bbox_coder.with_rot, origin=(0.5, 0.5, 0.5)) - box_indices = bbox.points_in_boxes_batch(points) + box_indices = bbox.points_in_boxes_all(points) corner3d = bbox.corners minmax_box3d = corner3d.new(torch.Size((corner3d.shape[0], 6))) @@ -559,25 +559,25 @@ def get_proposal_stage_loss(self, Args: bbox_preds (dict): Predictions from forward of vote head. - size_class_targets (torch.Tensor): Ground truth \ + size_class_targets (torch.Tensor): Ground truth size class of each prediction bounding box. - size_res_targets (torch.Tensor): Ground truth \ + size_res_targets (torch.Tensor): Ground truth size residual of each prediction bounding box. - dir_class_targets (torch.Tensor): Ground truth \ + dir_class_targets (torch.Tensor): Ground truth direction class of each prediction bounding box. - dir_res_targets (torch.Tensor): Ground truth \ + dir_res_targets (torch.Tensor): Ground truth direction residual of each prediction bounding box. - center_targets (torch.Tensor): Ground truth center \ + center_targets (torch.Tensor): Ground truth center of each prediction bounding box. - mask_targets (torch.Tensor): Validation of each \ + mask_targets (torch.Tensor): Validation of each prediction bounding box. - objectness_targets (torch.Tensor): Ground truth \ + objectness_targets (torch.Tensor): Ground truth objectness label of each prediction bounding box. - objectness_weights (torch.Tensor): Weights of objectness \ + objectness_weights (torch.Tensor): Weights of objectness loss for each prediction bounding box. - box_loss_weights (torch.Tensor): Weights of regression \ + box_loss_weights (torch.Tensor): Weights of regression loss for each prediction bounding box. - valid_gt_weights (torch.Tensor): Validation of each \ + valid_gt_weights (torch.Tensor): Validation of each ground truth bounding box. Returns: @@ -662,12 +662,12 @@ def get_targets(self, Args: points (list[torch.Tensor]): Points of each batch. - gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth \ + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each batch. gt_labels_3d (list[torch.Tensor]): Labels of each batch. - pts_semantic_mask (None | list[torch.Tensor]): Point-wise semantic + pts_semantic_mask (list[torch.Tensor]): Point-wise semantic label of each batch. - pts_instance_mask (None | list[torch.Tensor]): Point-wise instance + pts_instance_mask (list[torch.Tensor]): Point-wise instance label of each batch. bbox_preds (torch.Tensor): Bounding box predictions of vote head. @@ -768,22 +768,22 @@ def get_targets_single(self, Args: points (torch.Tensor): Points of each batch. - gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth \ + gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth boxes of each batch. gt_labels_3d (torch.Tensor): Labels of each batch. - pts_semantic_mask (None | torch.Tensor): Point-wise semantic + pts_semantic_mask (torch.Tensor): Point-wise semantic label of each batch. - pts_instance_mask (None | torch.Tensor): Point-wise instance + pts_instance_mask (torch.Tensor): Point-wise instance label of each batch. aggregated_points (torch.Tensor): Aggregated points from vote aggregation layer. pred_surface_center (torch.Tensor): Prediction of surface center. pred_line_center (torch.Tensor): Prediction of line center. - pred_obj_surface_center (torch.Tensor): Objectness prediction \ + pred_obj_surface_center (torch.Tensor): Objectness prediction of surface center. - pred_obj_line_center (torch.Tensor): Objectness prediction of \ + pred_obj_line_center (torch.Tensor): Objectness prediction of line center. - pred_surface_sem (torch.Tensor): Semantic prediction of \ + pred_surface_sem (torch.Tensor): Semantic prediction of surface center. pred_line_sem (torch.Tensor): Semantic prediction of line center. Returns: diff --git a/mmdet3d/models/roi_heads/h3d_roi_head.py b/mmdet3d/models/roi_heads/h3d_roi_head.py index 4792083857..c579cbd6cd 100644 --- a/mmdet3d/models/roi_heads/h3d_roi_head.py +++ b/mmdet3d/models/roi_heads/h3d_roi_head.py @@ -64,15 +64,15 @@ def forward_train(self, feats_dict (dict): Contains features from the first stage. img_metas (list[dict]): Contain pcd and img's meta info. points (list[torch.Tensor]): Input points. - gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth \ + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each sample. gt_labels_3d (list[torch.Tensor]): Labels of each sample. - pts_semantic_mask (None | list[torch.Tensor]): Point-wise + pts_semantic_mask (list[torch.Tensor]): Point-wise semantic mask. - pts_instance_mask (None | list[torch.Tensor]): Point-wise + pts_instance_mask (list[torch.Tensor]): Point-wise instance mask. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify - which bounding. + gt_bboxes_ignore (list[torch.Tensor]): Specify + which bounding boxes to ignore. Returns: dict: losses from each head. diff --git a/mmdet3d/models/roi_heads/mask_heads/pointwise_semantic_head.py b/mmdet3d/models/roi_heads/mask_heads/pointwise_semantic_head.py index 6f7d114bde..41e4fae282 100644 --- a/mmdet3d/models/roi_heads/mask_heads/pointwise_semantic_head.py +++ b/mmdet3d/models/roi_heads/mask_heads/pointwise_semantic_head.py @@ -82,15 +82,15 @@ def get_targets_single(self, voxel_centers, gt_bboxes_3d, gt_labels_3d): sample. Args: - voxel_centers (torch.Tensor): The center of voxels in shape \ + voxel_centers (torch.Tensor): The center of voxels in shape (voxel_num, 3). - gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth boxes in \ + gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth boxes in shape (box_num, 7). - gt_labels_3d (torch.Tensor): Class labels of ground truths in \ + gt_labels_3d (torch.Tensor): Class labels of ground truths in shape (box_num). Returns: - tuple[torch.Tensor]: Segmentation targets with shape [voxel_num] \ + tuple[torch.Tensor]: Segmentation targets with shape [voxel_num] part prediction targets with shape [voxel_num, 3] """ gt_bboxes_3d = gt_bboxes_3d.to(voxel_centers.device) @@ -98,8 +98,8 @@ def get_targets_single(self, voxel_centers, gt_bboxes_3d, gt_labels_3d): part_targets = voxel_centers.new_zeros((voxel_centers.shape[0], 3), dtype=torch.float32) - box_idx = gt_bboxes_3d.points_in_boxes(voxel_centers) - enlarge_box_idx = enlarged_gt_boxes.points_in_boxes( + box_idx = gt_bboxes_3d.points_in_boxes_part(voxel_centers) + enlarge_box_idx = enlarged_gt_boxes.points_in_boxes_part( voxel_centers).long() gt_labels_pad = F.pad( @@ -130,19 +130,19 @@ def get_targets(self, voxels_dict, gt_bboxes_3d, gt_labels_3d): """generate segmentation and part prediction targets. Args: - voxel_centers (torch.Tensor): The center of voxels in shape \ + voxel_centers (torch.Tensor): The center of voxels in shape (voxel_num, 3). - gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth boxes in \ + gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth boxes in shape (box_num, 7). - gt_labels_3d (torch.Tensor): Class labels of ground truths in \ + gt_labels_3d (torch.Tensor): Class labels of ground truths in shape (box_num). Returns: dict: Prediction targets - - seg_targets (torch.Tensor): Segmentation targets \ + - seg_targets (torch.Tensor): Segmentation targets with shape [voxel_num]. - - part_targets (torch.Tensor): Part prediction targets \ + - part_targets (torch.Tensor): Part prediction targets with shape [voxel_num, 3]. """ batch_size = len(gt_labels_3d) diff --git a/mmdet3d/models/roi_heads/mask_heads/primitive_head.py b/mmdet3d/models/roi_heads/mask_heads/primitive_head.py index 4ccc7ebbeb..ee95e3c601 100644 --- a/mmdet3d/models/roi_heads/mask_heads/primitive_head.py +++ b/mmdet3d/models/roi_heads/mask_heads/primitive_head.py @@ -197,15 +197,15 @@ def loss(self, Args: bbox_preds (dict): Predictions from forward of primitive head. points (list[torch.Tensor]): Input points. - gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth \ + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each sample. gt_labels_3d (list[torch.Tensor]): Labels of each sample. - pts_semantic_mask (None | list[torch.Tensor]): Point-wise + pts_semantic_mask (list[torch.Tensor]): Point-wise semantic mask. - pts_instance_mask (None | list[torch.Tensor]): Point-wise + pts_instance_mask (list[torch.Tensor]): Point-wise instance mask. img_metas (list[dict]): Contain pcd and img's meta info. - gt_bboxes_ignore (None | list[torch.Tensor]): Specify + gt_bboxes_ignore (list[torch.Tensor]): Specify which bounding. Returns: @@ -265,12 +265,12 @@ def get_targets(self, Args: points (list[torch.Tensor]): Points of each batch. - gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth \ + gt_bboxes_3d (list[:obj:`BaseInstance3DBoxes`]): Ground truth bboxes of each batch. gt_labels_3d (list[torch.Tensor]): Labels of each batch. - pts_semantic_mask (None | list[torch.Tensor]): Point-wise semantic + pts_semantic_mask (list[torch.Tensor]): Point-wise semantic label of each batch. - pts_instance_mask (None | list[torch.Tensor]): Point-wise instance + pts_instance_mask (list[torch.Tensor]): Point-wise instance label of each batch. bbox_preds (dict): Predictions from forward of primitive head. @@ -332,12 +332,12 @@ def get_targets_single(self, Args: points (torch.Tensor): Points of each batch. - gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth \ + gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth boxes of each batch. gt_labels_3d (torch.Tensor): Labels of each batch. - pts_semantic_mask (None | torch.Tensor): Point-wise semantic + pts_semantic_mask (torch.Tensor): Point-wise semantic label of each batch. - pts_instance_mask (None | torch.Tensor): Point-wise instance + pts_instance_mask (torch.Tensor): Point-wise instance label of each batch. Returns: @@ -354,7 +354,7 @@ def get_targets_single(self, # Generate pts_semantic_mask and pts_instance_mask when they are None if pts_semantic_mask is None or pts_instance_mask is None: - points2box_mask = gt_bboxes_3d.points_in_boxes_batch(points) + points2box_mask = gt_bboxes_3d.points_in_boxes_all(points) assignment = points2box_mask.argmax(1) background_mask = points2box_mask.max(1)[0] == 0 diff --git a/mmdet3d/models/segmentors/base.py b/mmdet3d/models/segmentors/base.py index d4fc8917ff..eaa7acd02c 100644 --- a/mmdet3d/models/segmentors/base.py +++ b/mmdet3d/models/segmentors/base.py @@ -77,7 +77,7 @@ def show_results(self, Args: data (list[dict]): Input points and the information of the sample. result (list[dict]): Prediction results. - palette (list[list[int]]] | np.ndarray | None): The palette of + palette (list[list[int]]] | np.ndarray): The palette of segmentation map. If None is given, random palette will be generated. Default: None out_dir (str): Output directory of visualization result. diff --git a/mmdet3d/models/segmentors/encoder_decoder.py b/mmdet3d/models/segmentors/encoder_decoder.py index 353c461861..81ef29392e 100644 --- a/mmdet3d/models/segmentors/encoder_decoder.py +++ b/mmdet3d/models/segmentors/encoder_decoder.py @@ -186,7 +186,7 @@ def _input_generation(coords, use_normalized_coord=False): """Generating model input. - Generate input by subtracting patch center and adding additional \ + Generate input by subtracting patch center and adding additional features. Currently support colors and normalized xyz as features. Args: @@ -194,7 +194,7 @@ def _input_generation(coords, patch_center (torch.Tensor): Center coordinate of the patch. coord_max (torch.Tensor): Max coordinate of all 3D points. feats (torch.Tensor): Features of sampled points of shape [S, C]. - use_normalized_coord (bool, optional): Whether to use normalized \ + use_normalized_coord (bool, optional): Whether to use normalized xyz as additional features. Defaults to False. Returns: @@ -232,17 +232,17 @@ def _sliding_patch_generation(self, block_size (float, optional): Size of a patch to sample. sample_rate (float, optional): Stride used in sliding patch. Defaults to 0.5. - use_normalized_coord (bool, optional): Whether to use normalized \ + use_normalized_coord (bool, optional): Whether to use normalized xyz as additional features. Defaults to False. eps (float, optional): A value added to patch boundary to guarantee - points coverage. Default 1e-3. + points coverage. Defaults to 1e-3. Returns: np.ndarray | np.ndarray: - - patch_points (torch.Tensor): Points of different patches of \ + - patch_points (torch.Tensor): Points of different patches of shape [K, N, 3+C]. - - patch_idxs (torch.Tensor): Index of each point in \ + - patch_idxs (torch.Tensor): Index of each point in `patch_points`, of shape [K, N]. """ device = points.device diff --git a/mmdet3d/models/utils/clip_sigmoid.py b/mmdet3d/models/utils/clip_sigmoid.py index 5182de1139..aaed03f624 100644 --- a/mmdet3d/models/utils/clip_sigmoid.py +++ b/mmdet3d/models/utils/clip_sigmoid.py @@ -6,8 +6,8 @@ def clip_sigmoid(x, eps=1e-4): Args: x (torch.Tensor): Input feature map with the shape of [B, N, H, W]. - eps (float): Lower bound of the range to be clamped to. Defaults - to 1e-4. + eps (float, optional): Lower bound of the range to be clamped to. + Defaults to 1e-4. Returns: torch.Tensor: Feature map after sigmoid. diff --git a/mmdet3d/models/utils/mlp.py b/mmdet3d/models/utils/mlp.py index ca3bf1b67f..48c73facd4 100644 --- a/mmdet3d/models/utils/mlp.py +++ b/mmdet3d/models/utils/mlp.py @@ -9,15 +9,15 @@ class MLP(BaseModule): Pass features (B, C, N) through an MLP. Args: - in_channels (int): Number of channels of input features. + in_channels (int, optional): Number of channels of input features. Default: 18. - conv_channels (tuple[int]): Out channels of the convolution. + conv_channels (tuple[int], optional): Out channels of the convolution. Default: (256, 256). - conv_cfg (dict): Config of convolution. + conv_cfg (dict, optional): Config of convolution. Default: dict(type='Conv1d'). - norm_cfg (dict): Config of normalization. + norm_cfg (dict, optional): Config of normalization. Default: dict(type='BN1d'). - act_cfg (dict): Config of activation. + act_cfg (dict, optional): Config of activation. Default: dict(type='ReLU'). """ diff --git a/mmdet3d/models/voxel_encoders/pillar_encoder.py b/mmdet3d/models/voxel_encoders/pillar_encoder.py index d9666ae313..bfb5c5fe47 100644 --- a/mmdet3d/models/voxel_encoders/pillar_encoder.py +++ b/mmdet3d/models/voxel_encoders/pillar_encoder.py @@ -32,7 +32,7 @@ class PillarFeatureNet(nn.Module): Defaults to dict(type='BN1d', eps=1e-3, momentum=0.01). mode (str, optional): The mode to gather point features. Options are 'max' or 'avg'. Defaults to 'max'. - legacy (bool): Whether to use the new behavior or + legacy (bool, optional): Whether to use the new behavior or the original behavior. Defaults to True. """ diff --git a/mmdet3d/models/voxel_encoders/utils.py b/mmdet3d/models/voxel_encoders/utils.py index 68f8fdd79d..2da7ffaedc 100644 --- a/mmdet3d/models/voxel_encoders/utils.py +++ b/mmdet3d/models/voxel_encoders/utils.py @@ -112,11 +112,12 @@ class PFNLayer(nn.Module): Args: in_channels (int): Number of input channels. out_channels (int): Number of output channels. - norm_cfg (dict): Config dict of normalization layers - last_layer (bool): If last_layer, there is no concatenation of - features. - mode (str): Pooling model to gather features inside voxels. - Default to 'max'. + norm_cfg (dict, optional): Config dict of normalization layers. + Defaults to dict(type='BN1d', eps=1e-3, momentum=0.01). + last_layer (bool, optional): If last_layer, there is no + concatenation of features. Defaults to False. + mode (str, optional): Pooling model to gather features inside voxels. + Defaults to 'max'. """ def __init__(self, diff --git a/mmdet3d/models/voxel_encoders/voxel_encoder.py b/mmdet3d/models/voxel_encoders/voxel_encoder.py index f540d3981b..dbd67127f5 100644 --- a/mmdet3d/models/voxel_encoders/voxel_encoder.py +++ b/mmdet3d/models/voxel_encoders/voxel_encoder.py @@ -16,7 +16,7 @@ class HardSimpleVFE(nn.Module): It simply averages the values of points in a voxel. Args: - num_features (int): Number of features to use. Default: 4. + num_features (int, optional): Number of features to use. Default: 4. """ def __init__(self, num_features=4): @@ -92,25 +92,27 @@ class DynamicVFE(nn.Module): The number of points inside the voxel varies. Args: - in_channels (int): Input channels of VFE. Defaults to 4. - feat_channels (list(int)): Channels of features in VFE. - with_distance (bool): Whether to use the L2 distance of points to the - origin point. Default False. - with_cluster_center (bool): Whether to use the distance to cluster - center of points inside a voxel. Default to False. - with_voxel_center (bool): Whether to use the distance to center of - voxel for each points inside a voxel. Default to False. - voxel_size (tuple[float]): Size of a single voxel. Default to - (0.2, 0.2, 4). - point_cloud_range (tuple[float]): The range of points or voxels. - Default to (0, -40, -3, 70.4, 40, 1). - norm_cfg (dict): Config dict of normalization layers. - mode (str): The mode when pooling features of points inside a voxel. - Available options include 'max' and 'avg'. Default to 'max'. - fusion_layer (dict | None): The config dict of fusion layer used in - multi-modal detectors. Default to None. - return_point_feats (bool): Whether to return the features of each - points. Default to False. + in_channels (int, optional): Input channels of VFE. Defaults to 4. + feat_channels (list(int), optional): Channels of features in VFE. + with_distance (bool, optional): Whether to use the L2 distance of + points to the origin point. Defaults to False. + with_cluster_center (bool, optional): Whether to use the distance + to cluster center of points inside a voxel. Defaults to False. + with_voxel_center (bool, optional): Whether to use the distance + to center of voxel for each points inside a voxel. + Defaults to False. + voxel_size (tuple[float], optional): Size of a single voxel. + Defaults to (0.2, 0.2, 4). + point_cloud_range (tuple[float], optional): The range of points + or voxels. Defaults to (0, -40, -3, 70.4, 40, 1). + norm_cfg (dict, optional): Config dict of normalization layers. + mode (str, optional): The mode when pooling features of points + inside a voxel. Available options include 'max' and 'avg'. + Defaults to 'max'. + fusion_layer (dict, optional): The config dict of fusion + layer used in multi-modal detectors. Defaults to None. + return_point_feats (bool, optional): Whether to return the features + of each points. Defaults to False. """ def __init__(self, @@ -291,25 +293,26 @@ class HardVFE(nn.Module): image feature into voxel features in a point-wise manner. Args: - in_channels (int): Input channels of VFE. Defaults to 4. - feat_channels (list(int)): Channels of features in VFE. - with_distance (bool): Whether to use the L2 distance of points to the - origin point. Default False. - with_cluster_center (bool): Whether to use the distance to cluster - center of points inside a voxel. Default to False. - with_voxel_center (bool): Whether to use the distance to center of - voxel for each points inside a voxel. Default to False. - voxel_size (tuple[float]): Size of a single voxel. Default to - (0.2, 0.2, 4). - point_cloud_range (tuple[float]): The range of points or voxels. - Default to (0, -40, -3, 70.4, 40, 1). - norm_cfg (dict): Config dict of normalization layers. - mode (str): The mode when pooling features of points inside a voxel. - Available options include 'max' and 'avg'. Default to 'max'. - fusion_layer (dict | None): The config dict of fusion layer used in - multi-modal detectors. Default to None. - return_point_feats (bool): Whether to return the features of each - points. Default to False. + in_channels (int, optional): Input channels of VFE. Defaults to 4. + feat_channels (list(int), optional): Channels of features in VFE. + with_distance (bool, optional): Whether to use the L2 distance + of points to the origin point. Defaults to False. + with_cluster_center (bool, optional): Whether to use the distance + to cluster center of points inside a voxel. Defaults to False. + with_voxel_center (bool, optional): Whether to use the distance to + center of voxel for each points inside a voxel. Defaults to False. + voxel_size (tuple[float], optional): Size of a single voxel. + Defaults to (0.2, 0.2, 4). + point_cloud_range (tuple[float], optional): The range of points + or voxels. Defaults to (0, -40, -3, 70.4, 40, 1). + norm_cfg (dict, optional): Config dict of normalization layers. + mode (str, optional): The mode when pooling features of points inside a + voxel. Available options include 'max' and 'avg'. + Defaults to 'max'. + fusion_layer (dict, optional): The config dict of fusion layer + used in multi-modal detectors. Defaults to None. + return_point_feats (bool, optional): Whether to return the + features of each points. Defaults to False. """ def __init__(self, diff --git a/mmdet3d/ops/__init__.py b/mmdet3d/ops/__init__.py index 4ffe650014..8ce119f2e3 100644 --- a/mmdet3d/ops/__init__.py +++ b/mmdet3d/ops/__init__.py @@ -16,8 +16,8 @@ PAConvSAModule, PAConvSAModuleMSG, PointFPModule, PointSAModule, PointSAModuleMSG, build_sa_module) -from .roiaware_pool3d import (RoIAwarePool3d, points_in_boxes_batch, - points_in_boxes_cpu, points_in_boxes_gpu) +from .roiaware_pool3d import (RoIAwarePool3d, points_in_boxes_all, + points_in_boxes_cpu, points_in_boxes_part) from .sparse_block import (SparseBasicBlock, SparseBottleneck, make_sparse_convmodule) from .voxel import DynamicScatter, Voxelization, dynamic_scatter, voxelization @@ -28,12 +28,12 @@ 'NaiveSyncBatchNorm2d', 'batched_nms', 'Voxelization', 'voxelization', 'dynamic_scatter', 'DynamicScatter', 'sigmoid_focal_loss', 'SigmoidFocalLoss', 'SparseBasicBlock', 'SparseBottleneck', - 'RoIAwarePool3d', 'points_in_boxes_gpu', 'points_in_boxes_cpu', + 'RoIAwarePool3d', 'points_in_boxes_part', 'points_in_boxes_cpu', 'make_sparse_convmodule', 'ball_query', 'knn', 'furthest_point_sample', 'furthest_point_sample_with_dist', 'three_interpolate', 'three_nn', 'gather_points', 'grouping_operation', 'group_points', 'GroupAll', 'QueryAndGroup', 'PointSAModule', 'PointSAModuleMSG', 'PointFPModule', - 'points_in_boxes_batch', 'get_compiler_version', 'assign_score_withk', + 'points_in_boxes_all', 'get_compiler_version', 'assign_score_withk', 'get_compiling_cuda_version', 'Points_Sampler', 'build_sa_module', 'PAConv', 'PAConvCUDA', 'PAConvSAModuleMSG', 'PAConvSAModule', 'PAConvCUDASAModule', 'PAConvCUDASAModuleMSG' diff --git a/mmdet3d/ops/furthest_point_sample/points_sampler.py b/mmdet3d/ops/furthest_point_sample/points_sampler.py index 9a3bd2ae42..410d513780 100644 --- a/mmdet3d/ops/furthest_point_sample/points_sampler.py +++ b/mmdet3d/ops/furthest_point_sample/points_sampler.py @@ -36,13 +36,13 @@ class Points_Sampler(nn.Module): Args: num_point (list[int]): Number of sample points. - fps_mod_list (list[str]: Type of FPS method, valid mod + fps_mod_list (list[str], optional): Type of FPS method, valid mod ['F-FPS', 'D-FPS', 'FS'], Default: ['D-FPS']. F-FPS: using feature distances for FPS. D-FPS: using Euclidean distances of points for FPS. FS: using F-FPS and D-FPS simultaneously. - fps_sample_range_list (list[int]): Range of points to apply FPS. - Default: [-1]. + fps_sample_range_list (list[int], optional): + Range of points to apply FPS. Default: [-1]. """ def __init__(self, diff --git a/mmdet3d/ops/furthest_point_sample/utils.py b/mmdet3d/ops/furthest_point_sample/utils.py index 4ca235e13b..8727b9d4e8 100644 --- a/mmdet3d/ops/furthest_point_sample/utils.py +++ b/mmdet3d/ops/furthest_point_sample/utils.py @@ -7,7 +7,7 @@ def calc_square_dist(point_feat_a, point_feat_b, norm=True): Args: point_feat_a (Tensor): (B, N, C) Feature vector of each point. point_feat_b (Tensor): (B, M, C) Feature vector of each point. - norm (Bool): Whether to normalize the distance. + norm (Bool, optional): Whether to normalize the distance. Default: True. Returns: diff --git a/mmdet3d/ops/group_points/group_points.py b/mmdet3d/ops/group_points/group_points.py index 88122a88d5..07fd143901 100644 --- a/mmdet3d/ops/group_points/group_points.py +++ b/mmdet3d/ops/group_points/group_points.py @@ -14,22 +14,22 @@ class QueryAndGroup(nn.Module): Groups with a ball query of radius Args: - max_radius (float | None): The maximum radius of the balls. + max_radius (float): The maximum radius of the balls. If None is given, we will use kNN sampling instead of ball query. sample_num (int): Maximum number of features to gather in the ball. - min_radius (float): The minimum radius of the balls. - use_xyz (bool): Whether to use xyz. + min_radius (float, optional): The minimum radius of the balls. + Default: 0. + use_xyz (bool, optional): Whether to use xyz. Default: True. - return_grouped_xyz (bool): Whether to return grouped xyz. + return_grouped_xyz (bool, optional): Whether to return grouped xyz. Default: False. - normalize_xyz (bool): Whether to normalize xyz. + normalize_xyz (bool, optional): Whether to normalize xyz. Default: False. - uniform_sample (bool): Whether to sample uniformly. + uniform_sample (bool, optional): Whether to sample uniformly. Default: False - return_unique_cnt (bool): Whether to return the count of - unique samples. - Default: False. - return_grouped_idx (bool): Whether to return grouped idx. + return_unique_cnt (bool, optional): Whether to return the count of + unique samples. Default: False. + return_grouped_idx (bool, optional): Whether to return grouped idx. Default: False. """ diff --git a/mmdet3d/ops/iou3d/iou3d_utils.py b/mmdet3d/ops/iou3d/iou3d_utils.py index 6f36019e72..09bb33cdd6 100644 --- a/mmdet3d/ops/iou3d/iou3d_utils.py +++ b/mmdet3d/ops/iou3d/iou3d_utils.py @@ -4,7 +4,7 @@ def boxes_iou_bev(boxes_a, boxes_b): - """Calculate boxes IoU in the bird view. + """Calculate boxes IoU in the Bird's Eye View. Args: boxes_a (torch.Tensor): Input boxes a with shape (M, 5). @@ -22,24 +22,29 @@ def boxes_iou_bev(boxes_a, boxes_b): return ans_iou -def nms_gpu(boxes, scores, thresh, pre_maxsize=None, post_max_size=None): - """Nms function with gpu implementation. +def nms_gpu(boxes, scores, thresh, pre_max_size=None, post_max_size=None): + """NMS function GPU implementation (for BEV boxes). The overlap of two + boxes for IoU calculation is defined as the exact overlapping area of the + two boxes. In this function, one can also set `pre_max_size` and + `post_max_size`. Args: boxes (torch.Tensor): Input boxes with the shape of [N, 5] ([x1, y1, x2, y2, ry]). scores (torch.Tensor): Scores of boxes with the shape of [N]. thresh (int): Threshold. - pre_maxsize (int): Max size of boxes before nms. Default: None. - post_maxsize (int): Max size of boxes after nms. Default: None. + pre_max_size (int, optional): Max size of boxes before NMS. + Default: None. + post_max_size (int, optional): Max size of boxes after NMS. + Default: None. Returns: - torch.Tensor: Indexes after nms. + torch.Tensor: Indexes after NMS. """ order = scores.sort(0, descending=True)[1] - if pre_maxsize is not None: - order = order[:pre_maxsize] + if pre_max_size is not None: + order = order[:pre_max_size] boxes = boxes[order].contiguous() keep = torch.zeros(boxes.size(0), dtype=torch.long) @@ -51,12 +56,14 @@ def nms_gpu(boxes, scores, thresh, pre_maxsize=None, post_max_size=None): def nms_normal_gpu(boxes, scores, thresh): - """Normal non maximum suppression on GPU. + """Normal NMS function GPU implementation (for BEV boxes). The overlap of + two boxes for IoU calculation is defined as the exact overlapping area of + the two boxes WITH their yaw angle set to 0. Args: boxes (torch.Tensor): Input boxes with shape (N, 5). scores (torch.Tensor): Scores of predicted boxes with shape (N). - thresh (torch.Tensor): Threshold of non maximum suppression. + thresh (torch.Tensor): Threshold of NMS. Returns: torch.Tensor: Remaining indices with scores in descending order. diff --git a/mmdet3d/ops/pointnet_modules/paconv_sa_module.py b/mmdet3d/ops/pointnet_modules/paconv_sa_module.py index 4d5ac218f0..19040504b8 100644 --- a/mmdet3d/ops/pointnet_modules/paconv_sa_module.py +++ b/mmdet3d/ops/pointnet_modules/paconv_sa_module.py @@ -238,11 +238,12 @@ def forward( Args: points_xyz (Tensor): (B, N, 3) xyz coordinates of the features. - features (Tensor): (B, C, N) features of each point. + features (Tensor, optional): (B, C, N) features of each point. Default: None. - indices (Tensor): (B, num_point) Index of the features. + indices (Tensor, optional): (B, num_point) Index of the features. + Default: None. + target_xyz (Tensor, optional): (B, M, 3) new coords of the outputs. Default: None. - target_xyz (Tensor): (B, M, 3) new_xyz coordinates of the outputs. Returns: Tensor: (B, M, 3) where M is the number of points. diff --git a/mmdet3d/ops/pointnet_modules/point_fp_module.py b/mmdet3d/ops/pointnet_modules/point_fp_module.py index 212705baf7..59161d14fa 100644 --- a/mmdet3d/ops/pointnet_modules/point_fp_module.py +++ b/mmdet3d/ops/pointnet_modules/point_fp_module.py @@ -14,7 +14,7 @@ class PointFPModule(BaseModule): Args: mlp_channels (list[int]): List of mlp channels. - norm_cfg (dict): Type of normalization method. + norm_cfg (dict, optional): Type of normalization method. Default: dict(type='BN2d'). """ diff --git a/mmdet3d/ops/pointnet_modules/point_sa_module.py b/mmdet3d/ops/pointnet_modules/point_sa_module.py index 000ecb4fd4..f640b3d386 100644 --- a/mmdet3d/ops/pointnet_modules/point_sa_module.py +++ b/mmdet3d/ops/pointnet_modules/point_sa_module.py @@ -17,25 +17,25 @@ class BasePointSAModule(nn.Module): sample_nums (list[int]): Number of samples in each ball query. mlp_channels (list[list[int]]): Specify of the pointnet before the global pooling for each scale. - fps_mod (list[str]: Type of FPS method, valid mod + fps_mod (list[str], optional): Type of FPS method, valid mod ['F-FPS', 'D-FPS', 'FS'], Default: ['D-FPS']. F-FPS: using feature distances for FPS. D-FPS: using Euclidean distances of points for FPS. FS: using F-FPS and D-FPS simultaneously. - fps_sample_range_list (list[int]): Range of points to apply FPS. - Default: [-1]. - dilated_group (bool): Whether to use dilated ball query. + fps_sample_range_list (list[int], optional): + Range of points to apply FPS. Default: [-1]. + dilated_group (bool, optional): Whether to use dilated ball query. Default: False. - use_xyz (bool): Whether to use xyz. + use_xyz (bool, optional): Whether to use xyz. Default: True. - pool_mod (str): Type of pooling method. + pool_mod (str, optional): Type of pooling method. Default: 'max_pool'. - normalize_xyz (bool): Whether to normalize local XYZ with radius. - Default: False. - grouper_return_grouped_xyz (bool): Whether to return grouped xyz in - `QueryAndGroup`. Defaults to False. - grouper_return_grouped_idx (bool): Whether to return grouped idx in - `QueryAndGroup`. Defaults to False. + normalize_xyz (bool, optional): Whether to normalize local XYZ + with radius. Default: False. + grouper_return_grouped_xyz (bool, optional): Whether to return + grouped xyz in `QueryAndGroup`. Defaults to False. + grouper_return_grouped_idx (bool, optional): Whether to return + grouped idx in `QueryAndGroup`. Defaults to False. """ def __init__(self, @@ -110,9 +110,7 @@ def _sample_points(self, points_xyz, features, indices, target_xyz): Args: points_xyz (Tensor): (B, N, 3) xyz coordinates of the features. features (Tensor): (B, C, N) features of each point. - Default: None. indices (Tensor): (B, num_point) Index of the features. - Default: None. target_xyz (Tensor): (B, M, 3) new_xyz coordinates of the outputs. Returns: @@ -168,11 +166,12 @@ def forward( Args: points_xyz (Tensor): (B, N, 3) xyz coordinates of the features. - features (Tensor): (B, C, N) features of each point. + features (Tensor, optional): (B, C, N) features of each point. Default: None. - indices (Tensor): (B, num_point) Index of the features. + indices (Tensor, optional): (B, num_point) Index of the features. + Default: None. + target_xyz (Tensor, optional): (B, M, 3) new coords of the outputs. Default: None. - target_xyz (Tensor): (B, M, 3) new_xyz coordinates of the outputs. Returns: Tensor: (B, M, 3) where M is the number of points. @@ -222,26 +221,26 @@ class PointSAModuleMSG(BasePointSAModule): sample_nums (list[int]): Number of samples in each ball query. mlp_channels (list[list[int]]): Specify of the pointnet before the global pooling for each scale. - fps_mod (list[str]: Type of FPS method, valid mod + fps_mod (list[str], optional): Type of FPS method, valid mod ['F-FPS', 'D-FPS', 'FS'], Default: ['D-FPS']. F-FPS: using feature distances for FPS. D-FPS: using Euclidean distances of points for FPS. FS: using F-FPS and D-FPS simultaneously. - fps_sample_range_list (list[int]): Range of points to apply FPS. - Default: [-1]. - dilated_group (bool): Whether to use dilated ball query. + fps_sample_range_list (list[int], optional): Range of points to + apply FPS. Default: [-1]. + dilated_group (bool, optional): Whether to use dilated ball query. Default: False. - norm_cfg (dict): Type of normalization method. + norm_cfg (dict, optional): Type of normalization method. Default: dict(type='BN2d'). - use_xyz (bool): Whether to use xyz. + use_xyz (bool, optional): Whether to use xyz. Default: True. - pool_mod (str): Type of pooling method. + pool_mod (str, optional): Type of pooling method. Default: 'max_pool'. - normalize_xyz (bool): Whether to normalize local XYZ with radius. - Default: False. - bias (bool | str): If specified as `auto`, it will be decided by the - norm_cfg. Bias will be set as True if `norm_cfg` is None, otherwise - False. Default: "auto". + normalize_xyz (bool, optional): Whether to normalize local XYZ + with radius. Default: False. + bias (bool | str, optional): If specified as `auto`, it will be + decided by `norm_cfg`. `bias` will be set as True if + `norm_cfg` is None, otherwise False. Default: 'auto'. """ def __init__(self, @@ -297,24 +296,24 @@ class PointSAModule(PointSAModuleMSG): Args: mlp_channels (list[int]): Specify of the pointnet before the global pooling for each scale. - num_point (int): Number of points. + num_point (int, optional): Number of points. Default: None. - radius (float): Radius to group with. + radius (float, optional): Radius to group with. Default: None. - num_sample (int): Number of samples in each ball query. + num_sample (int, optional): Number of samples in each ball query. Default: None. - norm_cfg (dict): Type of normalization method. + norm_cfg (dict, optional): Type of normalization method. Default: dict(type='BN2d'). - use_xyz (bool): Whether to use xyz. + use_xyz (bool, optional): Whether to use xyz. Default: True. - pool_mod (str): Type of pooling method. + pool_mod (str, optional): Type of pooling method. Default: 'max_pool'. - fps_mod (list[str]: Type of FPS method, valid mod + fps_mod (list[str], optional): Type of FPS method, valid mod ['F-FPS', 'D-FPS', 'FS'], Default: ['D-FPS']. - fps_sample_range_list (list[int]): Range of points to apply FPS. - Default: [-1]. - normalize_xyz (bool): Whether to normalize local XYZ with radius. - Default: False. + fps_sample_range_list (list[int], optional): Range of points + to apply FPS. Default: [-1]. + normalize_xyz (bool, optional): Whether to normalize local XYZ + with radius. Default: False. """ def __init__(self, diff --git a/mmdet3d/ops/roiaware_pool3d/__init__.py b/mmdet3d/ops/roiaware_pool3d/__init__.py index aba9e18d37..aaa29eb3ea 100644 --- a/mmdet3d/ops/roiaware_pool3d/__init__.py +++ b/mmdet3d/ops/roiaware_pool3d/__init__.py @@ -1,8 +1,8 @@ -from .points_in_boxes import (points_in_boxes_batch, points_in_boxes_cpu, - points_in_boxes_gpu) +from .points_in_boxes import (points_in_boxes_all, points_in_boxes_cpu, + points_in_boxes_part) from .roiaware_pool3d import RoIAwarePool3d __all__ = [ - 'RoIAwarePool3d', 'points_in_boxes_gpu', 'points_in_boxes_cpu', - 'points_in_boxes_batch' + 'RoIAwarePool3d', 'points_in_boxes_part', 'points_in_boxes_cpu', + 'points_in_boxes_all' ] diff --git a/mmdet3d/ops/roiaware_pool3d/points_in_boxes.py b/mmdet3d/ops/roiaware_pool3d/points_in_boxes.py index 14e16b9926..1240c20d30 100644 --- a/mmdet3d/ops/roiaware_pool3d/points_in_boxes.py +++ b/mmdet3d/ops/roiaware_pool3d/points_in_boxes.py @@ -3,13 +3,13 @@ from . import roiaware_pool3d_ext -def points_in_boxes_gpu(points, boxes): - """Find points that are in boxes (CUDA) +def points_in_boxes_part(points, boxes): + """Find the box in which each point is (CUDA). Args: points (torch.Tensor): [B, M, 3], [x, y, z] in LiDAR/DEPTH coordinate boxes (torch.Tensor): [B, T, 7], - num_valid_boxes <= T, [x, y, z, dx, dy, dz, rz] in + num_valid_boxes <= T, [x, y, z, x_size, y_size, z_size, rz] in LiDAR/DEPTH coordinate, (x, y, z) is the bottom center Returns: @@ -43,25 +43,26 @@ def points_in_boxes_gpu(points, boxes): if torch.cuda.current_device() != points_device: torch.cuda.set_device(points_device) - roiaware_pool3d_ext.points_in_boxes_gpu(boxes.contiguous(), - points.contiguous(), - box_idxs_of_pts) + roiaware_pool3d_ext.points_in_boxes_part(boxes.contiguous(), + points.contiguous(), + box_idxs_of_pts) return box_idxs_of_pts def points_in_boxes_cpu(points, boxes): - """Find points that are in boxes (CPU) + """Find all boxes in which each point is (CPU). The CPU version of + :meth:`points_in_boxes_all`. Args: points (torch.Tensor): [B, M, 3], [x, y, z] in LiDAR/DEPTH coordinate boxes (torch.Tensor): [B, T, 7], - num_valid_boxes <= T, [x, y, z, dx, dy, dz, rz], + num_valid_boxes <= T, [x, y, z, x_size, y_size, z_size, rz], (x, y, z) is the bottom center. Returns: - box_idxs_of_pts (torch.Tensor): (B, M, T), default background = 0 + box_idxs_of_pts (torch.Tensor): (B, M, T), default background = 0. """ assert points.shape[0] == boxes.shape[0], \ f'Points and boxes should have the same batch size, ' \ @@ -86,17 +87,17 @@ def points_in_boxes_cpu(points, boxes): return point_indices -def points_in_boxes_batch(points, boxes): - """Find points that are in boxes (CUDA) +def points_in_boxes_all(points, boxes): + """Find all boxes in which each point is (CUDA). Args: points (torch.Tensor): [B, M, 3], [x, y, z] in LiDAR/DEPTH coordinate boxes (torch.Tensor): [B, T, 7], - num_valid_boxes <= T, [x, y, z, dx, dy, dz, rz], + num_valid_boxes <= T, [x, y, z, x_size, y_size, z_size, rz], (x, y, z) is the bottom center. Returns: - box_idxs_of_pts (torch.Tensor): (B, M, T), default background = 0 + box_idxs_of_pts (torch.Tensor): (B, M, T), default background = 0. """ assert boxes.shape[0] == points.shape[0], \ f'Points and boxes should have the same batch size, ' \ @@ -120,8 +121,8 @@ def points_in_boxes_batch(points, boxes): if torch.cuda.current_device() != points_device: torch.cuda.set_device(points_device) - roiaware_pool3d_ext.points_in_boxes_batch(boxes.contiguous(), - points.contiguous(), - box_idxs_of_pts) + roiaware_pool3d_ext.points_in_boxes_all(boxes.contiguous(), + points.contiguous(), + box_idxs_of_pts) return box_idxs_of_pts diff --git a/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cpu.cpp b/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cpu.cpp index 7e5956b67e..f8c5494d2e 100644 --- a/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cpu.cpp +++ b/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cpu.cpp @@ -23,23 +23,23 @@ inline void lidar_to_local_coords_cpu(float shift_x, float shift_y, float rz, inline int check_pt_in_box3d_cpu(const float *pt, const float *box3d, float &local_x, float &local_y) { // param pt: (x, y, z) - // param box3d: (cx, cy, cz, w, l, h, rz) in LiDAR coordinate, cz in the + // param box3d: (cx, cy, cz, x_size, y_size, z_size, rz) in LiDAR coordinate, cz in the // bottom center float x = pt[0], y = pt[1], z = pt[2]; float cx = box3d[0], cy = box3d[1], cz = box3d[2]; - float dx = box3d[3], dy = box3d[4], dz = box3d[5], rz = box3d[6]; - cz += dz / 2.0; // shift to the center since cz in box3d is the bottom center + float x_size = box3d[3], y_size = box3d[4], z_size = box3d[5], rz = box3d[6]; + cz += z_size / 2.0; // shift to the center since cz in box3d is the bottom center - if (fabsf(z - cz) > dz / 2.0) return 0; + if (fabsf(z - cz) > z_size / 2.0) return 0; lidar_to_local_coords_cpu(x - cx, y - cy, rz, local_x, local_y); - float in_flag = (local_x > -dx / 2.0) & (local_x < dx / 2.0) & - (local_y > -dy / 2.0) & (local_y < dy / 2.0); + float in_flag = (local_x > -x_size / 2.0) & (local_x < x_size / 2.0) & + (local_y > -y_size / 2.0) & (local_y < y_size / 2.0); return in_flag; } int points_in_boxes_cpu(at::Tensor boxes_tensor, at::Tensor pts_tensor, at::Tensor pts_indices_tensor) { - // params boxes: (N, 7) [x, y, z, w, l, h, rz] in LiDAR coordinate, z is the + // params boxes: (N, 7) [x, y, z, x_size, y_size, z_size, rz] in LiDAR coordinate, z is the // bottom center, each box DO NOT overlaps params pts: (npoints, 3) [x, y, z] // in LiDAR coordinate params pts_indices: (N, npoints) diff --git a/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cuda.cu b/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cuda.cu index 4fed2002f1..4b90897e3a 100644 --- a/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cuda.cu +++ b/mmdet3d/ops/roiaware_pool3d/src/points_in_boxes_cuda.cu @@ -32,25 +32,25 @@ __device__ inline void lidar_to_local_coords(float shift_x, float shift_y, __device__ inline int check_pt_in_box3d(const float *pt, const float *box3d, float &local_x, float &local_y) { // param pt: (x, y, z) - // param box3d: (cx, cy, cz, w, l, h, rz) in LiDAR coordinate, cz in the + // param box3d: (cx, cy, cz, x_size, y_size, z_size, rz) in LiDAR coordinate, cz in the // bottom center float x = pt[0], y = pt[1], z = pt[2]; float cx = box3d[0], cy = box3d[1], cz = box3d[2]; - float dx = box3d[3], dy = box3d[4], dz = box3d[5], rz = box3d[6]; - cz += dz / 2.0; // shift to the center since cz in box3d is the bottom center + float x_size = box3d[3], y_size = box3d[4], z_size = box3d[5], rz = box3d[6]; + cz += z_size / 2.0; // shift to the center since cz in box3d is the bottom center - if (fabsf(z - cz) > dz / 2.0) return 0; + if (fabsf(z - cz) > z_size / 2.0) return 0; lidar_to_local_coords(x - cx, y - cy, rz, local_x, local_y); - float in_flag = (local_x > -dx / 2.0) & (local_x < dx / 2.0) & - (local_y > -dy / 2.0) & (local_y < dy / 2.0); + float in_flag = (local_x > -x_size / 2.0) & (local_x < x_size / 2.0) & + (local_y > -y_size / 2.0) & (local_y < y_size / 2.0); return in_flag; } -__global__ void points_in_boxes_kernel(int batch_size, int boxes_num, - int pts_num, const float *boxes, - const float *pts, - int *box_idx_of_points) { - // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is +__global__ void points_in_boxes_part_kernel(int batch_size, int boxes_num, + int pts_num, const float *boxes, + const float *pts, + int *box_idx_of_points) { + // params boxes: (B, N, 7) [x, y, z, x_size, y_size, z_size, rz] in LiDAR coordinate, z is // the bottom center, each box DO NOT overlaps params pts: (B, npoints, 3) [x, // y, z] in LiDAR coordinate params boxes_idx_of_points: (B, npoints), default // -1 @@ -74,11 +74,11 @@ __global__ void points_in_boxes_kernel(int batch_size, int boxes_num, } } -__global__ void points_in_boxes_batch_kernel(int batch_size, int boxes_num, - int pts_num, const float *boxes, - const float *pts, - int *box_idx_of_points) { - // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is +__global__ void points_in_boxes_all_kernel(int batch_size, int boxes_num, + int pts_num, const float *boxes, + const float *pts, + int *box_idx_of_points) { + // params boxes: (B, N, 7) [x, y, z, x_size, y_size, z_size, rz] in LiDAR coordinate, z is // the bottom center, each box DO NOT overlaps params pts: (B, npoints, 3) [x, // y, z] in LiDAR coordinate params boxes_idx_of_points: (B, npoints), default // -1 @@ -102,10 +102,10 @@ __global__ void points_in_boxes_batch_kernel(int batch_size, int boxes_num, } } -void points_in_boxes_launcher(int batch_size, int boxes_num, int pts_num, - const float *boxes, const float *pts, - int *box_idx_of_points) { - // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is +void points_in_boxes_part_launcher(int batch_size, int boxes_num, int pts_num, + const float *boxes, const float *pts, + int *box_idx_of_points) { + // params boxes: (B, N, 7) [x, y, z, x_size, y_size, z_size, rz] in LiDAR coordinate, z is // the bottom center, each box DO NOT overlaps params pts: (B, npoints, 3) [x, // y, z] in LiDAR coordinate params boxes_idx_of_points: (B, npoints), default // -1 @@ -113,8 +113,8 @@ void points_in_boxes_launcher(int batch_size, int boxes_num, int pts_num, dim3 blocks(DIVUP(pts_num, THREADS_PER_BLOCK), batch_size); dim3 threads(THREADS_PER_BLOCK); - points_in_boxes_kernel<<>>(batch_size, boxes_num, pts_num, - boxes, pts, box_idx_of_points); + points_in_boxes_part_kernel<<>>(batch_size, boxes_num, pts_num, + boxes, pts, box_idx_of_points); err = cudaGetLastError(); if (cudaSuccess != err) { @@ -127,17 +127,17 @@ void points_in_boxes_launcher(int batch_size, int boxes_num, int pts_num, #endif } -void points_in_boxes_batch_launcher(int batch_size, int boxes_num, int pts_num, - const float *boxes, const float *pts, - int *box_idx_of_points) { - // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is +void points_in_boxes_all_launcher(int batch_size, int boxes_num, int pts_num, + const float *boxes, const float *pts, + int *box_idx_of_points) { + // params boxes: (B, N, 7) [x, y, z, x_size, y_size, z_size, rz] in LiDAR coordinate, z is // the bottom center, each box params pts: (B, npoints, 3) [x, y, z] in // LiDAR coordinate params boxes_idx_of_points: (B, npoints), default -1 cudaError_t err; dim3 blocks(DIVUP(pts_num, THREADS_PER_BLOCK), batch_size); dim3 threads(THREADS_PER_BLOCK); - points_in_boxes_batch_kernel<<>>( + points_in_boxes_all_kernel<<>>( batch_size, boxes_num, pts_num, boxes, pts, box_idx_of_points); err = cudaGetLastError(); @@ -151,9 +151,9 @@ void points_in_boxes_batch_launcher(int batch_size, int boxes_num, int pts_num, #endif } -int points_in_boxes_gpu(at::Tensor boxes_tensor, at::Tensor pts_tensor, - at::Tensor box_idx_of_points_tensor) { - // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is +int points_in_boxes_part(at::Tensor boxes_tensor, at::Tensor pts_tensor, + at::Tensor box_idx_of_points_tensor) { + // params boxes: (B, N, 7) [x, y, z, x_size, y_size, z_size, rz] in LiDAR coordinate, z is // the bottom center, each box DO NOT overlaps params pts: (B, npoints, 3) [x, // y, z] in LiDAR coordinate params boxes_idx_of_points: (B, npoints), default // -1 @@ -170,15 +170,15 @@ int points_in_boxes_gpu(at::Tensor boxes_tensor, at::Tensor pts_tensor, const float *pts = pts_tensor.data_ptr(); int *box_idx_of_points = box_idx_of_points_tensor.data_ptr(); - points_in_boxes_launcher(batch_size, boxes_num, pts_num, boxes, pts, - box_idx_of_points); + points_in_boxes_part_launcher(batch_size, boxes_num, pts_num, boxes, pts, + box_idx_of_points); return 1; } -int points_in_boxes_batch(at::Tensor boxes_tensor, at::Tensor pts_tensor, - at::Tensor box_idx_of_points_tensor) { - // params boxes: (B, N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate, z is +int points_in_boxes_all(at::Tensor boxes_tensor, at::Tensor pts_tensor, + at::Tensor box_idx_of_points_tensor) { + // params boxes: (B, N, 7) [x, y, z, x_size, y_size, z_size, rz] in LiDAR coordinate, z is // the bottom center. params pts: (B, npoints, 3) [x, y, z] in LiDAR // coordinate params boxes_idx_of_points: (B, npoints), default -1 @@ -194,8 +194,8 @@ int points_in_boxes_batch(at::Tensor boxes_tensor, at::Tensor pts_tensor, const float *pts = pts_tensor.data_ptr(); int *box_idx_of_points = box_idx_of_points_tensor.data_ptr(); - points_in_boxes_batch_launcher(batch_size, boxes_num, pts_num, boxes, pts, - box_idx_of_points); + points_in_boxes_all_launcher(batch_size, boxes_num, pts_num, boxes, pts, + box_idx_of_points); return 1; } diff --git a/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d.cpp b/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d.cpp index cd743b18bb..607d783eb5 100644 --- a/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d.cpp +++ b/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d.cpp @@ -40,16 +40,16 @@ int roiaware_pool3d_gpu_backward(at::Tensor pts_idx_of_voxels, int points_in_boxes_cpu(at::Tensor boxes_tensor, at::Tensor pts_tensor, at::Tensor pts_indices_tensor); -int points_in_boxes_gpu(at::Tensor boxes_tensor, at::Tensor pts_tensor, - at::Tensor box_idx_of_points_tensor); +int points_in_boxes_part(at::Tensor boxes_tensor, at::Tensor pts_tensor, + at::Tensor box_idx_of_points_tensor); -int points_in_boxes_batch(at::Tensor boxes_tensor, at::Tensor pts_tensor, - at::Tensor box_idx_of_points_tensor); +int points_in_boxes_all(at::Tensor boxes_tensor, at::Tensor pts_tensor, + at::Tensor box_idx_of_points_tensor); int roiaware_pool3d_gpu(at::Tensor rois, at::Tensor pts, at::Tensor pts_feature, at::Tensor argmax, at::Tensor pts_idx_of_voxels, at::Tensor pooled_features, int pool_method) { - // params rois: (N, 7) [x, y, z, w, l, h, ry] in LiDAR coordinate + // params rois: (N, 7) [x, y, z, x_size, y_size, z_size, ry] in LiDAR coordinate // params pts: (npoints, 3) [x, y, z] in LiDAR coordinate // params pts_feature: (npoints, C) // params argmax: (N, out_x, out_y, out_z, C) @@ -127,10 +127,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("forward", &roiaware_pool3d_gpu, "roiaware pool3d forward (CUDA)"); m.def("backward", &roiaware_pool3d_gpu_backward, "roiaware pool3d backward (CUDA)"); - m.def("points_in_boxes_gpu", &points_in_boxes_gpu, - "points_in_boxes_gpu forward (CUDA)"); - m.def("points_in_boxes_batch", &points_in_boxes_batch, - "points_in_boxes_batch forward (CUDA)"); + m.def("points_in_boxes_part", &points_in_boxes_part, + "points_in_boxes_part forward (CUDA)"); + m.def("points_in_boxes_all", &points_in_boxes_all, + "points_in_boxes_all forward (CUDA)"); m.def("points_in_boxes_cpu", &points_in_boxes_cpu, "points_in_boxes_cpu forward (CPU)"); } diff --git a/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d_kernel.cu b/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d_kernel.cu index c1c948e96a..8f62e891de 100644 --- a/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d_kernel.cu +++ b/mmdet3d/ops/roiaware_pool3d/src/roiaware_pool3d_kernel.cu @@ -25,17 +25,17 @@ __device__ inline void lidar_to_local_coords(float shift_x, float shift_y, __device__ inline int check_pt_in_box3d(const float *pt, const float *box3d, float &local_x, float &local_y) { // param pt: (x, y, z) - // param box3d: (cx, cy, cz, dx, dy, dz, rz) in LiDAR coordinate, cz in the + // param box3d: (cx, cy, cz, x_size, y_size, z_size, rz) in LiDAR coordinate, cz in the // bottom center float x = pt[0], y = pt[1], z = pt[2]; float cx = box3d[0], cy = box3d[1], cz = box3d[2]; - float dx = box3d[3], dy = box3d[4], dz = box3d[5], rz = box3d[6]; - cz += dz / 2.0; // shift to the center since cz in box3d is the bottom center + float x_size = box3d[3], y_size = box3d[4], z_size = box3d[5], rz = box3d[6]; + cz += z_size / 2.0; // shift to the center since cz in box3d is the bottom center - if (fabsf(z - cz) > dz / 2.0) return 0; + if (fabsf(z - cz) > z_size / 2.0) return 0; lidar_to_local_coords(x - cx, y - cy, rz, local_x, local_y); - float in_flag = (local_x > -dx / 2.0) & (local_x < dx / 2.0) & - (local_y > -dy / 2.0) & (local_y < dy / 2.0); + float in_flag = (local_x > -x_size / 2.0) & (local_x < x_size / 2.0) & + (local_y > -y_size / 2.0) & (local_y < y_size / 2.0); return in_flag; } @@ -43,7 +43,7 @@ __global__ void generate_pts_mask_for_box3d(int boxes_num, int pts_num, int out_x, int out_y, int out_z, const float *rois, const float *pts, int *pts_mask) { - // params rois: (N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate + // params rois: (N, 7) [x, y, z, x_size, y_size, z_size, rz] in LiDAR coordinate // params pts: (npoints, 3) [x, y, z] // params pts_mask: (N, npoints): -1 means point does not in this box, // otherwise: encode (x_idxs, y_idxs, z_idxs) by binary bit @@ -61,14 +61,14 @@ __global__ void generate_pts_mask_for_box3d(int boxes_num, int pts_num, pts_mask[0] = -1; if (cur_in_flag > 0) { float local_z = pts[2] - rois[2]; - float dx = rois[3], dy = rois[4], dz = rois[5]; + float x_size = rois[3], y_size = rois[4], z_size = rois[5]; - float x_res = dx / out_x; - float y_res = dy / out_y; - float z_res = dz / out_z; + float x_res = x_size / out_x; + float y_res = y_size / out_y; + float z_res = z_size / out_z; - unsigned int x_idx = int((local_x + dx / 2) / x_res); - unsigned int y_idx = int((local_y + dy / 2) / y_res); + unsigned int x_idx = int((local_x + x_size / 2) / x_res); + unsigned int y_idx = int((local_y + y_size / 2) / y_res); unsigned int z_idx = int(local_z / z_res); x_idx = min(max(x_idx, 0), out_x - 1); @@ -229,7 +229,7 @@ void roiaware_pool3d_launcher(int boxes_num, int pts_num, int channels, const float *pts_feature, int *argmax, int *pts_idx_of_voxels, float *pooled_features, int pool_method) { - // params rois: (N, 7) [x, y, z, dx, dy, dz, rz] in LiDAR coordinate + // params rois: (N, 7) [x, y, z, x_size, y_size, z_size, rz] in LiDAR coordinate // params pts: (npoints, 3) [x, y, z] in LiDAR coordinate // params pts_feature: (npoints, C) // params argmax: (N, out_x, out_y, out_z, C) diff --git a/mmdet3d/ops/sparse_block.py b/mmdet3d/ops/sparse_block.py index 37c65989b3..c89cd53db5 100644 --- a/mmdet3d/ops/sparse_block.py +++ b/mmdet3d/ops/sparse_block.py @@ -13,12 +13,12 @@ class SparseBottleneck(Bottleneck, spconv.SparseModule): Args: inplanes (int): inplanes of block. planes (int): planes of block. - stride (int): stride of the first block. Default: 1 - downsample (None | Module): down sample module for block. - conv_cfg (dict): dictionary to construct and config conv layer. - Default: None - norm_cfg (dict): dictionary to construct and config norm layer. - Default: dict(type='BN') + stride (int, optional): stride of the first block. Default: 1. + downsample (Module, optional): down sample module for block. + conv_cfg (dict, optional): dictionary to construct and config conv + layer. Default: None. + norm_cfg (dict, optional): dictionary to construct and config norm + layer. Default: dict(type='BN'). """ expansion = 4 @@ -72,12 +72,12 @@ class SparseBasicBlock(BasicBlock, spconv.SparseModule): Args: inplanes (int): inplanes of block. planes (int): planes of block. - stride (int): stride of the first block. Default: 1 - downsample (None | Module): down sample module for block. - conv_cfg (dict): dictionary to construct and config conv layer. - Default: None - norm_cfg (dict): dictionary to construct and config norm layer. - Default: dict(type='BN') + stride (int, optional): stride of the first block. Default: 1. + downsample (Module, optional): down sample module for block. + conv_cfg (dict, optional): dictionary to construct and config conv + layer. Default: None. + norm_cfg (dict, optional): dictionary to construct and config norm + layer. Default: dict(type='BN'). """ expansion = 1 diff --git a/tests/test_models/test_common_modules/test_roiaware_pool3d.py b/tests/test_models/test_common_modules/test_roiaware_pool3d.py index c005be6a1f..c6d104cd35 100644 --- a/tests/test_models/test_common_modules/test_roiaware_pool3d.py +++ b/tests/test_models/test_common_modules/test_roiaware_pool3d.py @@ -2,9 +2,9 @@ import pytest import torch -from mmdet3d.ops.roiaware_pool3d import (RoIAwarePool3d, points_in_boxes_batch, +from mmdet3d.ops.roiaware_pool3d import (RoIAwarePool3d, points_in_boxes_all, points_in_boxes_cpu, - points_in_boxes_gpu) + points_in_boxes_part) def test_RoIAwarePool3d(): @@ -41,7 +41,7 @@ def test_RoIAwarePool3d(): torch.tensor(49.750).cuda(), 1e-3) -def test_points_in_boxes_gpu(): +def test_points_in_boxes_part(): if not torch.cuda.is_available(): pytest.skip('test requires GPU and torch+cuda') boxes = torch.tensor( @@ -57,7 +57,7 @@ def test_points_in_boxes_gpu(): [0, 0, 0], [6, 7, 8], [-2, -3, -4], [6, 4, 9]]], dtype=torch.float32).cuda() # points (b, m, 3) in lidar coordinate - point_indices = points_in_boxes_gpu(points=pts, boxes=boxes) + point_indices = points_in_boxes_part(points=pts, boxes=boxes) expected_point_indices = torch.tensor( [[0, 0, 0, 0, 0, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1]], dtype=torch.int32).cuda() @@ -70,7 +70,7 @@ def test_points_in_boxes_gpu(): [[[4, 6.928, 0], [6.928, 4, 0], [4, -6.928, 0], [6.928, -4, 0], [-4, 6.928, 0], [-6.928, 4, 0], [-4, -6.928, 0], [-6.928, -4, 0]]], dtype=torch.float32).cuda() - point_indices = points_in_boxes_gpu(points=pts, boxes=boxes) + point_indices = points_in_boxes_part(points=pts, boxes=boxes) expected_point_indices = torch.tensor([[-1, -1, 0, -1, 0, -1, -1, -1]], dtype=torch.int32).cuda() assert (point_indices == expected_point_indices).all() @@ -79,7 +79,7 @@ def test_points_in_boxes_gpu(): pts = pts.to('cuda:1') boxes = boxes.to('cuda:1') expected_point_indices = expected_point_indices.to('cuda:1') - point_indices = points_in_boxes_gpu(points=pts, boxes=boxes) + point_indices = points_in_boxes_part(points=pts, boxes=boxes) assert point_indices.shape == torch.Size([2, 8]) assert (point_indices == expected_point_indices).all() @@ -118,7 +118,7 @@ def test_points_in_boxes_cpu(): assert (point_indices == expected_point_indices).all() -def test_points_in_boxes_batch(): +def test_points_in_boxes_all(): if not torch.cuda.is_available(): pytest.skip('test requires GPU and torch+cuda') @@ -135,7 +135,7 @@ def test_points_in_boxes_batch(): ], [-21.3, -52, -5], [0, 0, 0], [6, 7, 8], [-2, -3, -4]]], dtype=torch.float32).cuda() # points (n, 3) in lidar coordinate - point_indices = points_in_boxes_batch(points=pts, boxes=boxes) + point_indices = points_in_boxes_all(points=pts, boxes=boxes) expected_point_indices = torch.tensor( [[[1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]], @@ -147,6 +147,6 @@ def test_points_in_boxes_batch(): pts = pts.to('cuda:1') boxes = boxes.to('cuda:1') expected_point_indices = expected_point_indices.to('cuda:1') - point_indices = points_in_boxes_batch(points=pts, boxes=boxes) + point_indices = points_in_boxes_all(points=pts, boxes=boxes) assert point_indices.shape == torch.Size([1, 15, 2]) assert (point_indices == expected_point_indices).all() diff --git a/tests/test_models/test_forward.py b/tests/test_models/test_forward.py index aa8c5679d6..5f83722cef 100644 --- a/tests/test_models/test_forward.py +++ b/tests/test_models/test_forward.py @@ -147,7 +147,7 @@ def _demo_mm_inputs(input_shape=(1, 3, 300, 300), input_shape (tuple): input batch dimensions - num_items (None | List[int]): + num_items (List[int]): specifies the number of boxes in each batch item num_classes (int): diff --git a/tests/test_models/test_heads/test_heads.py b/tests/test_models/test_heads/test_heads.py index c94e222471..3e89eaf07d 100644 --- a/tests/test_models/test_heads/test_heads.py +++ b/tests/test_models/test_heads/test_heads.py @@ -1143,7 +1143,7 @@ def test_groupfree3d_head(): assert ret_dict['s5.sem_scores'].shape == torch.Size([2, 256, 18]) # test losses - points = [torch.rand([50000, 4], device='cuda') for i in range(2)] + points = [torch.rand([5000, 4], device='cuda') for i in range(2)] gt_bbox1 = torch.rand([10, 7], dtype=torch.float32).cuda() gt_bbox2 = torch.rand([10, 7], dtype=torch.float32).cuda() @@ -1151,12 +1151,12 @@ def test_groupfree3d_head(): gt_bbox2 = DepthInstance3DBoxes(gt_bbox2) gt_bboxes = [gt_bbox1, gt_bbox2] - pts_instance_mask_1 = torch.randint(0, 10, [50000], device='cuda') - pts_instance_mask_2 = torch.randint(0, 10, [50000], device='cuda') + pts_instance_mask_1 = torch.randint(0, 10, [5000], device='cuda') + pts_instance_mask_2 = torch.randint(0, 10, [5000], device='cuda') pts_instance_mask = [pts_instance_mask_1, pts_instance_mask_2] - pts_semantic_mask_1 = torch.randint(0, 19, [50000], device='cuda') - pts_semantic_mask_2 = torch.randint(0, 19, [50000], device='cuda') + pts_semantic_mask_1 = torch.randint(0, 19, [5000], device='cuda') + pts_semantic_mask_2 = torch.randint(0, 19, [5000], device='cuda') pts_semantic_mask = [pts_semantic_mask_1, pts_semantic_mask_2] labels_1 = torch.randint(0, 18, [10], device='cuda') @@ -1177,7 +1177,7 @@ def test_groupfree3d_head(): # test multiclass_nms_single obj_scores = torch.rand([256], device='cuda') sem_scores = torch.rand([256, 18], device='cuda') - points = torch.rand([50000, 3], device='cuda') + points = torch.rand([5000, 3], device='cuda') bbox = torch.rand([256, 7], device='cuda') input_meta = dict(box_type_3d=DepthInstance3DBoxes) bbox_selected, score_selected, labels = \ @@ -1192,9 +1192,9 @@ def test_groupfree3d_head(): assert labels.shape[0] >= 0 # test get_boxes - points = torch.rand([1, 50000, 3], device='cuda') + points = torch.rand([1, 5000, 3], device='cuda') seed_points = torch.rand([1, 1024, 3], device='cuda') - seed_indices = torch.randint(0, 50000, [1, 1024], device='cuda') + seed_indices = torch.randint(0, 5000, [1, 1024], device='cuda') obj_scores = torch.rand([1, 256, 1], device='cuda') center = torch.rand([1, 256, 3], device='cuda') dir_class = torch.rand([1, 256, 1], device='cuda') diff --git a/tests/test_utils/test_box3d.py b/tests/test_utils/test_box3d.py index 810921c806..2520ce9aca 100644 --- a/tests/test_utils/test_box3d.py +++ b/tests/test_utils/test_box3d.py @@ -4,9 +4,9 @@ import unittest from mmdet3d.core.bbox import (BaseInstance3DBoxes, Box3DMode, - CameraInstance3DBoxes, DepthInstance3DBoxes, - LiDARInstance3DBoxes, bbox3d2roi, - bbox3d_mapping_back) + CameraInstance3DBoxes, Coord3DMode, + DepthInstance3DBoxes, LiDARInstance3DBoxes, + bbox3d2roi, bbox3d_mapping_back) from mmdet3d.core.bbox.structures.utils import (get_box_type, limit_period, points_cam2img, rotation_3d_in_axis, @@ -408,6 +408,13 @@ def test_lidar_boxes3d(): assert torch.allclose(boxes.tensor, expected_tensor) # test bbox in_range_bev + expected_tensor = torch.tensor( + [[1.1282, -3.0508, 1.7598, 3.4090, -1.2079], + [8.0981, -4.9332, 1.5486, 4.0325, -1.3479], + [27.6424, -7.2409, 1.4782, 2.2425, 1.8421], + [20.0183, -28.4773, 1.5687, 3.4995, 1.9621], + [28.2147, -16.5020, 1.7497, 3.7911, -2.5179]]) + assert torch.allclose(boxes.bev, expected_tensor, atol=1e-3) expected_tensor = torch.tensor([1, 1, 1, 1, 1], dtype=torch.bool) mask = boxes.in_range_bev([0., -40., 70.4, 40.]) assert (mask == expected_tensor).all() @@ -999,6 +1006,14 @@ def test_camera_boxes3d(): mask = boxes.in_range_3d([-2, -5, 0, 20, 2, 22]) assert (mask == expected_tensor).all() + expected_tensor = torch.tensor( + [[3.0508, 1.1282, 1.7598, 3.4090, -5.9203], + [4.9332, 8.0981, 1.5486, 4.0325, -6.0603], + [7.2409, 27.6424, 1.4782, 2.2425, -2.8703], + [28.4773, 20.0183, 1.5687, 3.4995, -2.7503], + [16.5020, 28.2147, 1.7497, 3.7911, -0.9471]]) + assert torch.allclose(boxes.bev, expected_tensor, atol=1e-3) + # test properties assert torch.allclose(boxes.bottom_center, boxes.tensor[:, :3]) expected_tensor = ( @@ -1388,6 +1403,11 @@ def test_depth_boxes3d(): mask = boxes.nonempty() assert (mask == expected_tensor).all() + # test bbox in_range + expected_tensor = torch.tensor([0, 1], dtype=torch.bool) + mask = boxes.in_range_3d([1, 0, -2, 2, 1, 5]) + assert (mask == expected_tensor).all() + expected_tensor = torch.tensor([[[-0.1030, 0.6649, 0.1056], [-0.1030, 0.6649, 0.3852], [-0.1030, 0.9029, 0.3852], @@ -1408,7 +1428,7 @@ def test_depth_boxes3d(): # test points in boxes if torch.cuda.is_available(): - box_idxs_of_pts = boxes.points_in_boxes_batch(points.cuda()) + box_idxs_of_pts = boxes.points_in_boxes_all(points.cuda()) expected_idxs_of_pts = torch.tensor( [[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]], device='cuda:0', @@ -1466,23 +1486,25 @@ def test_depth_boxes3d(): def test_rotation_3d_in_axis(): - # # clockwise - # points = torch.tensor([[[-0.4599, -0.0471, 0.0000], - # [-0.4599, -0.0471, 1.8433], - # [-0.4599, 0.0471, 1.8433]], - # [[-0.2555, -0.2683, 0.0000], - # [-0.2555, -0.2683, 0.9072], - # [-0.2555, 0.2683, 0.9072]]]) - # rotated = rotation_3d_in_axis( - # points, torch.tensor([-np.pi / 10, np.pi / 10]), - # axis=0, clockwise=True) - # expected_rotated = torch.tensor([[[0.0000, -0.4228, -0.1869], - # [1.8433, -0.4228, -0.1869], - # [1.8433, -0.4519, -0.0973]], - # [[0.0000, -0.3259, -0.1762], - # [0.9072, -0.3259, -0.1762], - # [0.9072, -0.1601, 0.3341]]]) - # assert torch.allclose(rotated, expected_rotated, 1e-3) + # clockwise + points = torch.tensor([[[-0.4599, -0.0471, 0.0000], + [-0.4599, -0.0471, 1.8433], + [-0.4599, 0.0471, 1.8433]], + [[-0.2555, -0.2683, 0.0000], + [-0.2555, -0.2683, 0.9072], + [-0.2555, 0.2683, 0.9072]]]) + rotated = rotation_3d_in_axis( + points, + torch.tensor([-np.pi / 10, np.pi / 10]), + axis=0, + clockwise=True) + expected_rotated = torch.tensor( + [[[-0.4599, -0.0448, -0.0146], [-0.4599, -0.6144, 1.7385], + [-0.4599, -0.5248, 1.7676]], + [[-0.2555, -0.2552, 0.0829], [-0.2555, 0.0252, 0.9457], + [-0.2555, 0.5355, 0.7799]]], + dtype=torch.float32) + assert torch.allclose(rotated, expected_rotated, atol=1e-3) # anti-clockwise with return rotation mat points = torch.tensor([[[-0.4599, -0.0471, 0.0000], @@ -1621,3 +1643,128 @@ def test_points_cam2img(): point_2d_res = points_cam2img(points, proj_mat) expected_point_2d_res = torch.from_numpy(expected_point_2d_res) assert torch.allclose(point_2d_res, expected_point_2d_res, 1e-3) + + point_2d_res = points_cam2img(points, proj_mat, with_depth=True) + expected_point_2d_res = torch.tensor([[0.5832, 0.6496, 1.7577], + [0.6146, 0.7910, 1.5477], + [0.6994, 0.7782, 2.0091], + [0.5623, 0.6303, 1.8739], + [0.4359, 0.6532, 1.2056]]) + assert torch.allclose(point_2d_res, expected_point_2d_res, 1e-3) + + +def test_points_in_boxes(): + if not torch.cuda.is_available(): + pytest.skip('test requires GPU and torch+cuda') + lidar_pts = torch.tensor([[1.0, 4.3, 0.1], [1.0, 4.4, + 0.1], [1.1, 4.3, 0.1], + [0.9, 4.3, 0.1], [1.0, -0.3, 0.1], + [1.0, -0.4, 0.1], [2.9, 0.1, 6.0], + [-0.9, 3.9, 6.0]]).cuda() + lidar_boxes = torch.tensor([[1.0, 2.0, 0.0, 4.0, 4.0, 6.0, np.pi / 6], + [1.0, 2.0, 0.0, 4.0, 4.0, 6.0, np.pi / 2], + [1.0, 2.0, 0.0, 4.0, 4.0, 6.0, 7 * np.pi / 6], + [1.0, 2.0, 0.0, 4.0, 4.0, 6.0, -np.pi / 6]], + dtype=torch.float32).cuda() + lidar_boxes = LiDARInstance3DBoxes(lidar_boxes) + + point_indices = lidar_boxes.points_in_boxes_all(lidar_pts) + expected_point_indices = torch.tensor( + [[1, 0, 1, 1], [0, 0, 0, 0], [1, 0, 1, 0], [0, 0, 0, 1], [1, 0, 1, 1], + [0, 0, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0]], + dtype=torch.int32).cuda() + assert point_indices.shape == torch.Size([8, 4]) + assert (point_indices == expected_point_indices).all() + + lidar_pts = torch.tensor([[1.0, 4.3, 0.1], [1.0, 4.4, + 0.1], [1.1, 4.3, 0.1], + [0.9, 4.3, 0.1], [1.0, -0.3, 0.1], + [1.0, -0.4, 0.1], [2.9, 0.1, 6.0], + [-0.9, 3.9, 6.0]]).cuda() + lidar_boxes = torch.tensor([[1.0, 2.0, 0.0, 4.0, 4.0, 6.0, np.pi / 6], + [1.0, 2.0, 0.0, 4.0, 4.0, 6.0, np.pi / 2], + [1.0, 2.0, 0.0, 4.0, 4.0, 6.0, 7 * np.pi / 6], + [1.0, 2.0, 0.0, 4.0, 4.0, 6.0, -np.pi / 6]], + dtype=torch.float32).cuda() + lidar_boxes = LiDARInstance3DBoxes(lidar_boxes) + + point_indices = lidar_boxes.points_in_boxes_part(lidar_pts) + expected_point_indices = torch.tensor([0, -1, 0, 3, 0, -1, 1, 1], + dtype=torch.int32).cuda() + assert point_indices.shape == torch.Size([8]) + assert (point_indices == expected_point_indices).all() + + depth_boxes = torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 0.3], + [-10.0, 23.0, 16.0, 10, 20, 20, 0.5]], + dtype=torch.float32).cuda() + depth_boxes = DepthInstance3DBoxes(depth_boxes) + depth_pts = torch.tensor( + [[[1, 2, 3.3], [1.2, 2.5, 3.0], [0.8, 2.1, 3.5], [1.6, 2.6, 3.6], + [0.8, 1.2, 3.9], [-9.2, 21.0, 18.2], [3.8, 7.9, 6.3], + [4.7, 3.5, -12.2], [3.8, 7.6, -2], [-10.6, -12.9, -20], [ + -16, -18, 9 + ], [-21.3, -52, -5], [0, 0, 0], [6, 7, 8], [-2, -3, -4]]], + dtype=torch.float32).cuda() + + point_indices = depth_boxes.points_in_boxes_all(depth_pts) + expected_point_indices = torch.tensor( + [[1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1], [0, 0], [0, 0], + [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0], [0, 0]], + dtype=torch.int32).cuda() + assert point_indices.shape == torch.Size([15, 2]) + assert (point_indices == expected_point_indices).all() + + point_indices = depth_boxes.points_in_boxes_part(depth_pts) + expected_point_indices = torch.tensor( + [0, 0, 0, 0, 0, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1], + dtype=torch.int32).cuda() + assert point_indices.shape == torch.Size([15]) + assert (point_indices == expected_point_indices).all() + + depth_boxes = torch.tensor([[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 0.3], + [-10.0, 23.0, 16.0, 10, 20, 20, 0.5], + [1.0, 2.0, 0.0, 4.0, 4.0, 6.0, np.pi / 6], + [1.0, 2.0, 0.0, 4.0, 4.0, 6.0, np.pi / 2], + [1.0, 2.0, 0.0, 4.0, 4.0, 6.0, 7 * np.pi / 6], + [1.0, 2.0, 0.0, 4.0, 4.0, 6.0, -np.pi / 6]], + dtype=torch.float32).cuda() + cam_boxes = DepthInstance3DBoxes(depth_boxes).convert_to(Box3DMode.CAM) + depth_pts = torch.tensor( + [[1, 2, 3.3], [1.2, 2.5, 3.0], [0.8, 2.1, 3.5], [1.6, 2.6, 3.6], + [0.8, 1.2, 3.9], [-9.2, 21.0, 18.2], [3.8, 7.9, 6.3], + [4.7, 3.5, -12.2], [3.8, 7.6, -2], [-10.6, -12.9, -20], [-16, -18, 9], + [-21.3, -52, -5], [0, 0, 0], [6, 7, 8], [-2, -3, -4], [1.0, 4.3, 0.1], + [1.0, 4.4, 0.1], [1.1, 4.3, 0.1], [0.9, 4.3, 0.1], [1.0, -0.3, 0.1], + [1.0, -0.4, 0.1], [2.9, 0.1, 6.0], [-0.9, 3.9, 6.0]], + dtype=torch.float32).cuda() + + cam_pts = DepthPoints(depth_pts).convert_to(Coord3DMode.CAM).tensor + + point_indices = cam_boxes.points_in_boxes_all(cam_pts) + expected_point_indices = torch.tensor( + [[1, 0, 1, 1, 1, 1], [1, 0, 1, 1, 1, 1], [1, 0, 1, 1, 1, 1], + [1, 0, 1, 1, 1, 1], [1, 0, 1, 1, 1, 1], [0, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], + [0, 0, 0, 1, 0, 1], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], + [0, 0, 1, 1, 1, 1], [0, 0, 0, 1, 0, 0], [0, 0, 0, 1, 0, 1], + [0, 0, 1, 1, 1, 0], [0, 0, 1, 1, 1, 1], [0, 0, 0, 1, 0, 0], + [1, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0]], + dtype=torch.int32).cuda() + assert point_indices.shape == torch.Size([23, 6]) + assert (point_indices == expected_point_indices).all() + + point_indices = cam_boxes.points_in_boxes_batch(cam_pts) + assert (point_indices == expected_point_indices).all() + + point_indices = cam_boxes.points_in_boxes_part(cam_pts) + expected_point_indices = torch.tensor([ + 0, 0, 0, 0, 0, 1, -1, -1, -1, -1, -1, -1, 3, -1, -1, 2, 3, 3, 2, 2, 3, + 0, 0 + ], + dtype=torch.int32).cuda() + assert point_indices.shape == torch.Size([23]) + assert (point_indices == expected_point_indices).all() + + point_indices = cam_boxes.points_in_boxes(cam_pts) + assert (point_indices == expected_point_indices).all() diff --git a/tests/test_utils/test_box_np_ops.py b/tests/test_utils/test_box_np_ops.py index a6beab0b27..4a4ccc14cf 100644 --- a/tests/test_utils/test_box_np_ops.py +++ b/tests/test_utils/test_box_np_ops.py @@ -62,3 +62,21 @@ def test_center_to_corner_box2d(): expected_corner = np.array([[[-4.24264, -1.41421], [1.41421, 4.24264], [4.24264, 1.41421], [-1.41421, -4.24264]]]) assert np.allclose(corner, expected_corner) + + +def test_points_in_convex_polygon_jit(): + from mmdet3d.core.bbox.box_np_ops import points_in_convex_polygon_jit + points = np.array([[0.4, 0.4], [0.5, 0.5], [0.6, 0.6]]) + polygons = np.array([[[1.0, 0.0], [0.0, 1.0], [0.0, 0.5], [0.0, 0.0]], + [[1.0, 0.0], [1.0, 1.0], [0.5, 1.0], [0.0, 1.0]], + [[1.0, 0.0], [0.0, 1.0], [-1.0, 0.0], [0.0, -1.0]]]) + res = points_in_convex_polygon_jit(points, polygons) + expected_res = np.array([[1, 0, 1], [0, 0, 0], [0, 1, 0]]).astype(np.bool) + assert np.allclose(res, expected_res) + + polygons = np.array([[[0.0, 0.0], [0.0, 1.0], [0.5, 0.5], [1.0, 0.0]], + [[0.0, 1.0], [1.0, 1.0], [1.0, 0.5], [1.0, 0.0]], + [[1.0, 0.0], [0.0, -1.0], [-1.0, 0.0], [0.0, 1.1]]]) + res = points_in_convex_polygon_jit(points, polygons, clockwise=True) + expected_res = np.array([[1, 0, 1], [0, 0, 1], [0, 1, 0]]).astype(np.bool) + assert np.allclose(res, expected_res) diff --git a/tests/test_utils/test_coord_3d_mode.py b/tests/test_utils/test_coord_3d_mode.py index 123c84103a..6a18120b77 100644 --- a/tests/test_utils/test_coord_3d_mode.py +++ b/tests/test_utils/test_coord_3d_mode.py @@ -262,11 +262,11 @@ def test_boxes_conversion(): convert_depth_boxes = Coord3DMode.convert(cam_boxes, Coord3DMode.CAM, Coord3DMode.DEPTH) expected_tensor = torch.tensor( - [[1.7802, 1.7501, 2.5162, 1.7500, 1.6500, 3.3900, -1.4800], - [8.9594, 1.6357, 2.4567, 1.5400, 1.5700, 4.0100, -1.6200], - [28.2967, 1.3033, -0.5558, 1.4700, 1.4800, 2.2300, 1.5700], - [26.6690, 1.7361, 21.8230, 1.5600, 1.4000, 3.4800, 1.6900], - [31.3198, 1.6218, 8.1621, 1.7400, 1.4800, 3.7700, -2.7900]]) + [[1.7802, -1.7501, -2.5162, 1.7500, 1.6500, 3.3900, -1.4800], + [8.9594, -1.6357, -2.4567, 1.5400, 1.5700, 4.0100, -1.6200], + [28.2967, -1.3033, 0.5558, 1.4700, 1.4800, 2.2300, 1.5700], + [26.6690, -1.7361, -21.8230, 1.5600, 1.4000, 3.4800, 1.6900], + [31.3198, -1.6218, -8.1621, 1.7400, 1.4800, 3.7700, -2.7900]]) assert torch.allclose(expected_tensor, convert_depth_boxes.tensor, 1e-3) # test LIDAR to CAM and DEPTH @@ -326,11 +326,11 @@ def test_boxes_conversion(): convert_cam_boxes = Coord3DMode.convert(depth_boxes, Coord3DMode.DEPTH, Coord3DMode.CAM) expected_tensor = torch.tensor( - [[1.7802, -1.7501, -2.5162, 1.7500, 1.6500, 3.3900, -1.4800], - [8.9594, -1.6357, -2.4567, 1.5400, 1.5700, 4.0100, -1.6200], - [28.2967, -1.3033, 0.5558, 1.4700, 1.4800, 2.2300, 1.5700], - [26.6690, -1.7361, -21.8230, 1.5600, 1.4000, 3.4800, 1.6900], - [31.3198, -1.6218, -8.1621, 1.7400, 1.4800, 3.7700, -2.7900]]) + [[1.7802, 1.7501, 2.5162, 1.7500, 1.6500, 3.3900, -1.4800], + [8.9594, 1.6357, 2.4567, 1.5400, 1.5700, 4.0100, -1.6200], + [28.2967, 1.3033, -0.5558, 1.4700, 1.4800, 2.2300, 1.5700], + [26.6690, 1.7361, 21.8230, 1.5600, 1.4000, 3.4800, 1.6900], + [31.3198, 1.6218, 8.1621, 1.7400, 1.4800, 3.7700, -2.7900]]) assert torch.allclose(expected_tensor, convert_cam_boxes.tensor, 1e-3) convert_lidar_boxes = Coord3DMode.convert(depth_boxes, Coord3DMode.DEPTH, diff --git a/tests/test_utils/test_points.py b/tests/test_utils/test_points.py index aaa4325623..e63936b3bd 100644 --- a/tests/test_utils/test_points.py +++ b/tests/test_utils/test_points.py @@ -65,6 +65,7 @@ def test_base_points(): ]]) assert torch.allclose(expected_tensor, base_points.tensor) + assert torch.allclose(expected_tensor[:, :2], base_points.bev) assert torch.allclose(expected_tensor[:, :3], base_points.coord) assert torch.allclose(expected_tensor[:, 3:6], base_points.color) assert torch.allclose(expected_tensor[:, 6], base_points.height) @@ -326,6 +327,7 @@ def test_cam_points(): ]]) assert torch.allclose(expected_tensor, cam_points.tensor) + assert torch.allclose(expected_tensor[:, [0, 2]], cam_points.bev) assert torch.allclose(expected_tensor[:, :3], cam_points.coord) assert torch.allclose(expected_tensor[:, 3:6], cam_points.color) assert torch.allclose(expected_tensor[:, 6], cam_points.height) @@ -602,6 +604,7 @@ def test_lidar_points(): ]]) assert torch.allclose(expected_tensor, lidar_points.tensor) + assert torch.allclose(expected_tensor[:, :2], lidar_points.bev) assert torch.allclose(expected_tensor[:, :3], lidar_points.coord) assert torch.allclose(expected_tensor[:, 3:6], lidar_points.color) assert torch.allclose(expected_tensor[:, 6], lidar_points.height) @@ -878,6 +881,7 @@ def test_depth_points(): ]]) assert torch.allclose(expected_tensor, depth_points.tensor) + assert torch.allclose(expected_tensor[:, :2], depth_points.bev) assert torch.allclose(expected_tensor[:, :3], depth_points.coord) assert torch.allclose(expected_tensor[:, 3:6], depth_points.color) assert torch.allclose(expected_tensor[:, 6], depth_points.height) diff --git a/tools/create_data.py b/tools/create_data.py index 9935d6cfb4..fdb7e4b06f 100644 --- a/tools/create_data.py +++ b/tools/create_data.py @@ -60,7 +60,8 @@ def nuscenes_data_prep(root_path, version (str): Dataset version. dataset_name (str): The dataset class name. out_dir (str): Output directory of the groundtruth database info. - max_sweeps (int): Number of input consecutive frames. Default: 10 + max_sweeps (int, optional): Number of input consecutive frames. + Default: 10 """ nuscenes_converter.create_nuscenes_infos( root_path, info_prefix, version=version, max_sweeps=max_sweeps) @@ -101,7 +102,8 @@ def lyft_data_prep(root_path, dataset_name (str): The dataset class name. out_dir (str): Output directory of the groundtruth database info. Not used here if the groundtruth database is not generated. - max_sweeps (int): Number of input consecutive frames. Default: 10 + max_sweeps (int, optional): Number of input consecutive frames. + Default: 10 """ lyft_converter.create_lyft_infos( root_path, info_prefix, version=version, max_sweeps=max_sweeps) @@ -173,8 +175,9 @@ def waymo_data_prep(root_path, info_prefix (str): The prefix of info filenames. out_dir (str): Output directory of the generated info file. workers (int): Number of threads to be used. - max_sweeps (int): Number of input consecutive frames. Default: 5 \ - Here we store pose information of these frames for later use. + max_sweeps (int, optional): Number of input consecutive frames. + Default: 5. Here we store pose information of these frames + for later use. """ from tools.data_converter import waymo_converter as waymo diff --git a/tools/data_converter/create_gt_database.py b/tools/data_converter/create_gt_database.py index 5f6df3ab49..86040e1800 100644 --- a/tools/data_converter/create_gt_database.py +++ b/tools/data_converter/create_gt_database.py @@ -125,19 +125,19 @@ def create_groundtruth_database(dataset_class_name, dataset_class_name （str): Name of the input dataset. data_path (str): Path of the data. info_prefix (str): Prefix of the info file. - info_path (str): Path of the info file. + info_path (str, optional): Path of the info file. Default: None. - mask_anno_path (str): Path of the mask_anno. + mask_anno_path (str, optional): Path of the mask_anno. Default: None. - used_classes (list[str]): Classes have been used. + used_classes (list[str], optional): Classes have been used. Default: None. - database_save_path (str): Path to save database. + database_save_path (str, optional): Path to save database. Default: None. - db_info_save_path (str): Path to save db_info. + db_info_save_path (str, optional): Path to save db_info. Default: None. - relative_path (bool): Whether to use relative path. + relative_path (bool, optional): Whether to use relative path. Default: True. - with_mask (bool): Whether to use mask. + with_mask (bool, optional): Whether to use mask. Default: False. """ print(f'Create GT Database of {dataset_class_name}') diff --git a/tools/data_converter/indoor_converter.py b/tools/data_converter/indoor_converter.py index b408bd017d..66761b638c 100644 --- a/tools/data_converter/indoor_converter.py +++ b/tools/data_converter/indoor_converter.py @@ -18,10 +18,11 @@ def create_indoor_info_file(data_path, Args: data_path (str): Path of the data. - pkl_prefix (str): Prefix of the pkl to be saved. Default: 'sunrgbd'. - save_path (str): Path of the pkl to be saved. Default: None. - use_v1 (bool): Whether to use v1. Default: False. - workers (int): Number of threads to be used. Default: 4. + pkl_prefix (str, optional): Prefix of the pkl to be saved. + Default: 'sunrgbd'. + save_path (str, optional): Path of the pkl to be saved. Default: None. + use_v1 (bool, optional): Whether to use v1. Default: False. + workers (int, optional): Number of threads to be used. Default: 4. """ assert os.path.exists(data_path) assert pkl_prefix in ['sunrgbd', 'scannet', 's3dis'], \ diff --git a/tools/data_converter/kitti_converter.py b/tools/data_converter/kitti_converter.py index 796db9a02b..7784a6d203 100644 --- a/tools/data_converter/kitti_converter.py +++ b/tools/data_converter/kitti_converter.py @@ -158,7 +158,7 @@ def create_waymo_info_file(data_path, Args: data_path (str): Path of the data root. pkl_prefix (str): Prefix of the info file to be generated. - save_path (str | None): Path to save the info file. + save_path (str): Path to save the info file. relative_path (bool): Whether to use relative path. max_sweeps (int): Max sweeps before the detection frame to be used. """ @@ -237,11 +237,13 @@ def _create_reduced_point_cloud(data_path, Args: data_path (str): Path of original data. info_path (str): Path of data info. - save_path (str | None): Path to save reduced point cloud data. - Default: None. - back (bool): Whether to flip the points to back. - num_features (int): Number of point features. Default: 4. - front_camera_id (int): The referenced/front camera ID. Default: 2. + save_path (str, optional): Path to save reduced point cloud + data. Default: None. + back (bool, optional): Whether to flip the points to back. + Default: False. + num_features (int, optional): Number of point features. Default: 4. + front_camera_id (int, optional): The referenced/front camera ID. + Default: 2. """ kitti_infos = mmcv.load(info_path) @@ -297,14 +299,16 @@ def create_reduced_point_cloud(data_path, Args: data_path (str): Path of original data. pkl_prefix (str): Prefix of info files. - train_info_path (str | None): Path of training set info. + train_info_path (str, optional): Path of training set info. + Default: None. + val_info_path (str, optional): Path of validation set info. Default: None. - val_info_path (str | None): Path of validation set info. + test_info_path (str, optional): Path of test set info. Default: None. - test_info_path (str | None): Path of test set info. + save_path (str, optional): Path to save reduced point cloud data. Default: None. - save_path (str | None): Path to save reduced point cloud data. - with_back (bool): Whether to flip the points to back. + with_back (bool, optional): Whether to flip the points to back. + Default: False. """ if train_info_path is None: train_info_path = Path(data_path) / f'{pkl_prefix}_infos_train.pkl' @@ -334,7 +338,8 @@ def export_2d_annotation(root_path, info_path, mono3d=True): Args: root_path (str): Root path of the raw data. info_path (str): Path of the info file. - mono3d (bool): Whether to export mono3d annotation. Default: True. + mono3d (bool, optional): Whether to export mono3d annotation. + Default: True. """ # get bbox annotations for camera kitti_infos = mmcv.load(info_path) @@ -380,8 +385,8 @@ def get_2d_boxes(info, occluded, mono3d=True): Args: info: Information of the given sample data. - occluded: Integer (0, 1, 2, 3) indicating occlusion state: \ - 0 = fully visible, 1 = partly occluded, 2 = largely occluded, \ + occluded: Integer (0, 1, 2, 3) indicating occlusion state: + 0 = fully visible, 1 = partly occluded, 2 = largely occluded, 3 = unknown, -1 = DontCare mono3d (bool): Whether to get boxes with mono3d annotation. @@ -507,7 +512,7 @@ def generate_record(ann_rec, x1, y1, x2, y2, sample_data_token, filename): - area (float): 2d box area - category_name (str): category name - category_id (int): category id - - bbox (list[float]): left x, top y, dx, dy of 2d box + - bbox (list[float]): left x, top y, x_size, y_size of 2d box - iscrowd (int): whether the area is crowd """ repro_rec = OrderedDict() diff --git a/tools/data_converter/lyft_converter.py b/tools/data_converter/lyft_converter.py index be26f5b8e7..e2d9ddb283 100644 --- a/tools/data_converter/lyft_converter.py +++ b/tools/data_converter/lyft_converter.py @@ -24,10 +24,10 @@ def create_lyft_infos(root_path, Args: root_path (str): Path of the data root. info_prefix (str): Prefix of the info file to be generated. - version (str): Version of the data. - Default: 'v1.01-train' - max_sweeps (int): Max number of sweeps. - Default: 10 + version (str, optional): Version of the data. + Default: 'v1.01-train'. + max_sweeps (int, optional): Max number of sweeps. + Default: 10. """ lyft = Lyft( data_path=osp.join(root_path, version), @@ -99,9 +99,9 @@ def _fill_trainval_infos(lyft, lyft (:obj:`LyftDataset`): Dataset class in the Lyft dataset. train_scenes (list[str]): Basic information of training scenes. val_scenes (list[str]): Basic information of validation scenes. - test (bool): Whether use the test mode. In the test mode, no + test (bool, optional): Whether use the test mode. In the test mode, no annotations can be accessed. Default: False. - max_sweeps (int): Max number of sweeps. Default: 10. + max_sweeps (int, optional): Max number of sweeps. Default: 10. Returns: tuple[list[dict]]: Information of training set and @@ -192,7 +192,7 @@ def _fill_trainval_infos(lyft, # we need to convert box size to # the format of our lidar coordinate system - # which is dx, dy, dz (corresponding to l, w, h) + # which is x_size, y_size, z_size (corresponding to l, w, h) gt_boxes = np.concatenate([locs, dims[:, [1, 0, 2]], rots], axis=1) assert len(gt_boxes) == len( annotations), f'{len(gt_boxes)}, {len(annotations)}' diff --git a/tools/data_converter/nuscenes_converter.py b/tools/data_converter/nuscenes_converter.py index a43c8e28d4..0ae8c22a55 100644 --- a/tools/data_converter/nuscenes_converter.py +++ b/tools/data_converter/nuscenes_converter.py @@ -33,10 +33,10 @@ def create_nuscenes_infos(root_path, Args: root_path (str): Path of the data root. info_prefix (str): Prefix of the info file to be generated. - version (str): Version of the data. - Default: 'v1.0-trainval' - max_sweeps (int): Max number of sweeps. - Default: 10 + version (str, optional): Version of the data. + Default: 'v1.0-trainval'. + max_sweeps (int, optional): Max number of sweeps. + Default: 10. """ from nuscenes.nuscenes import NuScenes nusc = NuScenes(version=version, dataroot=root_path, verbose=True) @@ -151,9 +151,9 @@ def _fill_trainval_infos(nusc, nusc (:obj:`NuScenes`): Dataset class in the nuScenes dataset. train_scenes (list[str]): Basic information of training scenes. val_scenes (list[str]): Basic information of validation scenes. - test (bool): Whether use the test mode. In the test mode, no + test (bool, optional): Whether use the test mode. In test mode, no annotations can be accessed. Default: False. - max_sweeps (int): Max number of sweeps. Default: 10. + max_sweeps (int, optional): Max number of sweeps. Default: 10. Returns: tuple[list[dict]]: Information of training set and validation set @@ -250,7 +250,7 @@ def _fill_trainval_infos(nusc, names = np.array(names) # we need to convert box size to # the format of our lidar coordinate system - # which is dx, dy, dz (corresponding to l, w, h) + # which is x_size, y_size, z_size (corresponding to l, w, h) gt_boxes = np.concatenate([locs, dims[:, [1, 0, 2]], rots], axis=1) assert len(gt_boxes) == len( annotations), f'{len(gt_boxes)}, {len(annotations)}' @@ -290,7 +290,7 @@ def obtain_sensor2top(nusc, e2g_t (np.ndarray): Translation from ego to global in shape (1, 3). e2g_r_mat (np.ndarray): Rotation matrix from ego to global in shape (3, 3). - sensor_type (str): Sensor to calibrate. Default: 'lidar'. + sensor_type (str, optional): Sensor to calibrate. Default: 'lidar'. Returns: sweep (dict): Sweep information after transformation. @@ -339,7 +339,8 @@ def export_2d_annotation(root_path, info_path, version, mono3d=True): root_path (str): Root path of the raw data. info_path (str): Path of the info file. version (str): Dataset version. - mono3d (bool): Whether to export mono3d annotation. Default: True. + mono3d (bool, optional): Whether to export mono3d annotation. + Default: True. """ # get bbox annotations for camera camera_types = [ @@ -403,7 +404,7 @@ def get_2d_boxes(nusc, """Get the 2D annotation records for a given `sample_data_token`. Args: - sample_data_token (str): Sample data token belonging to a camera \ + sample_data_token (str): Sample data token belonging to a camera keyframe. visibilities (list[str]): Visibility filter. mono3d (bool): Whether to get boxes with mono3d annotation. diff --git a/tools/data_converter/s3dis_data_utils.py b/tools/data_converter/s3dis_data_utils.py index f144692847..e356b0aef6 100644 --- a/tools/data_converter/s3dis_data_utils.py +++ b/tools/data_converter/s3dis_data_utils.py @@ -12,7 +12,7 @@ class S3DISData(object): Args: root_path (str): Root path of the raw data. - split (str): Set split type of the data. Default: 'Area_1'. + split (str, optional): Set split type of the data. Default: 'Area_1'. """ def __init__(self, root_path, split='Area_1'): @@ -47,9 +47,11 @@ def get_infos(self, num_workers=4, has_label=True, sample_id_list=None): This method gets information from the raw data. Args: - num_workers (int): Number of threads to be used. Default: 4. - has_label (bool): Whether the data has label. Default: True. - sample_id_list (list[int]): Index list of the sample. + num_workers (int, optional): Number of threads to be used. + Default: 4. + has_label (bool, optional): Whether the data has label. + Default: True. + sample_id_list (list[int], optional): Index list of the sample. Default: None. Returns: @@ -153,10 +155,11 @@ class S3DISSegData(object): Args: data_root (str): Root path of the raw data. ann_file (str): The generated scannet infos. - split (str): Set split type of the data. Default: 'train'. - num_points (int): Number of points in each data input. Default: 8192. - label_weight_func (function): Function to compute the label weight. - Default: None. + split (str, optional): Set split type of the data. Default: 'train'. + num_points (int, optional): Number of points in each data input. + Default: 8192. + label_weight_func (function, optional): Function to compute the + label weight. Default: None. """ def __init__(self, @@ -208,7 +211,7 @@ def _convert_to_label(self, mask): return label def get_scene_idxs_and_label_weight(self): - """Compute scene_idxs for data sampling and label weight for loss \ + """Compute scene_idxs for data sampling and label weight for loss calculation. We sample more times for scenes with more points. Label_weight is diff --git a/tools/data_converter/scannet_data_utils.py b/tools/data_converter/scannet_data_utils.py index 0b4bc64391..4570889b8b 100644 --- a/tools/data_converter/scannet_data_utils.py +++ b/tools/data_converter/scannet_data_utils.py @@ -12,7 +12,7 @@ class ScanNetData(object): Args: root_path (str): Root path of the raw data. - split (str): Set split type of the data. Default: 'train'. + split (str, optional): Set split type of the data. Default: 'train'. """ def __init__(self, root_path, split='train'): @@ -89,9 +89,11 @@ def get_infos(self, num_workers=4, has_label=True, sample_id_list=None): This method gets information from the raw data. Args: - num_workers (int): Number of threads to be used. Default: 4. - has_label (bool): Whether the data has label. Default: True. - sample_id_list (list[int]): Index list of the sample. + num_workers (int, optional): Number of threads to be used. + Default: 4. + has_label (bool, optional): Whether the data has label. + Default: True. + sample_id_list (list[int], optional): Index list of the sample. Default: None. Returns: @@ -200,10 +202,11 @@ class ScanNetSegData(object): Args: data_root (str): Root path of the raw data. ann_file (str): The generated scannet infos. - split (str): Set split type of the data. Default: 'train'. - num_points (int): Number of points in each data input. Default: 8192. - label_weight_func (function): Function to compute the label weight. - Default: None. + split (str, optional): Set split type of the data. Default: 'train'. + num_points (int, optional): Number of points in each data input. + Default: 8192. + label_weight_func (function, optional): Function to compute the + label weight. Default: None. """ def __init__(self, @@ -260,7 +263,7 @@ def _convert_to_label(self, mask): return label def get_scene_idxs_and_label_weight(self): - """Compute scene_idxs for data sampling and label weight for loss \ + """Compute scene_idxs for data sampling and label weight for loss calculation. We sample more times for scenes with more points. Label_weight is diff --git a/tools/data_converter/sunrgbd_data_utils.py b/tools/data_converter/sunrgbd_data_utils.py index 8efc7ea140..48ab46333b 100644 --- a/tools/data_converter/sunrgbd_data_utils.py +++ b/tools/data_converter/sunrgbd_data_utils.py @@ -41,7 +41,7 @@ def __init__(self, line): self.ymax = data[2] + data[4] self.box2d = np.array([self.xmin, self.ymin, self.xmax, self.ymax]) self.centroid = np.array([data[5], data[6], data[7]]) - # data[9] is dx (l), data[8] is dy (w), data[10] is dz (h) + # data[9] is x_size (l), data[8] is y_size (w), data[10] is z_size (h) # in our depth coordinate system, # l corresponds to the size along the x axis self.size = np.array([data[9], data[8], data[10]]) * 2 @@ -61,8 +61,8 @@ class SUNRGBDData(object): Args: root_path (str): Root path of the raw data. - split (str): Set split type of the data. Default: 'train'. - use_v1 (bool): Whether to use v1. Default: False. + split (str, optional): Set split type of the data. Default: 'train'. + use_v1 (bool, optional): Whether to use v1. Default: False. """ def __init__(self, root_path, split='train', use_v1=False): @@ -127,9 +127,11 @@ def get_infos(self, num_workers=4, has_label=True, sample_id_list=None): This method gets information from the raw data. Args: - num_workers (int): Number of threads to be used. Default: 4. - has_label (bool): Whether the data has label. Default: True. - sample_id_list (list[int]): Index list of the sample. + num_workers (int, optional): Number of threads to be used. + Default: 4. + has_label (bool, optional): Whether the data has label. + Default: True. + sample_id_list (list[int], optional): Index list of the sample. Default: None. Returns: diff --git a/tools/data_converter/waymo_converter.py b/tools/data_converter/waymo_converter.py index 3d4105fbff..272d293334 100644 --- a/tools/data_converter/waymo_converter.py +++ b/tools/data_converter/waymo_converter.py @@ -30,8 +30,8 @@ class Waymo2KITTI(object): save_dir (str): Directory to save data in KITTI format. prefix (str): Prefix of filename. In general, 0 for training, 1 for validation and 2 for testing. - workers (str): Number of workers for the parallel process. - test_mode (bool): Whether in the test_mode. Default: False. + workers (int, optional): Number of workers for the parallel process. + test_mode (bool, optional): Whether in the test_mode. Default: False. """ def __init__(self, @@ -401,8 +401,8 @@ def convert_range_image_to_point_cloud(self, camera projections corresponding with two returns. range_image_top_pose (:obj:`Transform`): Range image pixel pose for top lidar. - ri_index (int): 0 for the first return, 1 for the second return. - Default: 0. + ri_index (int, optional): 0 for the first return, + 1 for the second return. Default: 0. Returns: tuple[list[np.ndarray]]: (List of points with shape [N, 3], From 3d9268b63310174198d053c346e4c2ae22f9068e Mon Sep 17 00:00:00 2001 From: Xi Liu <75658786+xiliu8006@users.noreply.github.com> Date: Mon, 16 Aug 2021 18:00:33 +0800 Subject: [PATCH 07/51] [Feature] PointXYZWHLRBBoxCoder (#856) * support PointBasedBoxCoder * fix unittest bug * support unittest in gpu * support unittest in gpu * modified docstring * add args * add args --- mmdet3d/core/bbox/coders/__init__.py | 4 +- .../bbox/coders/point_xyzwhlr_bbox_coder.py | 116 ++++++++++++++++++ tests/test_utils/test_bbox_coders.py | 30 +++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 mmdet3d/core/bbox/coders/point_xyzwhlr_bbox_coder.py diff --git a/mmdet3d/core/bbox/coders/__init__.py b/mmdet3d/core/bbox/coders/__init__.py index d2c2dae1a4..c662e70c1f 100644 --- a/mmdet3d/core/bbox/coders/__init__.py +++ b/mmdet3d/core/bbox/coders/__init__.py @@ -4,8 +4,10 @@ from .delta_xyzwhlr_bbox_coder import DeltaXYZWLHRBBoxCoder from .groupfree3d_bbox_coder import GroupFree3DBBoxCoder from .partial_bin_based_bbox_coder import PartialBinBasedBBoxCoder +from .point_xyzwhlr_bbox_coder import PointXYZWHLRBBoxCoder __all__ = [ 'build_bbox_coder', 'DeltaXYZWLHRBBoxCoder', 'PartialBinBasedBBoxCoder', - 'CenterPointBBoxCoder', 'AnchorFreeBBoxCoder', 'GroupFree3DBBoxCoder' + 'CenterPointBBoxCoder', 'AnchorFreeBBoxCoder', 'GroupFree3DBBoxCoder', + 'PointXYZWHLRBBoxCoder' ] diff --git a/mmdet3d/core/bbox/coders/point_xyzwhlr_bbox_coder.py b/mmdet3d/core/bbox/coders/point_xyzwhlr_bbox_coder.py new file mode 100644 index 0000000000..a2bb9cc62f --- /dev/null +++ b/mmdet3d/core/bbox/coders/point_xyzwhlr_bbox_coder.py @@ -0,0 +1,116 @@ +import numpy as np +import torch + +from mmdet.core.bbox import BaseBBoxCoder +from mmdet.core.bbox.builder import BBOX_CODERS + + +@BBOX_CODERS.register_module() +class PointXYZWHLRBBoxCoder(BaseBBoxCoder): + """Point based bbox coder for 3D boxes. + + Args: + code_size (int): The dimension of boxes to be encoded. + use_mean_size (bool, optional): Whether using anchors based on class. + Defaults to True. + mean_size (list[list[float]], optional): Mean size of bboxes in + each class. Defaults to None. + """ + + def __init__(self, code_size=7, use_mean_size=True, mean_size=None): + super(PointXYZWHLRBBoxCoder, self).__init__() + self.code_size = code_size + self.use_mean_size = use_mean_size + if self.use_mean_size: + self.mean_size = torch.from_numpy(np.array(mean_size)).float() + assert self.mean_size.min() > 0, \ + f'The min of mean_size should > 0, however currently it is '\ + f'{self.mean_size.min()}, please check it in your config.' + + def encode(self, gt_bboxes_3d, points, gt_labels_3d=None): + """Encode ground truth to prediction targets. + + Args: + gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth bboxes + with shape (N, 7 + C). + points (torch.Tensor): Point cloud with shape (N, 3). + gt_labels_3d (torch.Tensor, optional): Ground truth classes. + Defaults to None. + + Returns: + torch.Tensor: Encoded boxes with shape (N, 8 + C). + """ + gt_bboxes_3d[:, 3:6] = torch.clamp_min(gt_bboxes_3d[:, 3:6], min=1e-5) + + xg, yg, zg, dxg, dyg, dzg, rg, *cgs = torch.split( + gt_bboxes_3d, 1, dim=-1) + xa, ya, za = torch.split(points, 1, dim=-1) + + if self.use_mean_size: + assert gt_labels_3d.max() <= self.mean_size.shape[0] - 1, \ + f'the max gt label {gt_labels_3d.max()} is bigger than' \ + f'anchor types {self.mean_size.shape[0] - 1}.' + self.mean_size = self.mean_size.to(gt_labels_3d.device) + point_anchor_size = self.mean_size[gt_labels_3d] + dxa, dya, dza = torch.split(point_anchor_size, 1, dim=-1) + diagonal = torch.sqrt(dxa**2 + dya**2) + xt = (xg - xa) / diagonal + yt = (yg - ya) / diagonal + zt = (zg - za) / dza + dxt = torch.log(dxg / dxa) + dyt = torch.log(dyg / dya) + dzt = torch.log(dzg / dza) + else: + xt = (xg - xa) + yt = (yg - ya) + zt = (zg - za) + dxt = torch.log(dxg) + dyt = torch.log(dyg) + dzt = torch.log(dzg) + + return torch.cat( + [xt, yt, zt, dxt, dyt, dzt, + torch.cos(rg), + torch.sin(rg), *cgs], + dim=-1) + + def decode(self, box_encodings, points, pred_labels_3d=None): + """Decode predicted parts and points to bbox3d. + + Args: + box_encodings (torch.Tensor): Encoded boxes with shape (N, 8 + C). + points (torch.Tensor): Point cloud with shape (N, 3). + pred_labels_3d (torch.Tensor): Bbox predicted labels (N, M). + + Returns: + torch.Tensor: Decoded boxes with shape (N, 7 + C) + """ + xt, yt, zt, dxt, dyt, dzt, cost, sint, *cts = torch.split( + box_encodings, 1, dim=-1) + xa, ya, za = torch.split(points, 1, dim=-1) + + if self.use_mean_size: + assert pred_labels_3d.max() <= self.mean_size.shape[0] - 1, \ + f'The max pred label {pred_labels_3d.max()} is bigger than' \ + f'anchor types {self.mean_size.shape[0] - 1}.' + self.mean_size = self.mean_size.to(pred_labels_3d.device) + point_anchor_size = self.mean_size[pred_labels_3d] + dxa, dya, dza = torch.split(point_anchor_size, 1, dim=-1) + diagonal = torch.sqrt(dxa**2 + dya**2) + xg = xt * diagonal + xa + yg = yt * diagonal + ya + zg = zt * dza + za + + dxg = torch.exp(dxt) * dxa + dyg = torch.exp(dyt) * dya + dzg = torch.exp(dzt) * dza + else: + xg = xt + xa + yg = yt + ya + zg = zt + za + dxg, dyg, dzg = torch.split( + torch.exp(box_encodings[..., 3:6]), 1, dim=-1) + + rg = torch.atan2(sint, cost) + + return torch.cat([xg, yg, zg, dxg, dyg, dzg, rg, *cts], dim=-1) diff --git a/tests/test_utils/test_bbox_coders.py b/tests/test_utils/test_bbox_coders.py index 5ba9fbfb28..ceae5409ed 100644 --- a/tests/test_utils/test_bbox_coders.py +++ b/tests/test_utils/test_bbox_coders.py @@ -351,3 +351,33 @@ def test_centerpoint_bbox_coder(): assert temp[i]['bboxes'].shape == torch.Size([500, 9]) assert temp[i]['scores'].shape == torch.Size([500]) assert temp[i]['labels'].shape == torch.Size([500]) + + +def test_point_xyzwhlr_bbox_coder(): + bbox_coder_cfg = dict( + type='PointXYZWHLRBBoxCoder', + use_mean_size=True, + mean_size=[[3.9, 1.6, 1.56], [0.8, 0.6, 1.73], [1.76, 0.6, 1.73]]) + boxcoder = build_bbox_coder(bbox_coder_cfg) + + # test encode + gt_bboxes_3d = torch.tensor( + [[13.3329, 2.3514, -0.7004, 1.7508, 0.4702, 1.7909, -3.0522], + [2.2068, -2.6994, -0.3277, 3.8703, 1.6602, 1.6913, -1.9057], + [5.5269, 2.5085, -1.0129, 1.1496, 0.8006, 1.8887, 2.1756]]) + + points = torch.tensor([[13.70, 2.40, 0.12], [3.20, -3.00, 0.2], + [5.70, 2.20, -0.4]]) + + gt_labels_3d = torch.tensor([2, 0, 1]) + + bbox_target = boxcoder.encode(gt_bboxes_3d, points, gt_labels_3d) + expected_bbox_target = torch.tensor([[ + -0.1974, -0.0261, -0.4742, -0.0052, -0.2438, 0.0346, -0.9960, -0.0893 + ], [-0.2356, 0.0713, -0.3383, -0.0076, 0.0369, 0.0808, -0.3287, -0.9444 + ], [-0.1731, 0.3085, -0.3543, 0.3626, 0.2884, 0.0878, -0.5686, + 0.8226]]) + assert torch.allclose(expected_bbox_target, bbox_target, atol=1e-4) + # test decode + bbox3d_out = boxcoder.decode(bbox_target, points, gt_labels_3d) + assert torch.allclose(bbox3d_out, gt_bboxes_3d, atol=1e-4) From 338923770c23622c792be83edea2d5619e6a13b3 Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Fri, 13 Aug 2021 10:50:31 +0800 Subject: [PATCH 08/51] [Enhance] Change Groupfree3D config (#855) * All mods * PointSample * PointSample --- configs/groupfree3d/README.md | 8 ++++---- ...roupfree3d_8x4_scannet-3d-18class-L12-O256.py | 14 +++----------- ...groupfree3d_8x4_scannet-3d-18class-L6-O256.py | 14 +++----------- ...free3d_8x4_scannet-3d-18class-w2x-L12-O256.py | 14 +++----------- ...free3d_8x4_scannet-3d-18class-w2x-L12-O512.py | 14 +++----------- configs/groupfree3d/metafile.yml | 16 ++++++++-------- 6 files changed, 24 insertions(+), 56 deletions(-) diff --git a/configs/groupfree3d/README.md b/configs/groupfree3d/README.md index 8bd4238245..2dd01f1cc9 100644 --- a/configs/groupfree3d/README.md +++ b/configs/groupfree3d/README.md @@ -21,10 +21,10 @@ We implement Group-Free-3D and provide the result and checkpoints on ScanNet dat | Method | Backbone | Lr schd | Mem (GB) | Inf time (fps) | AP@0.25 |AP@0.5| Download | | :------: | :---------: | :-----: | :------: | :------------: | :----: |:----: | :------: | -| [L6, O256](./groupfree3d_8x4_scannet-3d-18class-L6-O256.py ) | PointNet++ | 3x |6.7||65.59 (65.67*)|48.43 (47.74*)|[model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347-3499eb55.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347.log.json)| -| [L12, O256](./groupfree3d_8x4_scannet-3d-18class-L12-O256.py ) | PointNet++ | 3x |9.4||67.68 (66.22*)|49.30 (48.95*)|[model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256/groupfree3d_8x4_scannet-3d-18class-L12-O256_20210702_150907-1c5551ad.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256/groupfree3d_8x4_scannet-3d-18class-L12-O256_20210702_150907.log.json)| -| [L12, O256](./groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256.py ) | PointNet++w2x | 3x |13.3||67.09 (67.30*)|50.76 (50.44*)|[model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256_20210702_200301-944f0ac0.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256_20210702_200301.log.json)| -| [L12, O512](./groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512.py ) | PointNet++w2x | 3x |18.8||68.31 (68.20*)|51.73 (51.31*)|[model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512_20210702_220204-187b71c7.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512_20210702_220204.log.json)| +| [L6, O256](./groupfree3d_8x4_scannet-3d-18class-L6-O256.py ) | PointNet++ | 3x |6.7||66.32 (65.67*)|47.82 (47.74*)|[model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347-3499eb55.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347.log.json)| +| [L12, O256](./groupfree3d_8x4_scannet-3d-18class-L12-O256.py ) | PointNet++ | 3x |9.4||66.57 (66.22*)|48.21 (48.95*)|[model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256/groupfree3d_8x4_scannet-3d-18class-L12-O256_20210702_150907-1c5551ad.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256/groupfree3d_8x4_scannet-3d-18class-L12-O256_20210702_150907.log.json)| +| [L12, O256](./groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256.py ) | PointNet++w2x | 3x |13.3||68.20 (67.30*)|51.02 (50.44*)|[model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256_20210702_200301-944f0ac0.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256_20210702_200301.log.json)| +| [L12, O512](./groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512.py ) | PointNet++w2x | 3x |18.8||68.22 (68.20*)|52.61 (51.31*)|[model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512_20210702_220204-187b71c7.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512_20210702_220204.log.json)| **Notes:** diff --git a/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256.py b/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256.py index 1b58971107..987bcec679 100644 --- a/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256.py +++ b/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256.py @@ -147,7 +147,7 @@ workers_per_gpu=4, train=dict( type='RepeatDataset', - times=1, + times=5, dataset=dict( type=dataset_type, data_root=data_root, @@ -192,16 +192,8 @@ })) optimizer_config = dict(grad_clip=dict(max_norm=0.1, norm_type=2)) -lr_config = dict(policy='step', warmup=None, step=[280, 340]) +lr_config = dict(policy='step', warmup=None, step=[56, 68]) # runtime settings -runner = dict(type='EpochBasedRunner', max_epochs=400) +runner = dict(type='EpochBasedRunner', max_epochs=80) checkpoint_config = dict(interval=1, max_keep_ckpts=10) -# yapf:disable -log_config = dict( - interval=30, - hooks=[ - dict(type='TextLoggerHook'), - dict(type='TensorboardLoggerHook') - ]) -# yapf:enable diff --git a/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py b/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py index 1970d91385..62821293fc 100644 --- a/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py +++ b/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py @@ -146,7 +146,7 @@ workers_per_gpu=4, train=dict( type='RepeatDataset', - times=1, + times=5, dataset=dict( type=dataset_type, data_root=data_root, @@ -191,16 +191,8 @@ })) optimizer_config = dict(grad_clip=dict(max_norm=0.1, norm_type=2)) -lr_config = dict(policy='step', warmup=None, step=[280, 340]) +lr_config = dict(policy='step', warmup=None, step=[56, 68]) # runtime settings -runner = dict(type='EpochBasedRunner', max_epochs=400) +runner = dict(type='EpochBasedRunner', max_epochs=80) checkpoint_config = dict(interval=1, max_keep_ckpts=10) -# yapf:disable -log_config = dict( - interval=30, - hooks=[ - dict(type='TextLoggerHook'), - dict(type='TensorboardLoggerHook') - ]) -# yapf:enable diff --git a/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256.py b/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256.py index d903146c5a..8551b7401e 100644 --- a/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256.py +++ b/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256.py @@ -162,7 +162,7 @@ workers_per_gpu=4, train=dict( type='RepeatDataset', - times=1, + times=5, dataset=dict( type=dataset_type, data_root=data_root, @@ -207,16 +207,8 @@ })) optimizer_config = dict(grad_clip=dict(max_norm=0.1, norm_type=2)) -lr_config = dict(policy='step', warmup=None, step=[280, 340]) +lr_config = dict(policy='step', warmup=None, step=[56, 68]) # runtime settings -runner = dict(type='EpochBasedRunner', max_epochs=400) +runner = dict(type='EpochBasedRunner', max_epochs=80) checkpoint_config = dict(interval=1, max_keep_ckpts=10) -# yapf:disable -log_config = dict( - interval=30, - hooks=[ - dict(type='TextLoggerHook'), - dict(type='TensorboardLoggerHook') - ]) -# yapf:enable diff --git a/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512.py b/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512.py index d22a57297b..199e08bf1a 100644 --- a/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512.py +++ b/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512.py @@ -163,7 +163,7 @@ workers_per_gpu=4, train=dict( type='RepeatDataset', - times=1, + times=5, dataset=dict( type=dataset_type, data_root=data_root, @@ -208,16 +208,8 @@ })) optimizer_config = dict(grad_clip=dict(max_norm=0.1, norm_type=2)) -lr_config = dict(policy='step', warmup=None, step=[280, 340]) +lr_config = dict(policy='step', warmup=None, step=[56, 68]) # runtime settings -runner = dict(type='EpochBasedRunner', max_epochs=400) +runner = dict(type='EpochBasedRunner', max_epochs=80) checkpoint_config = dict(interval=1, max_keep_ckpts=10) -# yapf:disable -log_config = dict( - interval=30, - hooks=[ - dict(type='TextLoggerHook'), - dict(type='TensorboardLoggerHook') - ]) -# yapf:enable diff --git a/configs/groupfree3d/metafile.yml b/configs/groupfree3d/metafile.yml index d0f94cca27..b369f1b0e0 100644 --- a/configs/groupfree3d/metafile.yml +++ b/configs/groupfree3d/metafile.yml @@ -20,8 +20,8 @@ Models: - Task: 3D Object Detection Dataset: ScanNet Metrics: - AP@0.25: 65.59 - AP@0.5: 48.43 + AP@0.25: 66.32 + AP@0.5: 47.82 Weights: https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256/groupfree3d_8x4_scannet-3d-18class-L6-O256_20210702_145347-3499eb55.pth - Name: groupfree3d_8x4_scannet-3d-18class-L12-O256.py @@ -34,8 +34,8 @@ Models: - Task: 3D Object Detection Dataset: ScanNet Metrics: - AP@0.25: 67.68 - AP@0.5: 49.30 + AP@0.25: 66.57 + AP@0.5: 48.21 Weights: https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L12-O256/groupfree3d_8x4_scannet-3d-18class-L12-O256_20210702_150907-1c5551ad.pth - Name: groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256.py @@ -48,8 +48,8 @@ Models: - Task: 3D Object Detection Dataset: ScanNet Metrics: - AP@0.25: 67.09 - AP@0.5: 50.76 + AP@0.25: 68.20 + AP@0.5: 51.02 Weights: https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O256_20210702_200301-944f0ac0.pth - Name: groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512.py @@ -62,6 +62,6 @@ Models: - Task: 3D Object Detection Dataset: ScanNet Metrics: - AP@0.25: 68.31 - AP@0.5: 51.73 + AP@0.25: 68.22 + AP@0.5: 52.61 Weights: https://download.openmmlab.com/mmdetection3d/v0.1.0_models/groupfree3d/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512/groupfree3d_8x4_scannet-3d-18class-w2x-L12-O512_20210702_220204-187b71c7.pth From fc4bb0c71f8c70afa6d3f45fbfe9943068c153bb Mon Sep 17 00:00:00 2001 From: Wenhao Wu <79644370+wHao-Wu@users.noreply.github.com> Date: Wed, 18 Aug 2021 23:02:14 +0800 Subject: [PATCH 09/51] [Doc] Add tutorials/data_pipeline Chinese version (#827) * [Doc] Add tutorials/data_pipeline Chinese version * refine doc * Use the absolute link * Use the absolute link Co-authored-by: Tai-Wang --- docs/tutorials/data_pipeline.md | 4 +- docs_zh-CN/tutorials/data_pipeline.md | 178 +++++++++++++++++++++++++- 2 files changed, 179 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/data_pipeline.md b/docs/tutorials/data_pipeline.md index 16f6ce48ba..57167f67b9 100644 --- a/docs/tutorials/data_pipeline.md +++ b/docs/tutorials/data_pipeline.md @@ -5,7 +5,7 @@ Following typical conventions, we use `Dataset` and `DataLoader` for data loading with multiple workers. `Dataset` returns a dict of data items corresponding the arguments of models' forward method. -Since the data in object detection may not be the same size (image size, gt bbox size, etc.), +Since the data in object detection may not be the same size (point number, gt bbox size, etc.), we introduce a new `DataContainer` type in MMCV to help collect and distribute data of different size. See [here](https://github.com/open-mmlab/mmcv/blob/master/mmcv/parallel/data_container.py) for more details. @@ -15,7 +15,7 @@ defines how to process the annotations and a data pipeline defines all the steps A pipeline consists of a sequence of operations. Each operation takes a dict as input and also output a dict for the next transform. We present a classical pipeline in the following figure. The blue blocks are pipeline operations. With the pipeline going on, each operator can add new keys (marked as green) to the result dict or update the existing keys (marked as orange). -![](../../resources/data_pipeline.png) +![](https://github.com/open-mmlab/mmdetection3d/blob/master/resources/data_pipeline.png) The operations are categorized into data loading, pre-processing, formatting and test-time augmentation. diff --git a/docs_zh-CN/tutorials/data_pipeline.md b/docs_zh-CN/tutorials/data_pipeline.md index d3100a191a..f8a613134e 100644 --- a/docs_zh-CN/tutorials/data_pipeline.md +++ b/docs_zh-CN/tutorials/data_pipeline.md @@ -1 +1,177 @@ -# 教程 3: 自定义数据预处理流程 \ No newline at end of file +# 教程 3: 自定义数据预处理流程 + +## 数据预处理流程的设计 + +遵循一般惯例，我们使用 `Dataset` 和 `DataLoader` 来调用多个进程进行数据的加载。`Dataset` 将会返回与模型前向传播的参数所对应的数据项构成的字典。因为目标检测中的数据的尺寸可能无法保持一致（如点云中点的数量、真实标注框的尺寸等），我们在 MMCV 中引入一个 `DataContainer` 类型，来帮助收集和分发不同尺寸的数据。请参考[此处](https://github.com/open-mmlab/mmcv/blob/master/mmcv/parallel/data_container.py)获取更多细节。 + +数据预处理流程和数据集之间是互相分离的两个部分，通常数据集定义了如何处理标注信息，而数据预处理流程定义了准备数据项字典的所有步骤。数据集预处理流程包含一系列的操作，每个操作将一个字典作为输入，并输出应用于下一个转换的一个新的字典。 + +我们将在下图中展示一个最经典的数据集预处理流程，其中蓝色框表示预处理流程中的各项操作。随着预处理的进行，每一个操作都会添加新的键值（图中标记为绿色）到输出字典中，或者更新当前存在的键值（图中标记为橙色）。 +![](https://github.com/open-mmlab/mmdetection3d/blob/master/resources/data_pipeline.png) + +预处理流程中的各项操作主要分为数据加载、预处理、格式化、测试时的数据增强。 + +接下来将展示一个用于 PointPillars 模型的数据集预处理流程的例子。 + +```python +train_pipeline = [ + dict( + type='LoadPointsFromFile', + load_dim=5, + use_dim=5, + file_client_args=file_client_args), + dict( + type='LoadPointsFromMultiSweeps', + sweeps_num=10, + file_client_args=file_client_args), + dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True), + dict( + type='GlobalRotScaleTrans', + rot_range=[-0.3925, 0.3925], + scale_ratio_range=[0.95, 1.05], + translation_std=[0, 0, 0]), + dict(type='RandomFlip3D', flip_ratio_bev_horizontal=0.5), + dict(type='PointsRangeFilter', point_cloud_range=point_cloud_range), + dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range), + dict(type='ObjectNameFilter', classes=class_names), + dict(type='PointShuffle'), + dict(type='DefaultFormatBundle3D', class_names=class_names), + dict(type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d']) +] +test_pipeline = [ + dict( + type='LoadPointsFromFile', + load_dim=5, + use_dim=5, + file_client_args=file_client_args), + dict( + type='LoadPointsFromMultiSweeps', + sweeps_num=10, + file_client_args=file_client_args), + dict( + type='MultiScaleFlipAug', + img_scale=(1333, 800), + pts_scale_ratio=1.0, + flip=False, + pcd_horizontal_flip=False, + pcd_vertical_flip=False, + transforms=[ + dict( + type='GlobalRotScaleTrans', + rot_range=[0, 0], + scale_ratio_range=[1., 1.], + translation_std=[0, 0, 0]), + dict(type='RandomFlip3D'), + dict( + type='PointsRangeFilter', point_cloud_range=point_cloud_range), + dict( + type='DefaultFormatBundle3D', + class_names=class_names, + with_label=False), + dict(type='Collect3D', keys=['points']) + ]) +] +``` + +对于每项操作，我们将列出相关的被添加/更新/移除的字典项。 + +### 数据加载 + +`LoadPointsFromFile` +- 添加：points + +`LoadPointsFromMultiSweeps` +- 更新：points + +`LoadAnnotations3D` +- 添加：gt_bboxes_3d, gt_labels_3d, gt_bboxes, gt_labels, pts_instance_mask, pts_semantic_mask, bbox3d_fields, pts_mask_fields, pts_seg_fields + +### 预处理 + +`GlobalRotScaleTrans` +- 添加：pcd_trans, pcd_rotation, pcd_scale_factor +- 更新：points, *bbox3d_fields + +`RandomFlip3D` +- 添加：flip, pcd_horizontal_flip, pcd_vertical_flip +- 更新：points, *bbox3d_fields + +`PointsRangeFilter` +- 更新：points + +`ObjectRangeFilter` +- 更新：gt_bboxes_3d, gt_labels_3d + +`ObjectNameFilter` +- 更新：gt_bboxes_3d, gt_labels_3d + +`PointShuffle` +- 更新：points + +`PointsRangeFilter` +- 更新：points + +### 格式化 + +`DefaultFormatBundle3D` +- 更新：points, gt_bboxes_3d, gt_labels_3d, gt_bboxes, gt_labels + +`Collect3D` +- 添加：img_meta （由 `meta_keys` 指定的键值构成的 img_meta） +- 移除：所有除 `keys` 指定的键值以外的其他键值 + +### 测试时的数据增强 + +`MultiScaleFlipAug` +- 更新: scale, pcd_scale_factor, flip, flip_direction, pcd_horizontal_flip, pcd_vertical_flip （与这些指定的参数对应的增强后的数据列表） + +## 扩展并使用自定义数据集预处理方法 + +1. 在任意文件中写入新的数据集预处理方法，如 `my_pipeline.py`，该预处理方法的输入和输出均为字典 + + ```python + from mmdet.datasets import PIPELINES + + @PIPELINES.register_module() + class MyTransform: + + def __call__(self, results): + results['dummy'] = True + return results + ``` + +2. 导入新的预处理方法类 + + ```python + from .my_pipeline import MyTransform + ``` + +3. 在配置文件中使用该数据集预处理方法 + + ```python + train_pipeline = [ + dict( + type='LoadPointsFromFile', + load_dim=5, + use_dim=5, + file_client_args=file_client_args), + dict( + type='LoadPointsFromMultiSweeps', + sweeps_num=10, + file_client_args=file_client_args), + dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True), + dict( + type='GlobalRotScaleTrans', + rot_range=[-0.3925, 0.3925], + scale_ratio_range=[0.95, 1.05], + translation_std=[0, 0, 0]), + dict(type='RandomFlip3D', flip_ratio_bev_horizontal=0.5), + dict(type='PointsRangeFilter', point_cloud_range=point_cloud_range), + dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range), + dict(type='ObjectNameFilter', classes=class_names), + dict(type='MyTransform'), + dict(type='PointShuffle'), + dict(type='DefaultFormatBundle3D', class_names=class_names), + dict(type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d']) + ] + ``` From 0f81e498a09d8e91435d775db265fca202ad77fb Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Wed, 18 Aug 2021 23:02:57 +0800 Subject: [PATCH 10/51] [Doc] Add Chinese doc for `scannet_det.md` (#836) * Part * Complete * Fix comments * Fix comments --- docs/datasets/scannet_det.md | 40 ++-- docs/datasets/sunrgbd_det.md | 2 +- docs_zh-CN/datasets/scannet_det.md | 303 +++++++++++++++++++++++++++++ docs_zh-CN/datasets/sunrgbd_det.md | 16 +- 4 files changed, 333 insertions(+), 28 deletions(-) diff --git a/docs/datasets/scannet_det.md b/docs/datasets/scannet_det.md index b8075b3590..6e35dfd9d4 100644 --- a/docs/datasets/scannet_det.md +++ b/docs/datasets/scannet_det.md @@ -33,10 +33,10 @@ mmdetection3d Under folder `scans` there are overall 1201 train and 312 validation folders in which raw point cloud data and relevant annotations are saved. For instance, under folder `scene0001_01` the files are as below: - `scene0001_01_vh_clean_2.ply`: Mesh file storing coordinates and colors of each vertex. The mesh's vertices are taken as raw point cloud data. -- `scene0001_01.aggregation.json`: Aggregation file including object id, segments id and label. -- `scene0001_01_vh_clean_2.0.010000.segs.json`: Segmentation file including segments id and vertex. +- `scene0001_01.aggregation.json`: Aggregation file including object ID, segments ID and label. +- `scene0001_01_vh_clean_2.0.010000.segs.json`: Segmentation file including segments ID and vertex. - `scene0001_01.txt`: Meta file including axis-aligned matrix, etc. -- `scene0001_01_vh_clean_2.labels.ply` +- `scene0001_01_vh_clean_2.labels.ply`: Annotation file containing the category of each vertex. Export ScanNet data by running `python batch_load_scannet_data.py`. The main steps include: @@ -79,7 +79,7 @@ def export(mesh_file, # perform global alignment of mesh vertices pts = np.ones((mesh_vertices.shape[0], 4)) - # raw point cloud in homogeneous coordinats, each row: [x, y, z, 1] + # raw point cloud in homogeneous coordinates, each row: [x, y, z, 1] pts[:, 0:3] = mesh_vertices[:, 0:3] # transform raw mesh vertices to aligned mesh vertices pts = np.dot(pts, axis_align_matrix.transpose()) # Nx4 @@ -125,9 +125,9 @@ def export(mesh_file, ``` -After exporting each scan, the raw point cloud could be downsampled, e.g. to 50000, if the number of points is too large (the raw point cloud won't be downsampled if it's also used in 3d semantic segmentation task). In addition, invalid semantic labels outside of `nyu40id` standard or optional `DONOT CARE` classes should be filtered. Finally, the point cloud data, semantic labels, instance labels and ground truth bounding boxes should be saved in `.npy` files. +After exporting each scan, the raw point cloud could be downsampled, e.g. to 50000, if the number of points is too large (the raw point cloud won't be downsampled if it's also used in 3D semantic segmentation task). In addition, invalid semantic labels outside of `nyu40id` standard or optional `DONOT CARE` classes should be filtered. Finally, the point cloud data, semantic labels, instance labels and ground truth bounding boxes should be saved in `.npy` files. -### Export ScanNet RGB data +### Export ScanNet RGB data (optional) By exporting ScanNet RGB data, for each scene we load a set of RGB images with corresponding 4x4 pose matrices, and a single 4x4 camera intrinsic matrix. Note, that this step is optional and can be skipped if multi-view detection is not planned to use. @@ -135,7 +135,7 @@ By exporting ScanNet RGB data, for each scene we load a set of RGB images with c python extract_posed_images.py ``` -Each of 1201 train, 312 validation and 100 test scenes contains a single `.sens` file. For instance, for scene `0001_01` we have `data/scannet/scans/scene0001_01/0001_01.sens`. For this scene all images and poses are extracted to `data/scannet/posed_images/scene0001_01`. Specifically, there will be 300 image files xxxxx.jpg, 300 camera pose files xxxxx.txt and a single `intrinsic.txt` file. Typically, single scene contains several thousand images. By default, we extract only 300 of them with resulting weight of <100 Gb. To extract more images, use `--max-images-per-scene` parameter. +Each of 1201 train, 312 validation and 100 test scenes contains a single `.sens` file. For instance, for scene `0001_01` we have `data/scannet/scans/scene0001_01/0001_01.sens`. For this scene all images and poses are extracted to `data/scannet/posed_images/scene0001_01`. Specifically, there will be 300 image files xxxxx.jpg, 300 camera pose files xxxxx.txt and a single `intrinsic.txt` file. Typically, single scene contains several thousand images. By default, we extract only 300 of them with resulting space occupation of <100 Gb. To extract more images, use `--max-images-per-scene` parameter. ### Create dataset @@ -221,9 +221,9 @@ scannet ├── scannet_infos_test.pkl ``` -- `points/xxxxx.bin`: The `axis-unaligned` point cloud data after downsample. Since ScanNet 3D detection task takes axis-aligned point clouds as input, while ScanNet 3D semantic segmentation task takes unaligned points, we choose to store unaligned points and their axis-align transform matrix. Note: the points would be axis-aligned in pre-processing pipeline `GlobalAlignment` of 3D detection task. +- `points/xxxxx.bin`: The `axis-unaligned` point cloud data after downsample. Since ScanNet 3D detection task takes axis-aligned point clouds as input, while ScanNet 3D semantic segmentation task takes unaligned points, we choose to store unaligned points and their axis-align transform matrix. Note: the points would be axis-aligned in pre-processing pipeline [`GlobalAlignment`](https://github.com/open-mmlab/mmdetection3d/blob/9f0b01caf6aefed861ef4c3eb197c09362d26b32/mmdet3d/datasets/pipelines/transforms_3d.py#L423) of 3D detection task. - `instance_mask/xxxxx.bin`: The instance label for each point, value range: [0, NUM_INSTANCES], 0: unannotated. -- `semantic_mask/xxxxx.bin`: The semantic label for each point, value range: [1, 40], i.e. `nyu40id` standard. Note: the `nyu40id` id will be mapped to train id in train pipeline `PointSegClassMapping`. +- `semantic_mask/xxxxx.bin`: The semantic label for each point, value range: [1, 40], i.e. `nyu40id` standard. Note: the `nyu40id` ID will be mapped to train ID in train pipeline `PointSegClassMapping`. - `posed_images/scenexxxx_xx`: The set of `.jpg` images with `.txt` 4x4 poses and the single `.txt` file with camera intrinsic matrix. - `scannet_infos_train.pkl`: The train data infos, the detailed info of each scan is as follows: - info['point_cloud']: {'num_features': 6, 'lidar_idx': sample_idx}. @@ -233,14 +233,16 @@ scannet - info['annos']: The annotations of each scan. - annotations['gt_num']: The number of ground truths. - annotations['name']： The semantic name of all ground truths, e.g. `chair`. - - annotations['location']: The gravity center of the axis-aligned 3D bounding boxes. Shape: [K, 3], K is the number of ground truths. - - annotations['dimensions']: The dimensions of the axis-aligned 3D bounding boxes, i.e. (x_size, y_size, z_size), shape: [K, 3]. - - annotations['gt_boxes_upright_depth']: The axis-aligned 3D bounding boxes, each bounding box is (x, y, z, x_size, y_size, z_size), shape: [K, 6]. - - annotations['unaligned_location']: The gravity center of the axis-unaligned 3D bounding boxes. - - annotations['unaligned_dimensions']: The dimensions of the axis-unaligned 3D bounding boxes. - - annotations['unaligned_gt_boxes_upright_depth']: The axis-unaligned 3D bounding boxes. + - annotations['location']: The gravity center of the axis-aligned 3D bounding boxes in depth coordinate system. Shape: [K, 3], K is the number of ground truths. + - annotations['dimensions']: The dimensions of the axis-aligned 3D bounding boxes in depth coordinate system, i.e. (x_size, y_size, z_size), shape: [K, 3]. + - annotations['gt_boxes_upright_depth']: The axis-aligned 3D bounding boxes in depth coordinate system, each bounding box is (x, y, z, x_size, y_size, z_size), shape: [K, 6]. + - annotations['unaligned_location']: The gravity center of the axis-unaligned 3D bounding boxes in depth coordinate system. + - annotations['unaligned_dimensions']: The dimensions of the axis-unaligned 3D bounding boxes in depth coordinate system. + - annotations['unaligned_gt_boxes_upright_depth']: The axis-unaligned 3D bounding boxes in depth coordinate system. - annotations['index']: The index of all ground truths, i.e. [0, K). - - annotations['class']: The train class id of the bounding boxes, value range: [0, 18), shape: [K, ]. + - annotations['class']: The train class ID of the bounding boxes, value range: [0, 18), shape: [K, ]. +- `scannet_infos_val.pkl`: The val data infos, which shares the same format as `scannet_infos_train.pkl`. +- `scannet_infos_test.pkl`: The test data infos, which almost shares the same format as `scannet_infos_train.pkl` except for the lack of annotation. ## Training pipeline @@ -289,14 +291,14 @@ train_pipeline = [ ``` - `GlobalAlignment`: The previous point cloud would be axis-aligned using the axis-aligned matrix. -- `PointSegClassMapping`: Only the valid category ids will be mapped to class label ids like [0, 18) during training. +- `PointSegClassMapping`: Only the valid category IDs will be mapped to class label IDs like [0, 18) during training. - Data augmentation: - `PointSample`: downsample the input point cloud. - `RandomFlip3D`: randomly flip the input point cloud horizontally or vertically. - - `GlobalRotScaleTrans`: rotate the input point cloud, usually in the range of [-5, 5] (degrees) for ScanNet; then scale the input point cloud, usually by 1.0 for ScanNet; finally translate the input point cloud, usually by 0 for ScanNet. + - `GlobalRotScaleTrans`: rotate the input point cloud, usually in the range of [-5, 5] (degrees) for ScanNet; then scale the input point cloud, usually by 1.0 for ScanNet (which means no scaling); finally translate the input point cloud, usually by 0 for ScanNet (which means no translation). ## Metrics Typically mean Average Precision (mAP) is used for evaluation on ScanNet, e.g. `mAP@0.25` and `mAP@0.5`. In detail, a generic function to compute precision and recall for 3D object detection for multiple classes is called, please refer to [indoor_eval](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3D/core/evaluation/indoor_eval.py). -As introduced in section `Export ScanNet data`, all ground truth 3D bounding box are axis-aligned, i.e. the yaw is zero. So the yaw target of network predicted 3D bounding box is also zero and axis-aligned 3D non-maximum suppression (NMS) is adopted during post-processing without reagrd to rotation. +As introduced in section `Export ScanNet data`, all ground truth 3D bounding box are axis-aligned, i.e. the yaw is zero. So the yaw target of network predicted 3D bounding box is also zero and axis-aligned 3D Non-Maximum Suppression (NMS), which is regardless of rotation, is adopted during post-processing . diff --git a/docs/datasets/sunrgbd_det.md b/docs/datasets/sunrgbd_det.md index 2a8e95d628..ccb4511b70 100644 --- a/docs/datasets/sunrgbd_det.md +++ b/docs/datasets/sunrgbd_det.md @@ -290,7 +290,7 @@ train_pipeline = [ Data augmentation for point clouds: - `RandomFlip3D`: randomly flip the input point cloud horizontally or vertically. -- `GlobalRotScaleTrans`: rotate the input point cloud, usually in the range of [-30, 30] (degrees) for SUN RGB-D; then scale the input point cloud, usually in the range of [0.85, 1.15] for SUN RGB-D; finally translate the input point cloud, usually by 0 for SUN RGB-D. +- `GlobalRotScaleTrans`: rotate the input point cloud, usually in the range of [-30, 30] (degrees) for SUN RGB-D; then scale the input point cloud, usually in the range of [0.85, 1.15] for SUN RGB-D; finally translate the input point cloud, usually by 0 for SUN RGB-D (which means no translation). - `PointSample`: downsample the input point cloud. A typical train pipeline of SUN RGB-D for multi-modality (point cloud and image) 3D detection is as follows. diff --git a/docs_zh-CN/datasets/scannet_det.md b/docs_zh-CN/datasets/scannet_det.md index 3fcc137595..0052bded88 100644 --- a/docs_zh-CN/datasets/scannet_det.md +++ b/docs_zh-CN/datasets/scannet_det.md @@ -1 +1,304 @@ # 3D 目标检测 Scannet 数据集 + +## 数据集准备 + +请参考 ScanNet 的[指南](https://github.com/open-mmlab/mmdetection3d/blob/master/data/scannet/README.md/)以查看总体流程。 + +### 提取 ScanNet 点云数据 + +通过提取 ScanNet 数据，我们加载原始点云文件，并生成包括语义标签、实例标签和真实物体包围框在内的相关标注。 + +```shell +python batch_load_scannet_data.py +``` + +数据处理之前的文件目录结构如下： + +``` +mmdetection3d +├── mmdet3d +├── tools +├── configs +├── data +│ ├── scannet +│ │ ├── meta_data +│ │ ├── scans +│ │ │ ├── scenexxxx_xx +│ │ ├── batch_load_scannet_data.py +│ │ ├── load_scannet_data.py +│ │ ├── scannet_utils.py +│ │ ├── README.md +``` + +在 `scans` 文件夹下总共有 1201 个训练样本文件夹和 312 个验证样本文件夹，其中存有未处理的点云数据和相关的标注。比如说，在文件夹 `scene0001_01` 下文件是这样组织的： + +- `scene0001_01_vh_clean_2.ply`: 存有每个顶点坐标和颜色的网格文件。网格的顶点被直接用作未处理的点云数据。 +- `scene0001_01.aggregation.json`: 包含物体 ID、分割部分 ID、标签的标注文件。 +- `scene0001_01_vh_clean_2.0.010000.segs.json`: 包含分割部分 ID 和顶点的分割标注文件。 +- `scene0001_01.txt`: 包括对齐矩阵等的元文件。 +- `scene0001_01_vh_clean_2.labels.ply`：包含每个顶点类别的标注文件。 + +通过运行 `python batch_load_scannet_data.py` 来提取 ScanNet 数据。主要步骤包括： + +- 从原始文件中提取出点云、实例标签、语义标签和包围框标签文件。 +- 下采样原始点云并过滤掉不合法的类别。 +- 保存处理后的点云数据和相关的标注文件。 + +`load_scannet_data.py` 中的核心函数 `export` 如下： + +```python +def export(mesh_file, + agg_file, + seg_file, + meta_file, + label_map_file, + output_file=None, + test_mode=False): + + # 标签映射文件：./data/scannet/meta_data/scannetv2-labels.combined.tsv + # 该标签映射文件中有多种标签标准，比如 'nyu40id' + label_map = scannet_utils.read_label_mapping( + label_map_file, label_from='raw_category', label_to='nyu40id') + # 加载原始点云数据，特征包括6维：XYZRGB + mesh_vertices = scannet_utils.read_mesh_vertices_rgb(mesh_file) + + # 加载场景坐标轴对齐矩阵：一个 4x4 的变换矩阵 + # 将传感器坐标系下的原始点转化到另一个坐标系下 + # 该坐标系与房屋的两边平行（也就是与坐标轴平行） + lines = open(meta_file).readlines() + # 测试集的数据没有对齐矩阵 + axis_align_matrix = np.eye(4) + for line in lines: + if 'axisAlignment' in line: + axis_align_matrix = [ + float(x) + for x in line.rstrip().strip('axisAlignment = ').split(' ') + ] + break + axis_align_matrix = np.array(axis_align_matrix).reshape((4, 4)) + + # 对网格顶点进行全局的对齐 + pts = np.ones((mesh_vertices.shape[0], 4)) + # 同种类坐标下的原始点云，每一行的数据是 [x, y, z, 1] + pts[:, 0:3] = mesh_vertices[:, 0:3] + # 将原始网格顶点转换为对齐后的顶点 + pts = np.dot(pts, axis_align_matrix.transpose()) # Nx4 + aligned_mesh_vertices = np.concatenate([pts[:, 0:3], mesh_vertices[:, 3:]], + axis=1) + + # 加载语义与实例标签 + if not test_mode: + # 每个物体都有一个语义标签，并且包含几个分割部分 + object_id_to_segs, label_to_segs = read_aggregation(agg_file) + # 很多点属于同一分割部分 + seg_to_verts, num_verts = read_segmentation(seg_file) + label_ids = np.zeros(shape=(num_verts), dtype=np.uint32) + object_id_to_label_id = {} + for label, segs in label_to_segs.items(): + label_id = label_map[label] + for seg in segs: + verts = seg_to_verts[seg] + # 每个点都有一个语义标签 + label_ids[verts] = label_id + instance_ids = np.zeros( + shape=(num_verts), dtype=np.uint32) # 0：未标注的 + for object_id, segs in object_id_to_segs.items(): + for seg in segs: + verts = seg_to_verts[seg] + # object_id 从 1 开始计数，比如 1,2,3,.,,,.NUM_INSTANCES + # 每个点都属于一个物体 + instance_ids[verts] = object_id + if object_id not in object_id_to_label_id: + object_id_to_label_id[object_id] = label_ids[verts][0] + # 包围框格式为 [x, y, z, dx, dy, dz, label_id] + # [x, y, z] 是包围框的重力中心, [dx, dy, dz] 是与坐标轴平行的 + # [label_id] 是 'nyu40id' 标准下的语义标签 + # 注意：因为三维包围框是与坐标轴平行的，所以旋转角是 0 + unaligned_bboxes = extract_bbox(mesh_vertices, object_id_to_segs, + object_id_to_label_id, instance_ids) + aligned_bboxes = extract_bbox(aligned_mesh_vertices, object_id_to_segs, + object_id_to_label_id, instance_ids) + ... + + return mesh_vertices, label_ids, instance_ids, unaligned_bboxes, \ + aligned_bboxes, object_id_to_label_id, axis_align_matrix + +``` + +在从每个场景的扫描文件提取数据后，如果原始点云点数过多，可以将其下采样（比如到 50000 个点），但在三维语义分割任务中，点云不会被下采样。此外，在 `nyu40id` 标准之外的不合法语义标签或者可选的 `DONOT CARE` 类别标签应被过滤。最终，点云文件、语义标签、实例标签和真实物体的集合应被存储于 `.npy` 文件中。 + +### 提取 ScanNet RGB 色彩数据（可选的） + +通过提取 ScanNet RGB 色彩数据，对于每个场景我们加载 RGB 图像与配套 4x4 位姿矩阵、单个 4x4 相机内参矩阵的集合。请注意，这一步是可选的，除非要运行多视图物体检测，否则可以略去这步。 + +```shell +python extract_posed_images.py +``` + +1201 个训练样本，312 个验证样本和 100 个测试样本中的每一个都包含一个单独的 `.sens` 文件。比如说，对于场景 `0001_01` 我们有 `data/scannet/scans/scene0001_01/0001_01.sens`。对于这个场景所有图像和位姿数据都被提取至 `data/scannet/posed_images/scene0001_01`。具体来说，该文件夹下会有 300 个 xxxxx.jpg 格式的图像数据，300 个 xxxxx.txt 格式的相机位姿数据和一个单独的 `intrinsic.txt` 内参文件。通常来说，一个场景包含数千张图像。默认情况下，我们只会提取其中的 300 张，从而只占用少于 100 Gb 的空间。要想提取更多图像，请使用 `--max-images-per-scene` 参数。 + +### 创建数据集 + +```shell +python tools/create_data.py scannet --root-path ./data/scannet \ +--out-dir ./data/scannet --extra-tag scannet +``` + +上述提取的点云文件，语义类别标注文件，和物体实例标注文件被进一步以 `.bin` 格式保存。与此同时 `.pkl` 格式的文件被生成并用于训练和验证。获取数据信息的核心函数 `process_single_scene` 如下： + +```python +def process_single_scene(sample_idx): + + # 分别以 .bin 格式保存点云文件，语义类别标注文件和物体实例标注文件 + # 获取 info['pts_path']，info['pts_instance_mask_path'] 和 info['pts_semantic_mask_path'] + ... + + # 获取标注 + if has_label: + annotations = {} + # 包围框的形状为 [k, 6 + class] + aligned_box_label = self.get_aligned_box_label(sample_idx) + unaligned_box_label = self.get_unaligned_box_label(sample_idx) + annotations['gt_num'] = aligned_box_label.shape[0] + if annotations['gt_num'] != 0: + aligned_box = aligned_box_label[:, :-1] # k, 6 + unaligned_box = unaligned_box_label[:, :-1] + classes = aligned_box_label[:, -1] # k + annotations['name'] = np.array([ + self.label2cat[self.cat_ids2class[classes[i]]] + for i in range(annotations['gt_num']) + ]) + # 为了向后兼容，默认的参数名赋予了与坐标轴平行的包围框 + # 我们同时保存了对应的与坐标轴不平行的包围框的信息 + annotations['location'] = aligned_box[:, :3] + annotations['dimensions'] = aligned_box[:, 3:6] + annotations['gt_boxes_upright_depth'] = aligned_box + annotations['unaligned_location'] = unaligned_box[:, :3] + annotations['unaligned_dimensions'] = unaligned_box[:, 3:6] + annotations[ + 'unaligned_gt_boxes_upright_depth'] = unaligned_box + annotations['index'] = np.arange( + annotations['gt_num'], dtype=np.int32) + annotations['class'] = np.array([ + self.cat_ids2class[classes[i]] + for i in range(annotations['gt_num']) + ]) + axis_align_matrix = self.get_axis_align_matrix(sample_idx) + annotations['axis_align_matrix'] = axis_align_matrix # 4x4 + info['annos'] = annotations + return info +``` + +如上数据处理后，文件目录结构应如下： + +``` +scannet +├── meta_data +├── batch_load_scannet_data.py +├── load_scannet_data.py +├── scannet_utils.py +├── README.md +├── scans +├── scans_test +├── scannet_instance_data +├── points +│ ├── xxxxx.bin +├── instance_mask +│ ├── xxxxx.bin +├── semantic_mask +│ ├── xxxxx.bin +├── seg_info +│ ├── train_label_weight.npy +│ ├── train_resampled_scene_idxs.npy +│ ├── val_label_weight.npy +│ ├── val_resampled_scene_idxs.npy +├── posed_images +│ ├── scenexxxx_xx +│ │ ├── xxxxxx.txt +│ │ ├── xxxxxx.jpg +│ │ ├── intrinsic.txt +├── scannet_infos_train.pkl +├── scannet_infos_val.pkl +├── scannet_infos_test.pkl +``` + +- `points/xxxxx.bin`：下采样后，未与坐标轴平行（即没有对齐）的点云。因为 ScanNet 3D 检测任务将与坐标轴平行的点云作为输入，而 ScanNet 3D 语义分割任务将对齐前的点云作为输入，我们选择存储对齐前的点云和它们的对齐矩阵。请注意：在 3D 检测的预处理流程 [`GlobalAlignment`](https://github.com/open-mmlab/mmdetection3d/blob/9f0b01caf6aefed861ef4c3eb197c09362d26b32/mmdet3d/datasets/pipelines/transforms_3d.py#L423) 后，点云就都是与坐标轴平行的了。 +- `instance_mask/xxxxx.bin`：每个点的实例标签，值的范围为：[0, NUM_INSTANCES]，其中 0 表示没有标注。 +- `semantic_mask/xxxxx.bin`：每个点的语义标签，值的范围为：[1, 40], 也就是 `nyu40id` 的标准。请注意：在训练流程 `PointSegClassMapping` 中，`nyu40id` 的 ID 会被映射到训练 ID。 +- `posed_images/scenexxxx_xx`：`.jpg` 图像的集合，还包含 `.txt` 格式的 4x4 相机姿态和单个 `.txt` 格式的相机内参矩阵文件。 +- `scannet_infos_train.pkl`：训练集的数据信息，每个场景的具体信息如下： + - info['point_cloud']：`{'num_features': 6, 'lidar_idx': sample_idx}`，其中 `sample_idx` 为该场景的索引。 + - info['pts_path']：`points/xxxxx.bin` 的路径。 + - info['pts_instance_mask_path']：`instance_mask/xxxxx.bin` 的路径。 + - info['pts_semantic_mask_path']：`semantic_mask/xxxxx.bin` 的路径。 + - info['annos']：每个场景的标注。 + - annotations['gt_num']：真实物体 (ground truth) 的数量。 + - annotations['name']：所有真实物体的语义类别名称，比如 `chair`（椅子）。 + - annotations['location']：depth 坐标系下与坐标轴平行的三维包围框的重力中心 (gravity center)，形状为 [K, 3]，其中 K 是真实物体的数量。 + - annotations['dimensions']：depth 坐标系下与坐标轴平行的三维包围框的大小，形状为 [K, 3]。 + - annotations['gt_boxes_upright_depth']：depth 坐标系下与坐标轴平行的三维包围框 `(x, y, z, x_size, y_size, z_size, yaw)`，形状为 [K, 6]。 + - annotations['unaligned_location']：depth 坐标系下与坐标轴不平行（对齐前）的三维包围框的重力中心。 + - annotations['unaligned_dimensions']：depth 坐标系下与坐标轴不平行的三维包围框的大小。 + - annotations['unaligned_gt_boxes_upright_depth']：depth 坐标系下与坐标轴不平行的三维包围框。 + - annotations['index']：所有真实物体的索引，范围为 [0, K)。 + - annotations['class']：所有真实物体类别的标号，范围为 [0, 18)，形状为 [K, ]。 +- `scannet_infos_val.pkl`：验证集上的数据信息，与 `scannet_infos_train.pkl` 格式完全一致。 +- `scannet_infos_test.pkl`：测试集上的数据信息，与 `scannet_infos_train.pkl` 格式几乎完全一致，除了缺少标注。 + +## 训练流程 + +ScanNet 上 3D 物体检测的典型流程如下： + +```python +train_pipeline = [ + dict( + type='LoadPointsFromFile', + coord_type='DEPTH', + shift_height=True, + load_dim=6, + use_dim=[0, 1, 2]), + dict( + type='LoadAnnotations3D', + with_bbox_3d=True, + with_label_3d=True, + with_mask_3d=True, + with_seg_3d=True), + dict(type='GlobalAlignment', rotation_axis=2), + dict( + type='PointSegClassMapping', + valid_cat_ids=(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34, + 36, 39), + max_cat_id=40), + dict(type='PointSample', num_points=40000), + dict( + type='RandomFlip3D', + sync_2d=False, + flip_ratio_bev_horizontal=0.5, + flip_ratio_bev_vertical=0.5), + dict( + type='GlobalRotScaleTrans', + rot_range=[-0.087266, 0.087266], + scale_ratio_range=[1.0, 1.0], + shift_height=True), + dict(type='DefaultFormatBundle3D', class_names=class_names), + dict( + type='Collect3D', + keys=[ + 'points', 'gt_bboxes_3d', 'gt_labels_3d', 'pts_semantic_mask', + 'pts_instance_mask' + ]) +] +``` + +- `GlobalAlignment`：输入的点云在施加了坐标轴平行的矩阵后应被转换为与坐标轴平行的形式。 +- `PointSegClassMapping`：训练中，只有合法的类别 ID 才会被映射到类别标签，比如 [0, 18)。 +- 数据增强: + - `PointSample`：下采样输入点云。 + - `RandomFlip3D`：随机左右或前后翻转点云。 + - `GlobalRotScaleTrans`: 旋转输入点云，对于 ScanNet 角度通常落入 [-5, 5] （度）的范围；并放缩输入点云，对于 ScanNet 比例通常为 1.0（即不做缩放）；最后平移输入点云，对于 ScanNet 通常位移量为 0（即不做位移）。 + +## 评估指标 + +通常 mAP（全类平均精度）被用于 ScanNet 的检测任务的评估，比如 `mAP@0.25` 和 `mAP@0.5`。具体来说，评估时一个通用的计算 3D 物体检测多个类别的精度和召回率的函数被调用，可以参考 [indoor_eval](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3D/core/evaluation/indoor_eval.py)。 + +与在章节`提取 ScanNet 数据` 中介绍的那样，所有真实物体的三维包围框是与坐标轴平行的，也就是说旋转角为 0。因此，预测包围框的网络接受的包围框旋转角监督也是 0，且在后处理阶段我们使用适用于与坐标轴平行的包围框的非极大值抑制 (NMS) ，该过程不会考虑包围框的旋转。 diff --git a/docs_zh-CN/datasets/sunrgbd_det.md b/docs_zh-CN/datasets/sunrgbd_det.md index 0fc634e21e..c02023be1d 100644 --- a/docs_zh-CN/datasets/sunrgbd_det.md +++ b/docs_zh-CN/datasets/sunrgbd_det.md @@ -244,23 +244,23 @@ sunrgbd - info['image']：图像路径与元信息： - image['image_idx']：图像索引。 - image['image_shape']：图像张量的形状（即其尺寸）。 - - image['image_path']图像路径。 + - image['image_path']：图像路径。 - info['annos']：每个场景的标注： - - annotations['gt_num']：真实物体（ground truth）的数量 + - annotations['gt_num']：真实物体 (ground truth) 的数量。 - annotations['name']：所有真实物体的语义类别名称，比如 `chair`（椅子）。 - - annotations['location']：depth 坐标系下三维包围框的重力（gravity center）中心，形状为 [K, 3]，其中 K 是真实物体的数量。 + - annotations['location']：depth 坐标系下三维包围框的重力中心 (gravity center)，形状为 [K, 3]，其中 K 是真实物体的数量。 - annotations['dimensions']：depth 坐标系下三维包围框的大小，形状为 [K, 3]。 - annotations['rotation_y']：depth 坐标系下三维包围框的旋转角，形状为 [K, ]。 - annotations['gt_boxes_upright_depth']：depth 坐标系下三维包围框 `(x, y, z, x_size, y_size, z_size, yaw)`，形状为 [K, 7]。 - annotations['bbox']：二维包围框 `(x, y, x_size, y_size)`，形状为 [K, 4]。 - annotations['index']：所有真实物体的索引，范围为 [0, K)。 - - annotations['class']：所有真实物体类别的标号范围为 [0, 10)，形状为 [K, ]。 + - annotations['class']：所有真实物体类别的标号，范围为 [0, 10)，形状为 [K, ]。 - `sunrgbd_infos_val.pkl`：验证集上的数据信息，与 `sunrgbd_infos_train.pkl` 格式完全一致。 ## 训练流程 -SUN RGB-D 上纯点云 3D 物体检测的经典流程如下： +SUN RGB-D 上纯点云 3D 物体检测的典型流程如下： ```python train_pipeline = [ @@ -289,10 +289,10 @@ train_pipeline = [ 点云上的数据增强 - `RandomFlip3D`：随机左右或前后翻转输入点云。 -- `GlobalRotScaleTrans`：旋转输入点云，对于 SUN RGB-D 角度通常落入 [-30, 30] （度）的范围；并放缩输入点云，对于 SUN RGB-D 比例通常落入 [0.85, 1.15] 的范围；最后平移输入点云，对于 SUN RGB-D 通常位移量为 0。 +- `GlobalRotScaleTrans`：旋转输入点云，对于 SUN RGB-D 角度通常落入 [-30, 30] （度）的范围；并放缩输入点云，对于 SUN RGB-D 比例通常落入 [0.85, 1.15] 的范围；最后平移输入点云，对于 SUN RGB-D 通常位移量为 0（即不做位移）。 - `PointSample`：降采样输入点云。 -SUN RGB-D 上多模态（点云和图像）3D 物体检测的经典流程如下： +SUN RGB-D 上多模态（点云和图像）3D 物体检测的典型流程如下： ```python train_pipeline = [ @@ -340,6 +340,6 @@ train_pipeline = [ ## 度量指标 -与 ScanNet 一样，通常 mAP（全类平均精度）被用于SUN RGB-D 的检测任务的评估，比如 `mAP@0.25` 和 `mAP@0.5`。具体来说，评估时一个通用的计算 3D 物体检测多个类别的精度和召回率的函数被调用，可以参考 [`indoor_eval.py`](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/evaluation/indoor_eval.py)。 +与 ScanNet 一样，通常 mAP（全类平均精度）被用于 SUN RGB-D 的检测任务的评估，比如 `mAP@0.25` 和 `mAP@0.5`。具体来说，评估时一个通用的计算 3D 物体检测多个类别的精度和召回率的函数被调用，可以参考 [`indoor_eval.py`](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/evaluation/indoor_eval.py)。 因为 SUN RGB-D 包含有图像数据，所以图像上的物体检测也是可行的。举个例子，在 ImVoteNet 中，我们首先训练了一个图像检测器，并且也使用 mAP 指标，如 `mAP@0.5`，来评估其表现。我们使用 [MMDetection](https://github.com/open-mmlab/mmdetection) 库中的 `eval_map` 函数来计算 mAP。 From cac2ef8c3d0f8cff337f6e31a7e846e644fc9383 Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Wed, 18 Aug 2021 23:03:29 +0800 Subject: [PATCH 11/51] [Doc] Add Chinese doc for `waymo_det.md` (#859) * Add complete translation * Refinements * Fix comments * Fix a minor typo Co-authored-by: Tai-Wang --- docs/datasets/waymo_det.md | 26 ++--- docs_zh-CN/datasets/waymo_det.md | 171 ++++++++++++++++++++++++++++++- 2 files changed, 183 insertions(+), 14 deletions(-) diff --git a/docs/datasets/waymo_det.md b/docs/datasets/waymo_det.md index be7a91b757..60b6b90593 100644 --- a/docs/datasets/waymo_det.md +++ b/docs/datasets/waymo_det.md @@ -1,10 +1,10 @@ # Waymo Dataset -This page provides specific tutorials about the usage of MMDetection3D for waymo dataset. +This page provides specific tutorials about the usage of MMDetection3D for Waymo dataset. ## Prepare dataset -Before preparing waymo dataset, if you only installed requirements in `requirements/build.txt` and `requirements/runtime.txt` before, please install the official package for this dataset at first by running +Before preparing Waymo dataset, if you only installed requirements in `requirements/build.txt` and `requirements/runtime.txt` before, please install the official package for this dataset at first by running ``` # tf 2.1.0. @@ -22,7 +22,7 @@ pip install -r requirements/optional.txt ``` Like the general way to prepare dataset, it is recommended to symlink the dataset root to `$MMDETECTION3D/data`. -Due to the original waymo data format is based on `tfrecord`, we need to preprocess the raw data for convenient usage in the training and evaluation procedure. Our approach is to convert them into KITTI format. +Due to the original Waymo data format is based on `tfrecord`, we need to preprocess the raw data for convenient usage in the training and evaluation procedure. Our approach is to convert them into KITTI format. The folder structure should be organized as follows before our processing. @@ -43,13 +43,13 @@ mmdetection3d ``` -You can download Waymo open dataset V1.2 [HERE](https://waymo.com/open/download/) and its data split [HERE](https://drive.google.com/drive/folders/18BVuF_RYJF0NjZpt8SnfzANiakoRMf0o?usp=sharing). Then put tfrecord files into corresponding folders in `data/waymo/waymo_format/` and put the data split txt files into `data/waymo/kitti_format/ImageSets`. Download ground truth bin files for validation set [HERE](https://console.cloud.google.com/storage/browser/waymo_open_dataset_v_1_2_0/validation/ground_truth_objects) and put it into `data/waymo/waymo_format/`. A tip is that you can use `gsutil` to download the large-scale dataset with commands. You can take this [tool](https://github.com/RalphMao/Waymo-Dataset-Tool) as an example for more details. Subsequently, prepare waymo data by running +You can download Waymo open dataset V1.2 [HERE](https://waymo.com/open/download/) and its data split [HERE](https://drive.google.com/drive/folders/18BVuF_RYJF0NjZpt8SnfzANiakoRMf0o?usp=sharing). Then put `tfrecord` files into corresponding folders in `data/waymo/waymo_format/` and put the data split txt files into `data/waymo/kitti_format/ImageSets`. Download ground truth bin files for validation set [HERE](https://console.cloud.google.com/storage/browser/waymo_open_dataset_v_1_2_0/validation/ground_truth_objects) and put it into `data/waymo/waymo_format/`. A tip is that you can use `gsutil` to download the large-scale dataset with commands. You can take this [tool](https://github.com/RalphMao/Waymo-Dataset-Tool) as an example for more details. Subsequently, prepare Waymo data by running ```bash python tools/create_data.py waymo --root-path ./data/waymo/ --out-dir ./data/waymo/ --workers 128 --extra-tag waymo ``` -Note that if your local disk does not have enough space for saving converted data, you can change the `out-dir` to anywhere else. Just remember to create folders and prepare data there in advance and link them back to `data/waymo/kitti_format` after the data conversion. +Note that if your local disk does not have enough space for saving converted data, you can change the `--out-dir` to anywhere else. Just remember to create folders and prepare data there in advance and link them back to `data/waymo/kitti_format` after the data conversion. After the data conversion, the folder structure and info files should be organized as below. @@ -93,7 +93,7 @@ mmdetection3d ``` -Here because there are several cameras, we store the corresponding image and labels that can be projected to that camera respectively and save pose for further usage of consecutive frames point clouds. We use a coding way `{a}{bbb}{ccc}` to name the data for each frame, where `a` is the prefix for different split (`0` for training, `1` for validation and `2` for testing), `bbb` for segment index and `ccc` for frame index. You can easily locate the required frame according to this naming rule. We gather the data for training and validation together as KITTI and store the indices for different set in the ImageSet files. +Here because there are several cameras, we store the corresponding image and labels that can be projected to that camera respectively and save pose for further usage of consecutive frames point clouds. We use a coding way `{a}{bbb}{ccc}` to name the data for each frame, where `a` is the prefix for different split (`0` for training, `1` for validation and `2` for testing), `bbb` for segment index and `ccc` for frame index. You can easily locate the required frame according to this naming rule. We gather the data for training and validation together as KITTI and store the indices for different set in the `ImageSet` files. ## Training @@ -101,7 +101,7 @@ Considering there are many similar frames in the original dataset, we can basica ## Evaluation -For evaluation on waymo, please follow the [instruction](https://github.com/waymo-research/waymo-open-dataset/blob/master/docs/quick_start.md/) to build the binary file `compute_detection_metrics_main` for metrics computation and put it into `mmdet3d/core/evaluation/waymo_utils/`. Basically, you can follow the commands below to install bazel and build the file. +For evaluation on Waymo, please follow the [instruction](https://github.com/waymo-research/waymo-open-dataset/blob/master/docs/quick_start.md/) to build the binary file `compute_detection_metrics_main` for metrics computation and put it into `mmdet3d/core/evaluation/waymo_utils/`. Basically, you can follow the commands below to install `bazel` and build the file. ```shell git clone https://github.com/waymo-research/waymo-open-dataset.git waymo-od @@ -120,7 +120,7 @@ For evaluation on waymo, please follow the [instruction](https://github.com/waym cp bazel-bin/waymo_open_dataset/metrics/tools/compute_detection_metrics_main ../mmdetection3d/mmdet3d/core/evaluation/waymo_utils/ ``` -Then you can evaluate your models on waymo. An example to evaluate PointPillars on waymo with 8 GPUs with waymo metrics is as follows. +Then you can evaluate your models on Waymo. An example to evaluate PointPillars on Waymo with 8 GPUs with Waymo metrics is as follows. ```shell ./tools/slurm_test.sh ${PARTITION} ${JOB_NAME} configs/pointpillars/hv_pointpillars_secfpn_sbn-2x16_2x_waymo-3d-car.py \ @@ -129,19 +129,19 @@ Then you can evaluate your models on waymo. An example to evaluate PointPillars 'submission_prefix=results/waymo-car/kitti_results' ``` -`pklfile_prefix` should be given in the `--eval-options` if the bin file is needed to be generated. For metrics, `waymo` is the recommended official evaluation prototype. Currently, evaluating with choice `kitti` is adapted from KITTI and the results for each difficulty are not exactly the same as the definition of KITTI. Instead, most of objects are marked with difficulty 0 currently, which will be fixed in the future. The reasons of its instability include the large computation for evalution, the lack of occlusion and truncation in the converted data, different definitions of difficulty and different methods of computing average precision. +`pklfile_prefix` should be given in the `--eval-options` if the bin file is needed to be generated. For metrics, `waymo` is the recommended official evaluation prototype. Currently, evaluating with choice `kitti` is adapted from KITTI and the results for each difficulty are not exactly the same as the definition of KITTI. Instead, most of objects are marked with difficulty 0 currently, which will be fixed in the future. The reasons of its instability include the large computation for evalution, the lack of occlusion and truncation in the converted data, different definitions of difficulty and different methods of computing Average Precision. **Notice**: -1. Sometimes when using bazel to build `compute_detection_metrics_main`, an error `'round' is not a member of 'std'` may appear. We just need to remove the `std::` before `round` in that file. +1. Sometimes when using `bazel` to build `compute_detection_metrics_main`, an error `'round' is not a member of 'std'` may appear. We just need to remove the `std::` before `round` in that file. 2. Considering it takes a little long time to evaluate once, we recommend to evaluate only once at the end of model training. -3. To use tensorflow with cuda9, it is recommended to compile it from source. Apart from official tutorials, you can refer to this [link](https://github.com/SmileTM/Tensorflow2.X-GPU-CUDA9.0) for possibly suitable precompiled packages and useful information for compiling it from source. +3. To use TensorFlow with CUDA 9, it is recommended to compile it from source. Apart from official tutorials, you can refer to this [link](https://github.com/SmileTM/Tensorflow2.X-GPU-CUDA9.0) for possibly suitable precompiled packages and useful information for compiling it from source. ## Testing and make a submission -An example to test PointPillars on waymo with 8 GPUs, generate the bin files and make a submission to the leaderboard. +An example to test PointPillars on Waymo with 8 GPUs, generate the bin files and make a submission to the leaderboard. ```shell ./tools/slurm_test.sh ${PARTITION} ${JOB_NAME} configs/pointpillars/hv_pointpillars_secfpn_sbn-2x16_2x_waymo-3d-car.py \ @@ -167,4 +167,4 @@ After generating the bin file, you can simply build the binary file `create_subm gzip results/waymo-car/submission/my_model.tar ``` -For evaluation on the validation set with the eval server, you can also use the same way to generate a submission. Make sure you change the fields in submission.txtpb before running the command above. +For evaluation on the validation set with the eval server, you can also use the same way to generate a submission. Make sure you change the fields in `submission.txtpb` before running the command above. diff --git a/docs_zh-CN/datasets/waymo_det.md b/docs_zh-CN/datasets/waymo_det.md index 0fa64c7ebc..fe5e764ad3 100644 --- a/docs_zh-CN/datasets/waymo_det.md +++ b/docs_zh-CN/datasets/waymo_det.md @@ -1 +1,170 @@ -# Waymo数据集 \ No newline at end of file +# Waymo 数据集 + +本文档页包含了关于 MMDetection3D 中 Waymo 数据集用法的教程。 + +## 数据集准备 + +在准备 Waymo 数据集之前，如果您之前只安装了 `requirements/build.txt` 和 `requirements/runtime.txt` 中的依赖，请通过运行如下指令额外安装 Waymo 数据集所依赖的官方包： + +``` +# tf 2.1.0. +pip install waymo-open-dataset-tf-2-1-0==1.2.0 +# tf 2.0.0 +# pip install waymo-open-dataset-tf-2-0-0==1.2.0 +# tf 1.15.0 +# pip install waymo-open-dataset-tf-1-15-0==1.2.0 +``` + +或者 + +``` +pip install -r requirements/optional.txt +``` + +和准备数据集的通用方法一致，我们推荐将数据集根目录软链接至 `$MMDETECTION3D/data`。 +由于原始 Waymo 数据的格式基于 `tfrecord`，我们需要将原始数据进行预处理，以便于训练和测试时使用。我们的方法是将它们转换为 KITTI 格式。 + +处理之前，文件目录结构组织如下： + +``` +mmdetection3d +├── mmdet3d +├── tools +├── configs +├── data +│ ├── waymo +│ │ ├── waymo_format +│ │ │ ├── training +│ │ │ ├── validation +│ │ │ ├── testing +│ │ │ ├── gt.bin +│ │ ├── kitti_format +│ │ │ ├── ImageSets + +``` + +您可以在[这里](https://waymo.com/open/download/)下载 1.2 版本的 Waymo 公开数据集，并在[这里](https://drive.google.com/drive/folders/18BVuF_RYJF0NjZpt8SnfzANiakoRMf0o?usp=sharing)下载其训练/验证/测试集拆分文件。接下来，请将 `tfrecord` 文件放入 `data/waymo/waymo_format/` 下的对应文件夹，并将 txt 格式的数据集拆分文件放入 `data/waymo/kitti_format/ImageSets`。在[这里](https://console.cloud.google.com/storage/browser/waymo_open_dataset_v_1_2_0/validation/ground_truth_objects)下载验证集使用的 bin 格式真实标注 (Ground Truth) 文件并放入 `data/waymo/waymo_format/`。小窍门：您可以使用 `gsutil` 来在命令行下载大规模数据集。您可以将该[工具](https://github.com/RalphMao/Waymo-Dataset-Tool) 作为一个例子来查看更多细节。之后，通过运行如下指令准备 Waymo 数据： + +```bash +python tools/create_data.py waymo --root-path ./data/waymo/ --out-dir ./data/waymo/ --workers 128 --extra-tag waymo +``` + +请注意，如果您的本地磁盘没有足够空间保存转换后的数据，您可以将 `--out-dir` 改为其他目录；只要在创建文件夹、准备数据并转换格式后，将数据文件链接到 `data/waymo/kitti_format` 即可。 + +在数据转换后，文件目录结构应组织如下： + +``` +mmdetection3d +├── mmdet3d +├── tools +├── configs +├── data +│ ├── waymo +│ │ ├── waymo_format +│ │ │ ├── training +│ │ │ ├── validation +│ │ │ ├── testing +│ │ │ ├── gt.bin +│ │ ├── kitti_format +│ │ │ ├── ImageSets +│ │ │ ├── training +│ │ │ │ ├── calib +│ │ │ │ ├── image_0 +│ │ │ │ ├── image_1 +│ │ │ │ ├── image_2 +│ │ │ │ ├── image_3 +│ │ │ │ ├── image_4 +│ │ │ │ ├── label_0 +│ │ │ │ ├── label_1 +│ │ │ │ ├── label_2 +│ │ │ │ ├── label_3 +│ │ │ │ ├── label_4 +│ │ │ │ ├── label_all +│ │ │ │ ├── pose +│ │ │ │ ├── velodyne +│ │ │ ├── testing +│ │ │ │ ├── (the same as training) +│ │ │ ├── waymo_gt_database +│ │ │ ├── waymo_infos_trainval.pkl +│ │ │ ├── waymo_infos_train.pkl +│ │ │ ├── waymo_infos_val.pkl +│ │ │ ├── waymo_infos_test.pkl +│ │ │ ├── waymo_dbinfos_train.pkl + +``` + +因为 Waymo 数据的来源包含数个相机，这里我们将每个相机对应的图像和标签文件分别存储，并将相机位姿 (pose) 文件存储下来以供后续处理连续多帧的点云。我们使用 `{a}{bbb}{ccc}` 的名称编码方式为每帧数据命名，其中 `a` 是不同数据拆分的前缀（`0` 指代训练集，`1` 指代验证集，`2` 指代测试集），`bbb` 是分割部分 (segment) 的索引，而 `ccc` 是帧索引。您可以轻而易举地按照如上命名规则定位到所需的帧。我们将训练和验证所需数据按 KITTI 的方式集合在一起，然后将训练集/验证集/测试集的索引存储在 `ImageSet` 下的文件中。 + +## 训练 + +考虑到原始数据集中的数据有很多相似的帧，我们基本上可以主要使用一个子集来训练我们的模型。在我们初步的基线中，我们在每五帧图片中加载一帧。得益于我们的超参数设置和数据增强方案，我们得到了比 Waymo [原论文](https://arxiv.org/pdf/1912.04838.pdf)中更好的性能。请移步 `configs/pointpillars/` 下的 README.md 以查看更多配置和性能相关的细节。我们会尽快发布一个更完整的 Waymo 基准榜单 (benchmark)。 + +## 评估 + +为了在 Waymo 数据集上进行检测性能评估，请按照[此处指示](https://github.com/waymo-research/waymo-open-dataset/blob/master/docs/quick_start.md/)构建用于计算评估指标的二进制文件 `compute_detection_metrics_main`，并将它置于 `mmdet3d/core/evaluation/waymo_utils/` 下。您基本上可以按照下方命令安装 `bazel`，然后构建二进制文件： + + ```shell + git clone https://github.com/waymo-research/waymo-open-dataset.git waymo-od + cd waymo-od + git checkout remotes/origin/master + + sudo apt-get install --assume-yes pkg-config zip g++ zlib1g-dev unzip python3 python3-pip + wget https://github.com/bazelbuild/bazel/releases/download/0.28.0/bazel-0.28.0-installer-linux-x86_64.sh + sudo bash bazel-0.28.0-installer-linux-x86_64.sh + sudo apt install build-essential + + ./configure.sh + bazel clean + + bazel build waymo_open_dataset/metrics/tools/compute_detection_metrics_main + cp bazel-bin/waymo_open_dataset/metrics/tools/compute_detection_metrics_main ../mmdetection3d/mmdet3d/core/evaluation/waymo_utils/ + ``` + +接下来，您就可以在 Waymo 上评估您的模型了。如下示例是使用 8 个图形处理器 (GPU) 在 Waymo 上用 Waymo 评价指标评估 PointPillars 模型的情景： + + ```shell + ./tools/slurm_test.sh ${PARTITION} ${JOB_NAME} configs/pointpillars/hv_pointpillars_secfpn_sbn-2x16_2x_waymo-3d-car.py \ + checkpoints/hv_pointpillars_secfpn_sbn-2x16_2x_waymo-3d-car_latest.pth --out results/waymo-car/results_eval.pkl \ + --eval waymo --eval-options 'pklfile_prefix=results/waymo-car/kitti_results' \ + 'submission_prefix=results/waymo-car/kitti_results' + ``` + +如果需要生成 bin 文件，应在 `--eval-options` 中给出 `pklfile_prefix`。对于评价指标， `waymo` 是我们推荐的官方评估原型。目前，`kitti` 这一评估选项是从 KITTI 迁移而来的，且每个难度下的评估结果和 KITTI 数据集中定义得到的不尽相同——目前大多数物体被标记为难度 0（日后会修复）。`kitti` 评估选项的不稳定来源于很大的计算量，转换的数据中遮挡 (occlusion) 和截断 (truncation) 的缺失，难度的不同定义方式，以及不同的平均精度 (Average Precision) 计算方式。 + +**注意**: + +1. 有时用 `bazel` 构建 `compute_detection_metrics_main` 的过程中会出现如下错误：`'round' 不是 'std' 的成员` (`'round' is not a member of 'std'`)。我们只需要移除该文件中，`round` 前的 `std::`。 + +2. 考虑到 Waymo 上评估一次耗时不短，我们建议只在模型训练结束时进行评估。 + +3. 为了在 CUDA 9 环境使用 TensorFlow，我们建议通过编译 TensorFlow 源码的方式使用。除了官方教程之外，您还可以参考该[链接](https://github.com/SmileTM/Tensorflow2.X-GPU-CUDA9.0)以寻找可能合适的预编译包以及编译源码的实用攻略。 + +## 测试并提交到官方服务器 + +如下是一个使用 8 个图形处理器在 Waymo 上测试 PointPillars，生成 bin 文件并提交结果到官方榜单的例子： + + ```shell + ./tools/slurm_test.sh ${PARTITION} ${JOB_NAME} configs/pointpillars/hv_pointpillars_secfpn_sbn-2x16_2x_waymo-3d-car.py \ + checkpoints/hv_pointpillars_secfpn_sbn-2x16_2x_waymo-3d-car_latest.pth --out results/waymo-car/results_eval.pkl \ + --format-only --eval-options 'pklfile_prefix=results/waymo-car/kitti_results' \ + 'submission_prefix=results/waymo-car/kitti_results' + ``` + +在生成 bin 文件后，您可以简单地构建二进制文件 `create_submission`，并按照[指示](https://github.com/waymo-research/waymo-open-dataset/blob/master/docs/quick_start.md/) 创建一个提交文件。下面是一些示例： + + ```shell + cd ../waymo-od/ + bazel build waymo_open_dataset/metrics/tools/create_submission + cp bazel-bin/waymo_open_dataset/metrics/tools/create_submission ../mmdetection3d/mmdet3d/core/evaluation/waymo_utils/ + vim waymo_open_dataset/metrics/tools/submission.txtpb # set the metadata information + cp waymo_open_dataset/metrics/tools/submission.txtpb ../mmdetection3d/mmdet3d/core/evaluation/waymo_utils/ + + cd ../mmdetection3d + # suppose the result bin is in `results/waymo-car/submission` + mmdet3d/core/evaluation/waymo_utils/create_submission --input_filenames='results/waymo-car/kitti_results_test.bin' --output_filename='results/waymo-car/submission/model' --submission_filename='mmdet3d/core/evaluation/waymo_utils/submission.txtpb' + + tar cvf results/waymo-car/submission/my_model.tar results/waymo-car/submission/my_model/ + gzip results/waymo-car/submission/my_model.tar + ``` + +如果想用官方评估服务器评估您在验证集上的结果，您可以使用同样的方法生成提交文件，只需确保您在运行如上指令前更改 `submission.txtpb` 中的字段值即可。 From 8d0b12afcea0649adbae0e3fa4ac0b78465006de Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 18 Aug 2021 23:06:21 +0800 Subject: [PATCH 12/51] Remove 2D annotations on Lyft (#867) --- tools/create_data.py | 37 ++++---------------------- tools/data_converter/lyft_converter.py | 4 +++ 2 files changed, 9 insertions(+), 32 deletions(-) diff --git a/tools/create_data.py b/tools/create_data.py index fdb7e4b06f..8bb4bed5e5 100644 --- a/tools/create_data.py +++ b/tools/create_data.py @@ -82,46 +82,23 @@ def nuscenes_data_prep(root_path, f'{out_dir}/{info_prefix}_infos_train.pkl') -def lyft_data_prep(root_path, - info_prefix, - version, - dataset_name, - out_dir, - max_sweeps=10): +def lyft_data_prep(root_path, info_prefix, version, max_sweeps=10): """Prepare data related to Lyft dataset. - Related data consists of '.pkl' files recording basic infos, - and 2D annotations. - Although the ground truth database is not used in Lyft, it can also be - generated like nuScenes. + Related data consists of '.pkl' files recording basic infos. + Although the ground truth database and 2D annotations are not used in + Lyft, it can also be generated like nuScenes. Args: root_path (str): Path of dataset root. info_prefix (str): The prefix of info filenames. version (str): Dataset version. - dataset_name (str): The dataset class name. - out_dir (str): Output directory of the groundtruth database info. - Not used here if the groundtruth database is not generated. max_sweeps (int, optional): Number of input consecutive frames. - Default: 10 + Defaults to 10. """ lyft_converter.create_lyft_infos( root_path, info_prefix, version=version, max_sweeps=max_sweeps) - if version == 'v1.01-test': - return - - train_info_name = f'{info_prefix}_infos_train' - val_info_name = f'{info_prefix}_infos_val' - - info_train_path = osp.join(root_path, f'{train_info_name}.pkl') - info_val_path = osp.join(root_path, f'{val_info_name}.pkl') - - lyft_converter.export_2d_annotation( - root_path, info_train_path, version=version) - lyft_converter.export_2d_annotation( - root_path, info_val_path, version=version) - def scannet_data_prep(root_path, info_prefix, out_dir, workers): """Prepare the info file for scannet dataset. @@ -276,16 +253,12 @@ def waymo_data_prep(root_path, root_path=args.root_path, info_prefix=args.extra_tag, version=train_version, - dataset_name='LyftDataset', - out_dir=args.out_dir, max_sweeps=args.max_sweeps) test_version = f'{args.version}-test' lyft_data_prep( root_path=args.root_path, info_prefix=args.extra_tag, version=test_version, - dataset_name='LyftDataset', - out_dir=args.out_dir, max_sweeps=args.max_sweeps) elif args.dataset == 'waymo': waymo_data_prep( diff --git a/tools/data_converter/lyft_converter.py b/tools/data_converter/lyft_converter.py index e2d9ddb283..a31989809c 100644 --- a/tools/data_converter/lyft_converter.py +++ b/tools/data_converter/lyft_converter.py @@ -1,6 +1,7 @@ import mmcv import numpy as np import os +from logging import warning from lyft_dataset_sdk.lyftdataset import LyftDataset as Lyft from os import path as osp from pyquaternion import Quaternion @@ -219,6 +220,9 @@ def export_2d_annotation(root_path, info_path, version): info_path (str): Path of the info file. version (str): Dataset version. """ + warning.warn('DeprecationWarning: 2D annotations are not used on the ' + 'Lyft dataset. The function export_2d_annotation will be ' + 'deprecated.') # get bbox annotations for camera camera_types = [ 'CAM_FRONT', From 2375d3c1a2680ba52d98134aac4434467b035a7f Mon Sep 17 00:00:00 2001 From: dingchang Date: Thu, 19 Aug 2021 10:43:49 +0800 Subject: [PATCH 13/51] Add header for files (#869) * Add header for files * Add header for files * Add header for files * Add header for files --- .dev_scripts/gather_models.py | 1 + demo/mono_det_demo.py | 1 + demo/multi_modality_demo.py | 1 + demo/pc_seg_demo.py | 1 + demo/pcd_demo.py | 1 + mmdet3d/__init__.py | 1 + mmdet3d/apis/__init__.py | 1 + mmdet3d/apis/inference.py | 1 + mmdet3d/apis/test.py | 1 + mmdet3d/apis/train.py | 1 + mmdet3d/core/__init__.py | 1 + mmdet3d/core/anchor/__init__.py | 1 + mmdet3d/core/anchor/anchor_3d_generator.py | 1 + mmdet3d/core/bbox/__init__.py | 1 + mmdet3d/core/bbox/assigners/__init__.py | 1 + mmdet3d/core/bbox/box_np_ops.py | 1 + mmdet3d/core/bbox/coders/__init__.py | 1 + mmdet3d/core/bbox/coders/anchor_free_bbox_coder.py | 1 + mmdet3d/core/bbox/coders/centerpoint_bbox_coders.py | 1 + mmdet3d/core/bbox/coders/delta_xyzwhlr_bbox_coder.py | 1 + mmdet3d/core/bbox/coders/groupfree3d_bbox_coder.py | 1 + mmdet3d/core/bbox/coders/partial_bin_based_bbox_coder.py | 1 + mmdet3d/core/bbox/iou_calculators/__init__.py | 1 + mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py | 1 + mmdet3d/core/bbox/samplers/__init__.py | 1 + mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py | 1 + mmdet3d/core/bbox/structures/__init__.py | 1 + mmdet3d/core/bbox/structures/base_box3d.py | 1 + mmdet3d/core/bbox/structures/box_3d_mode.py | 1 + mmdet3d/core/bbox/structures/cam_box3d.py | 1 + mmdet3d/core/bbox/structures/coord_3d_mode.py | 1 + mmdet3d/core/bbox/structures/depth_box3d.py | 1 + mmdet3d/core/bbox/structures/lidar_box3d.py | 1 + mmdet3d/core/bbox/structures/utils.py | 1 + mmdet3d/core/bbox/transforms.py | 1 + mmdet3d/core/evaluation/__init__.py | 1 + mmdet3d/core/evaluation/indoor_eval.py | 1 + mmdet3d/core/evaluation/kitti_utils/__init__.py | 1 + mmdet3d/core/evaluation/kitti_utils/eval.py | 1 + mmdet3d/core/evaluation/kitti_utils/rotate_iou.py | 1 + mmdet3d/core/evaluation/lyft_eval.py | 1 + mmdet3d/core/evaluation/seg_eval.py | 1 + mmdet3d/core/evaluation/waymo_utils/prediction_kitti_to_waymo.py | 1 + mmdet3d/core/points/__init__.py | 1 + mmdet3d/core/points/base_points.py | 1 + mmdet3d/core/points/cam_points.py | 1 + mmdet3d/core/points/depth_points.py | 1 + mmdet3d/core/points/lidar_points.py | 1 + mmdet3d/core/post_processing/__init__.py | 1 + mmdet3d/core/post_processing/box3d_nms.py | 1 + mmdet3d/core/post_processing/merge_augs.py | 1 + mmdet3d/core/utils/__init__.py | 1 + mmdet3d/core/utils/gaussian.py | 1 + mmdet3d/core/visualizer/__init__.py | 1 + mmdet3d/core/visualizer/image_vis.py | 1 + mmdet3d/core/visualizer/open3d_vis.py | 1 + mmdet3d/core/visualizer/show_result.py | 1 + mmdet3d/core/voxel/__init__.py | 1 + mmdet3d/core/voxel/builder.py | 1 + mmdet3d/core/voxel/voxel_generator.py | 1 + mmdet3d/datasets/__init__.py | 1 + mmdet3d/datasets/builder.py | 1 + mmdet3d/datasets/custom_3d.py | 1 + mmdet3d/datasets/custom_3d_seg.py | 1 + mmdet3d/datasets/dataset_wrappers.py | 1 + mmdet3d/datasets/kitti2d_dataset.py | 1 + mmdet3d/datasets/kitti_dataset.py | 1 + mmdet3d/datasets/kitti_mono_dataset.py | 1 + mmdet3d/datasets/lyft_dataset.py | 1 + mmdet3d/datasets/nuscenes_dataset.py | 1 + mmdet3d/datasets/nuscenes_mono_dataset.py | 1 + mmdet3d/datasets/pipelines/__init__.py | 1 + mmdet3d/datasets/pipelines/data_augment_utils.py | 1 + mmdet3d/datasets/pipelines/dbsampler.py | 1 + mmdet3d/datasets/pipelines/formating.py | 1 + mmdet3d/datasets/pipelines/loading.py | 1 + mmdet3d/datasets/pipelines/test_time_aug.py | 1 + mmdet3d/datasets/pipelines/transforms_3d.py | 1 + mmdet3d/datasets/s3dis_dataset.py | 1 + mmdet3d/datasets/scannet_dataset.py | 1 + mmdet3d/datasets/semantickitti_dataset.py | 1 + mmdet3d/datasets/sunrgbd_dataset.py | 1 + mmdet3d/datasets/utils.py | 1 + mmdet3d/datasets/waymo_dataset.py | 1 + mmdet3d/models/__init__.py | 1 + mmdet3d/models/backbones/__init__.py | 1 + mmdet3d/models/backbones/base_pointnet.py | 1 + mmdet3d/models/backbones/multi_backbone.py | 1 + mmdet3d/models/backbones/nostem_regnet.py | 1 + mmdet3d/models/backbones/pointnet2_sa_msg.py | 1 + mmdet3d/models/backbones/pointnet2_sa_ssg.py | 1 + mmdet3d/models/backbones/second.py | 1 + mmdet3d/models/builder.py | 1 + mmdet3d/models/decode_heads/__init__.py | 1 + mmdet3d/models/decode_heads/decode_head.py | 1 + mmdet3d/models/decode_heads/paconv_head.py | 1 + mmdet3d/models/decode_heads/pointnet2_head.py | 1 + mmdet3d/models/dense_heads/__init__.py | 1 + mmdet3d/models/dense_heads/anchor3d_head.py | 1 + mmdet3d/models/dense_heads/anchor_free_mono3d_head.py | 1 + mmdet3d/models/dense_heads/base_conv_bbox_head.py | 1 + mmdet3d/models/dense_heads/base_mono3d_dense_head.py | 1 + mmdet3d/models/dense_heads/centerpoint_head.py | 1 + mmdet3d/models/dense_heads/fcos_mono3d_head.py | 1 + mmdet3d/models/dense_heads/free_anchor3d_head.py | 1 + mmdet3d/models/dense_heads/groupfree3d_head.py | 1 + mmdet3d/models/dense_heads/parta2_rpn_head.py | 1 + mmdet3d/models/dense_heads/shape_aware_head.py | 1 + mmdet3d/models/dense_heads/ssd_3d_head.py | 1 + mmdet3d/models/dense_heads/train_mixins.py | 1 + mmdet3d/models/dense_heads/vote_head.py | 1 + mmdet3d/models/detectors/__init__.py | 1 + mmdet3d/models/detectors/base.py | 1 + mmdet3d/models/detectors/centerpoint.py | 1 + mmdet3d/models/detectors/dynamic_voxelnet.py | 1 + mmdet3d/models/detectors/fcos_mono3d.py | 1 + mmdet3d/models/detectors/groupfree3dnet.py | 1 + mmdet3d/models/detectors/h3dnet.py | 1 + mmdet3d/models/detectors/imvotenet.py | 1 + mmdet3d/models/detectors/imvoxelnet.py | 1 + mmdet3d/models/detectors/mvx_faster_rcnn.py | 1 + mmdet3d/models/detectors/mvx_two_stage.py | 1 + mmdet3d/models/detectors/parta2.py | 1 + mmdet3d/models/detectors/single_stage.py | 1 + mmdet3d/models/detectors/single_stage_mono3d.py | 1 + mmdet3d/models/detectors/ssd3dnet.py | 1 + mmdet3d/models/detectors/two_stage.py | 1 + mmdet3d/models/detectors/votenet.py | 1 + mmdet3d/models/detectors/voxelnet.py | 1 + mmdet3d/models/fusion_layers/__init__.py | 1 + mmdet3d/models/fusion_layers/coord_transform.py | 1 + mmdet3d/models/fusion_layers/point_fusion.py | 1 + mmdet3d/models/fusion_layers/vote_fusion.py | 1 + mmdet3d/models/losses/__init__.py | 1 + mmdet3d/models/losses/axis_aligned_iou_loss.py | 1 + mmdet3d/models/losses/chamfer_distance.py | 1 + mmdet3d/models/losses/paconv_regularization_loss.py | 1 + mmdet3d/models/middle_encoders/__init__.py | 1 + mmdet3d/models/middle_encoders/pillar_scatter.py | 1 + mmdet3d/models/middle_encoders/sparse_encoder.py | 1 + mmdet3d/models/middle_encoders/sparse_unet.py | 1 + mmdet3d/models/model_utils/__init__.py | 1 + mmdet3d/models/model_utils/transformer.py | 1 + mmdet3d/models/model_utils/vote_module.py | 1 + mmdet3d/models/necks/__init__.py | 1 + mmdet3d/models/necks/imvoxel_neck.py | 1 + mmdet3d/models/necks/second_fpn.py | 1 + mmdet3d/models/roi_heads/__init__.py | 1 + mmdet3d/models/roi_heads/base_3droi_head.py | 1 + mmdet3d/models/roi_heads/bbox_heads/__init__.py | 1 + mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py | 1 + mmdet3d/models/roi_heads/bbox_heads/parta2_bbox_head.py | 1 + mmdet3d/models/roi_heads/h3d_roi_head.py | 1 + mmdet3d/models/roi_heads/mask_heads/__init__.py | 1 + mmdet3d/models/roi_heads/mask_heads/pointwise_semantic_head.py | 1 + mmdet3d/models/roi_heads/mask_heads/primitive_head.py | 1 + mmdet3d/models/roi_heads/part_aggregation_roi_head.py | 1 + mmdet3d/models/roi_heads/roi_extractors/__init__.py | 1 + .../models/roi_heads/roi_extractors/single_roiaware_extractor.py | 1 + mmdet3d/models/segmentors/__init__.py | 1 + mmdet3d/models/segmentors/base.py | 1 + mmdet3d/models/segmentors/encoder_decoder.py | 1 + mmdet3d/models/utils/__init__.py | 1 + mmdet3d/models/utils/clip_sigmoid.py | 1 + mmdet3d/models/utils/mlp.py | 1 + mmdet3d/models/voxel_encoders/__init__.py | 1 + mmdet3d/models/voxel_encoders/pillar_encoder.py | 1 + mmdet3d/models/voxel_encoders/utils.py | 1 + mmdet3d/models/voxel_encoders/voxel_encoder.py | 1 + mmdet3d/ops/__init__.py | 1 + mmdet3d/ops/pointnet_modules/__init__.py | 1 + mmdet3d/ops/pointnet_modules/builder.py | 1 + mmdet3d/ops/pointnet_modules/paconv_sa_module.py | 1 + mmdet3d/ops/pointnet_modules/point_fp_module.py | 1 + mmdet3d/ops/pointnet_modules/point_sa_module.py | 1 + mmdet3d/ops/sparse_block.py | 1 + mmdet3d/utils/__init__.py | 1 + mmdet3d/utils/collect_env.py | 1 + mmdet3d/utils/logger.py | 1 + tests/test_data/test_datasets/test_dataset_wrappers.py | 1 + tests/test_data/test_datasets/test_kitti_dataset.py | 1 + tests/test_data/test_datasets/test_kitti_mono_dataset.py | 1 + tests/test_data/test_datasets/test_lyft_dataset.py | 1 + tests/test_data/test_datasets/test_nuscene_dataset.py | 1 + tests/test_data/test_datasets/test_nuscenes_mono_dataset.py | 1 + tests/test_data/test_datasets/test_s3dis_dataset.py | 1 + tests/test_data/test_datasets/test_scannet_dataset.py | 1 + tests/test_data/test_datasets/test_semantickitti_dataset.py | 1 + tests/test_data/test_datasets/test_sunrgbd_dataset.py | 1 + tests/test_data/test_datasets/test_waymo_dataset.py | 1 + .../test_pipelines/test_augmentations/test_data_augment_utils.py | 1 + .../test_pipelines/test_augmentations/test_test_augment_utils.py | 1 + .../test_pipelines/test_augmentations/test_transforms_3d.py | 1 + tests/test_data/test_pipelines/test_indoor_pipeline.py | 1 + tests/test_data/test_pipelines/test_indoor_sample.py | 1 + .../test_loadings/test_load_images_from_multi_views.py | 1 + .../test_loadings/test_load_points_from_multi_sweeps.py | 1 + tests/test_data/test_pipelines/test_loadings/test_loading.py | 1 + tests/test_data/test_pipelines/test_outdoor_pipeline.py | 1 + tests/test_metrics/test_indoor_eval.py | 1 + tests/test_metrics/test_kitti_eval.py | 1 + tests/test_metrics/test_losses.py | 1 + tests/test_metrics/test_seg_eval.py | 1 + tests/test_models/test_backbones.py | 1 + tests/test_models/test_common_modules/test_middle_encoders.py | 1 + tests/test_models/test_common_modules/test_paconv_modules.py | 1 + tests/test_models/test_common_modules/test_paconv_ops.py | 1 + tests/test_models/test_common_modules/test_pointnet_modules.py | 1 + tests/test_models/test_common_modules/test_pointnet_ops.py | 1 + tests/test_models/test_common_modules/test_roiaware_pool3d.py | 1 + tests/test_models/test_common_modules/test_sparse_unet.py | 1 + tests/test_models/test_common_modules/test_vote_module.py | 1 + tests/test_models/test_detectors.py | 1 + tests/test_models/test_forward.py | 1 + tests/test_models/test_fusion/test_fusion_coord_trans.py | 1 + tests/test_models/test_fusion/test_point_fusion.py | 1 + tests/test_models/test_fusion/test_vote_fusion.py | 1 + tests/test_models/test_heads/test_heads.py | 1 + tests/test_models/test_heads/test_paconv_decode_head.py | 1 + tests/test_models/test_heads/test_parta2_bbox_head.py | 1 + tests/test_models/test_heads/test_pointnet2_decode_head.py | 1 + tests/test_models/test_heads/test_roi_extractors.py | 1 + tests/test_models/test_heads/test_semantic_heads.py | 1 + tests/test_models/test_necks/test_fpn.py | 1 + tests/test_models/test_necks/test_necks.py | 1 + tests/test_models/test_segmentors.py | 1 + tests/test_models/test_voxel_encoder/test_dynamic_scatter.py | 1 + tests/test_models/test_voxel_encoder/test_voxel_encoders.py | 1 + tests/test_models/test_voxel_encoder/test_voxel_generator.py | 1 + tests/test_models/test_voxel_encoder/test_voxelize.py | 1 + tests/test_runtime/test_apis.py | 1 + tests/test_runtime/test_config.py | 1 + tests/test_utils/test_anchors.py | 1 + tests/test_utils/test_assigners.py | 1 + tests/test_utils/test_bbox_coders.py | 1 + tests/test_utils/test_box3d.py | 1 + tests/test_utils/test_box_np_ops.py | 1 + tests/test_utils/test_coord_3d_mode.py | 1 + tests/test_utils/test_merge_augs.py | 1 + tests/test_utils/test_nms.py | 1 + tests/test_utils/test_points.py | 1 + tests/test_utils/test_samplers.py | 1 + tests/test_utils/test_utils.py | 1 + tools/analysis_tools/analyze_logs.py | 1 + tools/analysis_tools/benchmark.py | 1 + tools/analysis_tools/get_flops.py | 1 + tools/create_data.py | 1 + tools/data_converter/__init__.py | 1 + tools/data_converter/create_gt_database.py | 1 + tools/data_converter/indoor_converter.py | 1 + tools/data_converter/kitti_converter.py | 1 + tools/data_converter/kitti_data_utils.py | 1 + tools/data_converter/lyft_converter.py | 1 + tools/data_converter/lyft_data_fixer.py | 1 + tools/data_converter/nuimage_converter.py | 1 + tools/data_converter/nuscenes_converter.py | 1 + tools/data_converter/s3dis_data_utils.py | 1 + tools/data_converter/scannet_data_utils.py | 1 + tools/data_converter/sunrgbd_data_utils.py | 1 + tools/data_converter/waymo_converter.py | 1 + tools/misc/browse_dataset.py | 1 + tools/misc/fuse_conv_bn.py | 1 + tools/misc/print_config.py | 1 + tools/misc/visualize_results.py | 1 + tools/model_converters/convert_votenet_checkpoints.py | 1 + tools/model_converters/publish_model.py | 1 + tools/model_converters/regnet2mmdet.py | 1 + tools/test.py | 1 + tools/train.py | 1 + 269 files changed, 269 insertions(+) diff --git a/.dev_scripts/gather_models.py b/.dev_scripts/gather_models.py index bcaa28e559..58919fd444 100644 --- a/.dev_scripts/gather_models.py +++ b/.dev_scripts/gather_models.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. """Script to gather benchmarked models and prepare them for upload. Usage: diff --git a/demo/mono_det_demo.py b/demo/mono_det_demo.py index 68ba832e20..3b043a310b 100644 --- a/demo/mono_det_demo.py +++ b/demo/mono_det_demo.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from argparse import ArgumentParser from mmdet3d.apis import (inference_mono_3d_detector, init_model, diff --git a/demo/multi_modality_demo.py b/demo/multi_modality_demo.py index d6918fa4a9..67c509b927 100644 --- a/demo/multi_modality_demo.py +++ b/demo/multi_modality_demo.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from argparse import ArgumentParser from mmdet3d.apis import (inference_multi_modality_detector, init_model, diff --git a/demo/pc_seg_demo.py b/demo/pc_seg_demo.py index 858b914ade..b8ee0c429f 100644 --- a/demo/pc_seg_demo.py +++ b/demo/pc_seg_demo.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from argparse import ArgumentParser from mmdet3d.apis import inference_segmentor, init_model, show_result_meshlab diff --git a/demo/pcd_demo.py b/demo/pcd_demo.py index 5fbe31aa7c..253ba35c8c 100644 --- a/demo/pcd_demo.py +++ b/demo/pcd_demo.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from argparse import ArgumentParser from mmdet3d.apis import inference_detector, init_model, show_result_meshlab diff --git a/mmdet3d/__init__.py b/mmdet3d/__init__.py index 839cded3ba..ee89aa550d 100644 --- a/mmdet3d/__init__.py +++ b/mmdet3d/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import mmdet diff --git a/mmdet3d/apis/__init__.py b/mmdet3d/apis/__init__.py index 4c2292c1db..1063648ab1 100644 --- a/mmdet3d/apis/__init__.py +++ b/mmdet3d/apis/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .inference import (convert_SyncBN, inference_detector, inference_mono_3d_detector, inference_multi_modality_detector, inference_segmentor, diff --git a/mmdet3d/apis/inference.py b/mmdet3d/apis/inference.py index 35d10bad54..bd2dcdb385 100644 --- a/mmdet3d/apis/inference.py +++ b/mmdet3d/apis/inference.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import re diff --git a/mmdet3d/apis/test.py b/mmdet3d/apis/test.py index 858fe3dd26..bb899b93eb 100644 --- a/mmdet3d/apis/test.py +++ b/mmdet3d/apis/test.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import torch from mmcv.image import tensor2imgs diff --git a/mmdet3d/apis/train.py b/mmdet3d/apis/train.py index ca9f74fdff..93659468e4 100644 --- a/mmdet3d/apis/train.py +++ b/mmdet3d/apis/train.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.apis import train_detector from mmseg.apis import train_segmentor diff --git a/mmdet3d/core/__init__.py b/mmdet3d/core/__init__.py index 9d77b76987..ffb0c1acbf 100644 --- a/mmdet3d/core/__init__.py +++ b/mmdet3d/core/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .anchor import * # noqa: F401, F403 from .bbox import * # noqa: F401, F403 from .evaluation import * # noqa: F401, F403 diff --git a/mmdet3d/core/anchor/__init__.py b/mmdet3d/core/anchor/__init__.py index 346c1360ea..0393a02a3f 100644 --- a/mmdet3d/core/anchor/__init__.py +++ b/mmdet3d/core/anchor/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.core.anchor import build_anchor_generator from .anchor_3d_generator import (AlignedAnchor3DRangeGenerator, AlignedAnchor3DRangeGeneratorPerCls, diff --git a/mmdet3d/core/anchor/anchor_3d_generator.py b/mmdet3d/core/anchor/anchor_3d_generator.py index 1ea8426e51..aa9255b08a 100644 --- a/mmdet3d/core/anchor/anchor_3d_generator.py +++ b/mmdet3d/core/anchor/anchor_3d_generator.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import torch diff --git a/mmdet3d/core/bbox/__init__.py b/mmdet3d/core/bbox/__init__.py index afca4dd4bd..dd06b31c2f 100644 --- a/mmdet3d/core/bbox/__init__.py +++ b/mmdet3d/core/bbox/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .assigners import AssignResult, BaseAssigner, MaxIoUAssigner from .coders import DeltaXYZWLHRBBoxCoder # from .bbox_target import bbox_target diff --git a/mmdet3d/core/bbox/assigners/__init__.py b/mmdet3d/core/bbox/assigners/__init__.py index 44f51fc0f3..d14936871a 100644 --- a/mmdet3d/core/bbox/assigners/__init__.py +++ b/mmdet3d/core/bbox/assigners/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.core.bbox import AssignResult, BaseAssigner, MaxIoUAssigner __all__ = ['BaseAssigner', 'MaxIoUAssigner', 'AssignResult'] diff --git a/mmdet3d/core/bbox/box_np_ops.py b/mmdet3d/core/bbox/box_np_ops.py index 8990713a5d..26a43e60ed 100644 --- a/mmdet3d/core/bbox/box_np_ops.py +++ b/mmdet3d/core/bbox/box_np_ops.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. # TODO: clean the functions in this file and move the APIs into box structures # in the future # NOTICE: All functions in this file are valid for LiDAR or depth boxes only diff --git a/mmdet3d/core/bbox/coders/__init__.py b/mmdet3d/core/bbox/coders/__init__.py index c662e70c1f..08fbb410b3 100644 --- a/mmdet3d/core/bbox/coders/__init__.py +++ b/mmdet3d/core/bbox/coders/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.core.bbox import build_bbox_coder from .anchor_free_bbox_coder import AnchorFreeBBoxCoder from .centerpoint_bbox_coders import CenterPointBBoxCoder diff --git a/mmdet3d/core/bbox/coders/anchor_free_bbox_coder.py b/mmdet3d/core/bbox/coders/anchor_free_bbox_coder.py index b42e02b716..d64f38b5c9 100644 --- a/mmdet3d/core/bbox/coders/anchor_free_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/anchor_free_bbox_coder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/mmdet3d/core/bbox/coders/centerpoint_bbox_coders.py b/mmdet3d/core/bbox/coders/centerpoint_bbox_coders.py index 9506de3229..6d43a63d4b 100644 --- a/mmdet3d/core/bbox/coders/centerpoint_bbox_coders.py +++ b/mmdet3d/core/bbox/coders/centerpoint_bbox_coders.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet.core.bbox import BaseBBoxCoder diff --git a/mmdet3d/core/bbox/coders/delta_xyzwhlr_bbox_coder.py b/mmdet3d/core/bbox/coders/delta_xyzwhlr_bbox_coder.py index 28b6f5b3ec..931e839872 100644 --- a/mmdet3d/core/bbox/coders/delta_xyzwhlr_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/delta_xyzwhlr_bbox_coder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet.core.bbox import BaseBBoxCoder diff --git a/mmdet3d/core/bbox/coders/groupfree3d_bbox_coder.py b/mmdet3d/core/bbox/coders/groupfree3d_bbox_coder.py index 3b52696751..08d83e92c7 100644 --- a/mmdet3d/core/bbox/coders/groupfree3d_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/groupfree3d_bbox_coder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/mmdet3d/core/bbox/coders/partial_bin_based_bbox_coder.py b/mmdet3d/core/bbox/coders/partial_bin_based_bbox_coder.py index 0fa9dc921c..ed8020d70d 100644 --- a/mmdet3d/core/bbox/coders/partial_bin_based_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/partial_bin_based_bbox_coder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/mmdet3d/core/bbox/iou_calculators/__init__.py b/mmdet3d/core/bbox/iou_calculators/__init__.py index 9e07c43a54..d2faf69cd9 100644 --- a/mmdet3d/core/bbox/iou_calculators/__init__.py +++ b/mmdet3d/core/bbox/iou_calculators/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .iou3d_calculator import (AxisAlignedBboxOverlaps3D, BboxOverlaps3D, BboxOverlapsNearest3D, axis_aligned_bbox_overlaps_3d, bbox_overlaps_3d, diff --git a/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py b/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py index 211f9522ad..4afdabf2d6 100644 --- a/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py +++ b/mmdet3d/core/bbox/iou_calculators/iou3d_calculator.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet.core.bbox import bbox_overlaps diff --git a/mmdet3d/core/bbox/samplers/__init__.py b/mmdet3d/core/bbox/samplers/__init__.py index 90e5d00034..168780b2d0 100644 --- a/mmdet3d/core/bbox/samplers/__init__.py +++ b/mmdet3d/core/bbox/samplers/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.core.bbox.samplers import (BaseSampler, CombinedSampler, InstanceBalancedPosSampler, IoUBalancedNegSampler, OHEMSampler, diff --git a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py index b74fd5a8a5..c0de845f68 100644 --- a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py +++ b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet.core.bbox.builder import BBOX_SAMPLERS diff --git a/mmdet3d/core/bbox/structures/__init__.py b/mmdet3d/core/bbox/structures/__init__.py index 4779703145..8b010cf72d 100644 --- a/mmdet3d/core/bbox/structures/__init__.py +++ b/mmdet3d/core/bbox/structures/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .base_box3d import BaseInstance3DBoxes from .box_3d_mode import Box3DMode from .cam_box3d import CameraInstance3DBoxes diff --git a/mmdet3d/core/bbox/structures/base_box3d.py b/mmdet3d/core/bbox/structures/base_box3d.py index 37c7dc8709..ec182216b5 100644 --- a/mmdet3d/core/bbox/structures/base_box3d.py +++ b/mmdet3d/core/bbox/structures/base_box3d.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch import warnings diff --git a/mmdet3d/core/bbox/structures/box_3d_mode.py b/mmdet3d/core/bbox/structures/box_3d_mode.py index 1406b42175..72ecc360b1 100644 --- a/mmdet3d/core/bbox/structures/box_3d_mode.py +++ b/mmdet3d/core/bbox/structures/box_3d_mode.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from enum import IntEnum, unique diff --git a/mmdet3d/core/bbox/structures/cam_box3d.py b/mmdet3d/core/bbox/structures/cam_box3d.py index 509de233ba..dd2a8eed19 100644 --- a/mmdet3d/core/bbox/structures/cam_box3d.py +++ b/mmdet3d/core/bbox/structures/cam_box3d.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/mmdet3d/core/bbox/structures/coord_3d_mode.py b/mmdet3d/core/bbox/structures/coord_3d_mode.py index 1fb0566d0b..44dba26acb 100644 --- a/mmdet3d/core/bbox/structures/coord_3d_mode.py +++ b/mmdet3d/core/bbox/structures/coord_3d_mode.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from enum import IntEnum, unique diff --git a/mmdet3d/core/bbox/structures/depth_box3d.py b/mmdet3d/core/bbox/structures/depth_box3d.py index fb92aff827..38c1ec7437 100644 --- a/mmdet3d/core/bbox/structures/depth_box3d.py +++ b/mmdet3d/core/bbox/structures/depth_box3d.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/mmdet3d/core/bbox/structures/lidar_box3d.py b/mmdet3d/core/bbox/structures/lidar_box3d.py index 24efa32384..0d14655c63 100644 --- a/mmdet3d/core/bbox/structures/lidar_box3d.py +++ b/mmdet3d/core/bbox/structures/lidar_box3d.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/mmdet3d/core/bbox/structures/utils.py b/mmdet3d/core/bbox/structures/utils.py index ba37bc8842..2511fdb2a3 100644 --- a/mmdet3d/core/bbox/structures/utils.py +++ b/mmdet3d/core/bbox/structures/utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from logging import warning diff --git a/mmdet3d/core/bbox/transforms.py b/mmdet3d/core/bbox/transforms.py index e002924f10..8a2eb90f54 100644 --- a/mmdet3d/core/bbox/transforms.py +++ b/mmdet3d/core/bbox/transforms.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch diff --git a/mmdet3d/core/evaluation/__init__.py b/mmdet3d/core/evaluation/__init__.py index f8cd210a33..877458a05c 100644 --- a/mmdet3d/core/evaluation/__init__.py +++ b/mmdet3d/core/evaluation/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .indoor_eval import indoor_eval from .kitti_utils import kitti_eval, kitti_eval_coco_style from .lyft_eval import lyft_eval diff --git a/mmdet3d/core/evaluation/indoor_eval.py b/mmdet3d/core/evaluation/indoor_eval.py index d80506225b..2ff9877329 100644 --- a/mmdet3d/core/evaluation/indoor_eval.py +++ b/mmdet3d/core/evaluation/indoor_eval.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from mmcv.utils import print_log diff --git a/mmdet3d/core/evaluation/kitti_utils/__init__.py b/mmdet3d/core/evaluation/kitti_utils/__init__.py index b1fc7bc3dc..23c1cdf250 100644 --- a/mmdet3d/core/evaluation/kitti_utils/__init__.py +++ b/mmdet3d/core/evaluation/kitti_utils/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .eval import kitti_eval, kitti_eval_coco_style __all__ = ['kitti_eval', 'kitti_eval_coco_style'] diff --git a/mmdet3d/core/evaluation/kitti_utils/eval.py b/mmdet3d/core/evaluation/kitti_utils/eval.py index b67eb7cae9..93492c466c 100644 --- a/mmdet3d/core/evaluation/kitti_utils/eval.py +++ b/mmdet3d/core/evaluation/kitti_utils/eval.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import gc import io as sysio import numba diff --git a/mmdet3d/core/evaluation/kitti_utils/rotate_iou.py b/mmdet3d/core/evaluation/kitti_utils/rotate_iou.py index 33dccfd7f9..bd9cdad135 100644 --- a/mmdet3d/core/evaluation/kitti_utils/rotate_iou.py +++ b/mmdet3d/core/evaluation/kitti_utils/rotate_iou.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. ##################### # Based on https://github.com/hongzhenwang/RRPN-revise # Licensed under The MIT License diff --git a/mmdet3d/core/evaluation/lyft_eval.py b/mmdet3d/core/evaluation/lyft_eval.py index 71a73d09b5..6e4f21e4b8 100644 --- a/mmdet3d/core/evaluation/lyft_eval.py +++ b/mmdet3d/core/evaluation/lyft_eval.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np from lyft_dataset_sdk.eval.detection.mAP_evaluation import (Box3D, get_ap, diff --git a/mmdet3d/core/evaluation/seg_eval.py b/mmdet3d/core/evaluation/seg_eval.py index 2582498bd5..4a3166d685 100644 --- a/mmdet3d/core/evaluation/seg_eval.py +++ b/mmdet3d/core/evaluation/seg_eval.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np from mmcv.utils import print_log from terminaltables import AsciiTable diff --git a/mmdet3d/core/evaluation/waymo_utils/prediction_kitti_to_waymo.py b/mmdet3d/core/evaluation/waymo_utils/prediction_kitti_to_waymo.py index 86548a79ce..32d6eb640e 100644 --- a/mmdet3d/core/evaluation/waymo_utils/prediction_kitti_to_waymo.py +++ b/mmdet3d/core/evaluation/waymo_utils/prediction_kitti_to_waymo.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. r"""Adapted from `Waymo to KITTI converter `_. """ diff --git a/mmdet3d/core/points/__init__.py b/mmdet3d/core/points/__init__.py index 0575c3f35d..73d2d8338c 100644 --- a/mmdet3d/core/points/__init__.py +++ b/mmdet3d/core/points/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .base_points import BasePoints from .cam_points import CameraPoints from .depth_points import DepthPoints diff --git a/mmdet3d/core/points/base_points.py b/mmdet3d/core/points/base_points.py index a43b03b909..e740f4a6e3 100644 --- a/mmdet3d/core/points/base_points.py +++ b/mmdet3d/core/points/base_points.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch import warnings diff --git a/mmdet3d/core/points/cam_points.py b/mmdet3d/core/points/cam_points.py index 6f189269e6..a57c3db1e8 100644 --- a/mmdet3d/core/points/cam_points.py +++ b/mmdet3d/core/points/cam_points.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .base_points import BasePoints diff --git a/mmdet3d/core/points/depth_points.py b/mmdet3d/core/points/depth_points.py index de136de9ea..2d9221fb25 100644 --- a/mmdet3d/core/points/depth_points.py +++ b/mmdet3d/core/points/depth_points.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .base_points import BasePoints diff --git a/mmdet3d/core/points/lidar_points.py b/mmdet3d/core/points/lidar_points.py index 62df4bb183..ff4f57ab0e 100644 --- a/mmdet3d/core/points/lidar_points.py +++ b/mmdet3d/core/points/lidar_points.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .base_points import BasePoints diff --git a/mmdet3d/core/post_processing/__init__.py b/mmdet3d/core/post_processing/__init__.py index 2f67514add..a52a23ceb6 100644 --- a/mmdet3d/core/post_processing/__init__.py +++ b/mmdet3d/core/post_processing/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.core.post_processing import (merge_aug_bboxes, merge_aug_masks, merge_aug_proposals, merge_aug_scores, multiclass_nms) diff --git a/mmdet3d/core/post_processing/box3d_nms.py b/mmdet3d/core/post_processing/box3d_nms.py index 4c0b0dc867..e9d65d1bed 100644 --- a/mmdet3d/core/post_processing/box3d_nms.py +++ b/mmdet3d/core/post_processing/box3d_nms.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numba import numpy as np import torch diff --git a/mmdet3d/core/post_processing/merge_augs.py b/mmdet3d/core/post_processing/merge_augs.py index cf81c0b35b..e6bc8eb8a6 100644 --- a/mmdet3d/core/post_processing/merge_augs.py +++ b/mmdet3d/core/post_processing/merge_augs.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet3d.ops.iou3d.iou3d_utils import nms_gpu, nms_normal_gpu diff --git a/mmdet3d/core/utils/__init__.py b/mmdet3d/core/utils/__init__.py index 7aa874543c..99c88e1b80 100644 --- a/mmdet3d/core/utils/__init__.py +++ b/mmdet3d/core/utils/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .array_converter import ArrayConverter, array_converter from .gaussian import draw_heatmap_gaussian, gaussian_2d, gaussian_radius diff --git a/mmdet3d/core/utils/gaussian.py b/mmdet3d/core/utils/gaussian.py index aa1bc408a6..da9e3bb77e 100644 --- a/mmdet3d/core/utils/gaussian.py +++ b/mmdet3d/core/utils/gaussian.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/mmdet3d/core/visualizer/__init__.py b/mmdet3d/core/visualizer/__init__.py index 6c95282edb..bbf1e60fc4 100644 --- a/mmdet3d/core/visualizer/__init__.py +++ b/mmdet3d/core/visualizer/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .show_result import (show_multi_modality_result, show_result, show_seg_result) diff --git a/mmdet3d/core/visualizer/image_vis.py b/mmdet3d/core/visualizer/image_vis.py index 7e4679c529..4144bb7e5f 100644 --- a/mmdet3d/core/visualizer/image_vis.py +++ b/mmdet3d/core/visualizer/image_vis.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import cv2 import numpy as np diff --git a/mmdet3d/core/visualizer/open3d_vis.py b/mmdet3d/core/visualizer/open3d_vis.py index b15a9d2a9a..7a286b3c8a 100644 --- a/mmdet3d/core/visualizer/open3d_vis.py +++ b/mmdet3d/core/visualizer/open3d_vis.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import numpy as np import torch diff --git a/mmdet3d/core/visualizer/show_result.py b/mmdet3d/core/visualizer/show_result.py index fad828a4f9..b809bf755f 100644 --- a/mmdet3d/core/visualizer/show_result.py +++ b/mmdet3d/core/visualizer/show_result.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import trimesh diff --git a/mmdet3d/core/voxel/__init__.py b/mmdet3d/core/voxel/__init__.py index 7324f2521a..8d69543711 100644 --- a/mmdet3d/core/voxel/__init__.py +++ b/mmdet3d/core/voxel/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .builder import build_voxel_generator from .voxel_generator import VoxelGenerator diff --git a/mmdet3d/core/voxel/builder.py b/mmdet3d/core/voxel/builder.py index ec5c9d8c2e..bc663ee4ac 100644 --- a/mmdet3d/core/voxel/builder.py +++ b/mmdet3d/core/voxel/builder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv from . import voxel_generator diff --git a/mmdet3d/core/voxel/voxel_generator.py b/mmdet3d/core/voxel/voxel_generator.py index 748b347ef1..404f2cdc9b 100644 --- a/mmdet3d/core/voxel/voxel_generator.py +++ b/mmdet3d/core/voxel/voxel_generator.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numba import numpy as np diff --git a/mmdet3d/datasets/__init__.py b/mmdet3d/datasets/__init__.py index 2fcd650a1d..cb64c89d24 100644 --- a/mmdet3d/datasets/__init__.py +++ b/mmdet3d/datasets/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.datasets.builder import build_dataloader from .builder import DATASETS, build_dataset from .custom_3d import Custom3DDataset diff --git a/mmdet3d/datasets/builder.py b/mmdet3d/datasets/builder.py index 6263105aa7..00ebce0851 100644 --- a/mmdet3d/datasets/builder.py +++ b/mmdet3d/datasets/builder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import platform from mmcv.utils import Registry, build_from_cfg diff --git a/mmdet3d/datasets/custom_3d.py b/mmdet3d/datasets/custom_3d.py index d21a1b0a19..939390713c 100644 --- a/mmdet3d/datasets/custom_3d.py +++ b/mmdet3d/datasets/custom_3d.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import tempfile diff --git a/mmdet3d/datasets/custom_3d_seg.py b/mmdet3d/datasets/custom_3d_seg.py index 60d1cee36e..52b5d0b37f 100644 --- a/mmdet3d/datasets/custom_3d_seg.py +++ b/mmdet3d/datasets/custom_3d_seg.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import tempfile diff --git a/mmdet3d/datasets/dataset_wrappers.py b/mmdet3d/datasets/dataset_wrappers.py index 14eba04f13..2ae33279ea 100644 --- a/mmdet3d/datasets/dataset_wrappers.py +++ b/mmdet3d/datasets/dataset_wrappers.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np from .builder import DATASETS diff --git a/mmdet3d/datasets/kitti2d_dataset.py b/mmdet3d/datasets/kitti2d_dataset.py index 2004f90101..65d08eca11 100644 --- a/mmdet3d/datasets/kitti2d_dataset.py +++ b/mmdet3d/datasets/kitti2d_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np diff --git a/mmdet3d/datasets/kitti_dataset.py b/mmdet3d/datasets/kitti_dataset.py index 97d6917032..01fc8ae1c6 100644 --- a/mmdet3d/datasets/kitti_dataset.py +++ b/mmdet3d/datasets/kitti_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import mmcv import numpy as np diff --git a/mmdet3d/datasets/kitti_mono_dataset.py b/mmdet3d/datasets/kitti_mono_dataset.py index a3a17215e7..9b8e262f7b 100644 --- a/mmdet3d/datasets/kitti_mono_dataset.py +++ b/mmdet3d/datasets/kitti_mono_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import mmcv import numpy as np diff --git a/mmdet3d/datasets/lyft_dataset.py b/mmdet3d/datasets/lyft_dataset.py index f56450e53b..b1966bb428 100644 --- a/mmdet3d/datasets/lyft_dataset.py +++ b/mmdet3d/datasets/lyft_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import os diff --git a/mmdet3d/datasets/nuscenes_dataset.py b/mmdet3d/datasets/nuscenes_dataset.py index ac05a984fc..f5dae641e1 100644 --- a/mmdet3d/datasets/nuscenes_dataset.py +++ b/mmdet3d/datasets/nuscenes_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import pyquaternion diff --git a/mmdet3d/datasets/nuscenes_mono_dataset.py b/mmdet3d/datasets/nuscenes_mono_dataset.py index c39c68b648..f288e31950 100644 --- a/mmdet3d/datasets/nuscenes_mono_dataset.py +++ b/mmdet3d/datasets/nuscenes_mono_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import mmcv import numpy as np diff --git a/mmdet3d/datasets/pipelines/__init__.py b/mmdet3d/datasets/pipelines/__init__.py index f70114a063..68da65a0bf 100644 --- a/mmdet3d/datasets/pipelines/__init__.py +++ b/mmdet3d/datasets/pipelines/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.datasets.pipelines import Compose from .dbsampler import DataBaseSampler from .formating import Collect3D, DefaultFormatBundle, DefaultFormatBundle3D diff --git a/mmdet3d/datasets/pipelines/data_augment_utils.py b/mmdet3d/datasets/pipelines/data_augment_utils.py index 269b5e40b2..b68ca0c245 100644 --- a/mmdet3d/datasets/pipelines/data_augment_utils.py +++ b/mmdet3d/datasets/pipelines/data_augment_utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numba import numpy as np import warnings diff --git a/mmdet3d/datasets/pipelines/dbsampler.py b/mmdet3d/datasets/pipelines/dbsampler.py index 287da5ce26..82e9829db7 100644 --- a/mmdet3d/datasets/pipelines/dbsampler.py +++ b/mmdet3d/datasets/pipelines/dbsampler.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import mmcv import numpy as np diff --git a/mmdet3d/datasets/pipelines/formating.py b/mmdet3d/datasets/pipelines/formating.py index 06ec48fb30..2738992314 100644 --- a/mmdet3d/datasets/pipelines/formating.py +++ b/mmdet3d/datasets/pipelines/formating.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np from mmcv.parallel import DataContainer as DC diff --git a/mmdet3d/datasets/pipelines/loading.py b/mmdet3d/datasets/pipelines/loading.py index 21c9a6d226..f628b5041e 100644 --- a/mmdet3d/datasets/pipelines/loading.py +++ b/mmdet3d/datasets/pipelines/loading.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np diff --git a/mmdet3d/datasets/pipelines/test_time_aug.py b/mmdet3d/datasets/pipelines/test_time_aug.py index 9965ac23e4..398afc1339 100644 --- a/mmdet3d/datasets/pipelines/test_time_aug.py +++ b/mmdet3d/datasets/pipelines/test_time_aug.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import warnings from copy import deepcopy diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index 88706210d2..ad675dadef 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import warnings from mmcv import is_tuple_of diff --git a/mmdet3d/datasets/s3dis_dataset.py b/mmdet3d/datasets/s3dis_dataset.py index 69ac95e798..0955fb7b92 100644 --- a/mmdet3d/datasets/s3dis_dataset.py +++ b/mmdet3d/datasets/s3dis_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np from os import path as osp diff --git a/mmdet3d/datasets/scannet_dataset.py b/mmdet3d/datasets/scannet_dataset.py index 6854f72f7e..df03986f27 100644 --- a/mmdet3d/datasets/scannet_dataset.py +++ b/mmdet3d/datasets/scannet_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import tempfile import warnings diff --git a/mmdet3d/datasets/semantickitti_dataset.py b/mmdet3d/datasets/semantickitti_dataset.py index 446ff35f89..a87893f836 100644 --- a/mmdet3d/datasets/semantickitti_dataset.py +++ b/mmdet3d/datasets/semantickitti_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from os import path as osp from mmdet.datasets import DATASETS diff --git a/mmdet3d/datasets/sunrgbd_dataset.py b/mmdet3d/datasets/sunrgbd_dataset.py index 548a9683cc..4b9a5330ac 100644 --- a/mmdet3d/datasets/sunrgbd_dataset.py +++ b/mmdet3d/datasets/sunrgbd_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np from collections import OrderedDict from os import path as osp diff --git a/mmdet3d/datasets/utils.py b/mmdet3d/datasets/utils.py index 3264b725a9..2f4f09d26f 100644 --- a/mmdet3d/datasets/utils.py +++ b/mmdet3d/datasets/utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv # yapf: disable diff --git a/mmdet3d/datasets/waymo_dataset.py b/mmdet3d/datasets/waymo_dataset.py index 842aad5c9f..e5eeef50f7 100644 --- a/mmdet3d/datasets/waymo_dataset.py +++ b/mmdet3d/datasets/waymo_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import os diff --git a/mmdet3d/models/__init__.py b/mmdet3d/models/__init__.py index 50c453da70..e9a3925e3b 100644 --- a/mmdet3d/models/__init__.py +++ b/mmdet3d/models/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .backbones import * # noqa: F401,F403 from .builder import (FUSION_LAYERS, MIDDLE_ENCODERS, VOXEL_ENCODERS, build_backbone, build_detector, build_fusion_layer, diff --git a/mmdet3d/models/backbones/__init__.py b/mmdet3d/models/backbones/__init__.py index 469f879358..0251a10456 100644 --- a/mmdet3d/models/backbones/__init__.py +++ b/mmdet3d/models/backbones/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.backbones import SSDVGG, HRNet, ResNet, ResNetV1d, ResNeXt from .multi_backbone import MultiBackbone from .nostem_regnet import NoStemRegNet diff --git a/mmdet3d/models/backbones/base_pointnet.py b/mmdet3d/models/backbones/base_pointnet.py index 8ba5deecf0..fb7c05448b 100644 --- a/mmdet3d/models/backbones/base_pointnet.py +++ b/mmdet3d/models/backbones/base_pointnet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import warnings from abc import ABCMeta from mmcv.runner import BaseModule diff --git a/mmdet3d/models/backbones/multi_backbone.py b/mmdet3d/models/backbones/multi_backbone.py index 33374e85e3..b1e1da5b18 100644 --- a/mmdet3d/models/backbones/multi_backbone.py +++ b/mmdet3d/models/backbones/multi_backbone.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import torch import warnings diff --git a/mmdet3d/models/backbones/nostem_regnet.py b/mmdet3d/models/backbones/nostem_regnet.py index 75b4a5176d..e4c34e2451 100644 --- a/mmdet3d/models/backbones/nostem_regnet.py +++ b/mmdet3d/models/backbones/nostem_regnet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.backbones import RegNet from ..builder import BACKBONES diff --git a/mmdet3d/models/backbones/pointnet2_sa_msg.py b/mmdet3d/models/backbones/pointnet2_sa_msg.py index c641c5aaca..e2b0eca4a0 100644 --- a/mmdet3d/models/backbones/pointnet2_sa_msg.py +++ b/mmdet3d/models/backbones/pointnet2_sa_msg.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.cnn import ConvModule from mmcv.runner import auto_fp16 diff --git a/mmdet3d/models/backbones/pointnet2_sa_ssg.py b/mmdet3d/models/backbones/pointnet2_sa_ssg.py index 7cacf4a9df..eb5f4d6cad 100644 --- a/mmdet3d/models/backbones/pointnet2_sa_ssg.py +++ b/mmdet3d/models/backbones/pointnet2_sa_ssg.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.runner import auto_fp16 from torch import nn as nn diff --git a/mmdet3d/models/backbones/second.py b/mmdet3d/models/backbones/second.py index 01e853248a..680398c5d9 100644 --- a/mmdet3d/models/backbones/second.py +++ b/mmdet3d/models/backbones/second.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import warnings from mmcv.cnn import build_conv_layer, build_norm_layer from mmcv.runner import BaseModule diff --git a/mmdet3d/models/builder.py b/mmdet3d/models/builder.py index af5839ae50..09d062420b 100644 --- a/mmdet3d/models/builder.py +++ b/mmdet3d/models/builder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import warnings from mmcv.cnn import MODELS as MMCV_MODELS from mmcv.utils import Registry diff --git a/mmdet3d/models/decode_heads/__init__.py b/mmdet3d/models/decode_heads/__init__.py index 0ecf553392..e17d91da0c 100644 --- a/mmdet3d/models/decode_heads/__init__.py +++ b/mmdet3d/models/decode_heads/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .paconv_head import PAConvHead from .pointnet2_head import PointNet2Head diff --git a/mmdet3d/models/decode_heads/decode_head.py b/mmdet3d/models/decode_heads/decode_head.py index d3b4a695e5..fe9367d492 100644 --- a/mmdet3d/models/decode_heads/decode_head.py +++ b/mmdet3d/models/decode_heads/decode_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod from mmcv.cnn import normal_init from mmcv.runner import BaseModule, auto_fp16, force_fp32 diff --git a/mmdet3d/models/decode_heads/paconv_head.py b/mmdet3d/models/decode_heads/paconv_head.py index 62f1b9fcca..e662c976c2 100644 --- a/mmdet3d/models/decode_heads/paconv_head.py +++ b/mmdet3d/models/decode_heads/paconv_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.cnn.bricks import ConvModule from mmdet.models import HEADS diff --git a/mmdet3d/models/decode_heads/pointnet2_head.py b/mmdet3d/models/decode_heads/pointnet2_head.py index cc3e570b5f..0585df6ab4 100644 --- a/mmdet3d/models/decode_heads/pointnet2_head.py +++ b/mmdet3d/models/decode_heads/pointnet2_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.cnn.bricks import ConvModule from torch import nn as nn diff --git a/mmdet3d/models/dense_heads/__init__.py b/mmdet3d/models/dense_heads/__init__.py index b814d02d9b..d2283786db 100644 --- a/mmdet3d/models/dense_heads/__init__.py +++ b/mmdet3d/models/dense_heads/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .anchor3d_head import Anchor3DHead from .anchor_free_mono3d_head import AnchorFreeMono3DHead from .base_conv_bbox_head import BaseConvBboxHead diff --git a/mmdet3d/models/dense_heads/anchor3d_head.py b/mmdet3d/models/dense_heads/anchor3d_head.py index bb7681b6d3..14d70a2877 100644 --- a/mmdet3d/models/dense_heads/anchor3d_head.py +++ b/mmdet3d/models/dense_heads/anchor3d_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from mmcv.runner import BaseModule, force_fp32 diff --git a/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py b/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py index 90541118bb..a8131fc21e 100644 --- a/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py +++ b/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from abc import abstractmethod from mmcv.cnn import ConvModule, bias_init_with_prob, normal_init diff --git a/mmdet3d/models/dense_heads/base_conv_bbox_head.py b/mmdet3d/models/dense_heads/base_conv_bbox_head.py index 9958c93d32..dfd301d370 100644 --- a/mmdet3d/models/dense_heads/base_conv_bbox_head.py +++ b/mmdet3d/models/dense_heads/base_conv_bbox_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.cnn import ConvModule from mmcv.cnn.bricks import build_conv_layer from mmcv.runner import BaseModule diff --git a/mmdet3d/models/dense_heads/base_mono3d_dense_head.py b/mmdet3d/models/dense_heads/base_mono3d_dense_head.py index 05815acad4..322c7cf92d 100644 --- a/mmdet3d/models/dense_heads/base_mono3d_dense_head.py +++ b/mmdet3d/models/dense_heads/base_mono3d_dense_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod from mmcv.runner import BaseModule diff --git a/mmdet3d/models/dense_heads/centerpoint_head.py b/mmdet3d/models/dense_heads/centerpoint_head.py index e748e4e92a..b11dee9b4c 100644 --- a/mmdet3d/models/dense_heads/centerpoint_head.py +++ b/mmdet3d/models/dense_heads/centerpoint_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import numpy as np import torch diff --git a/mmdet3d/models/dense_heads/fcos_mono3d_head.py b/mmdet3d/models/dense_heads/fcos_mono3d_head.py index a9a794e446..c0b15a4581 100644 --- a/mmdet3d/models/dense_heads/fcos_mono3d_head.py +++ b/mmdet3d/models/dense_heads/fcos_mono3d_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from mmcv.cnn import Scale diff --git a/mmdet3d/models/dense_heads/free_anchor3d_head.py b/mmdet3d/models/dense_heads/free_anchor3d_head.py index 925220ec1c..533f600305 100644 --- a/mmdet3d/models/dense_heads/free_anchor3d_head.py +++ b/mmdet3d/models/dense_heads/free_anchor3d_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.runner import force_fp32 from torch.nn import functional as F diff --git a/mmdet3d/models/dense_heads/groupfree3d_head.py b/mmdet3d/models/dense_heads/groupfree3d_head.py index 4e37e26efc..9280e79cd3 100644 --- a/mmdet3d/models/dense_heads/groupfree3d_head.py +++ b/mmdet3d/models/dense_heads/groupfree3d_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import numpy as np import torch diff --git a/mmdet3d/models/dense_heads/parta2_rpn_head.py b/mmdet3d/models/dense_heads/parta2_rpn_head.py index 95c8adffb8..e6b5f9305c 100644 --- a/mmdet3d/models/dense_heads/parta2_rpn_head.py +++ b/mmdet3d/models/dense_heads/parta2_rpn_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from __future__ import division import numpy as np diff --git a/mmdet3d/models/dense_heads/shape_aware_head.py b/mmdet3d/models/dense_heads/shape_aware_head.py index 15d22a1f29..70b15f4fd6 100644 --- a/mmdet3d/models/dense_heads/shape_aware_head.py +++ b/mmdet3d/models/dense_heads/shape_aware_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch import warnings diff --git a/mmdet3d/models/dense_heads/ssd_3d_head.py b/mmdet3d/models/dense_heads/ssd_3d_head.py index 1da038d47d..83fc37cc2d 100644 --- a/mmdet3d/models/dense_heads/ssd_3d_head.py +++ b/mmdet3d/models/dense_heads/ssd_3d_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.ops.nms import batched_nms from mmcv.runner import force_fp32 diff --git a/mmdet3d/models/dense_heads/train_mixins.py b/mmdet3d/models/dense_heads/train_mixins.py index 6e4a8b3e38..c3bcf834de 100644 --- a/mmdet3d/models/dense_heads/train_mixins.py +++ b/mmdet3d/models/dense_heads/train_mixins.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/mmdet3d/models/dense_heads/vote_head.py b/mmdet3d/models/dense_heads/vote_head.py index 3887db1198..cc31460d56 100644 --- a/mmdet3d/models/dense_heads/vote_head.py +++ b/mmdet3d/models/dense_heads/vote_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from mmcv.runner import BaseModule, force_fp32 diff --git a/mmdet3d/models/detectors/__init__.py b/mmdet3d/models/detectors/__init__.py index b295c0aa06..a07454b6cc 100644 --- a/mmdet3d/models/detectors/__init__.py +++ b/mmdet3d/models/detectors/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .base import Base3DDetector from .centerpoint import CenterPoint from .dynamic_voxelnet import DynamicVoxelNet diff --git a/mmdet3d/models/detectors/base.py b/mmdet3d/models/detectors/base.py index b079d965f5..fd4b3882d0 100644 --- a/mmdet3d/models/detectors/base.py +++ b/mmdet3d/models/detectors/base.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import torch from mmcv.parallel import DataContainer as DC diff --git a/mmdet3d/models/detectors/centerpoint.py b/mmdet3d/models/detectors/centerpoint.py index c33d9dcd97..d6e971d2b9 100644 --- a/mmdet3d/models/detectors/centerpoint.py +++ b/mmdet3d/models/detectors/centerpoint.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet3d.core import bbox3d2result, merge_aug_bboxes_3d diff --git a/mmdet3d/models/detectors/dynamic_voxelnet.py b/mmdet3d/models/detectors/dynamic_voxelnet.py index c7defdff14..5e378e0385 100644 --- a/mmdet3d/models/detectors/dynamic_voxelnet.py +++ b/mmdet3d/models/detectors/dynamic_voxelnet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.runner import force_fp32 from torch.nn import functional as F diff --git a/mmdet3d/models/detectors/fcos_mono3d.py b/mmdet3d/models/detectors/fcos_mono3d.py index 89ddb6fe71..b1462a6c13 100644 --- a/mmdet3d/models/detectors/fcos_mono3d.py +++ b/mmdet3d/models/detectors/fcos_mono3d.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.builder import DETECTORS from .single_stage_mono3d import SingleStageMono3DDetector diff --git a/mmdet3d/models/detectors/groupfree3dnet.py b/mmdet3d/models/detectors/groupfree3dnet.py index 9ebac3284c..52b3fe8184 100644 --- a/mmdet3d/models/detectors/groupfree3dnet.py +++ b/mmdet3d/models/detectors/groupfree3dnet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet3d.core import bbox3d2result, merge_aug_bboxes_3d diff --git a/mmdet3d/models/detectors/h3dnet.py b/mmdet3d/models/detectors/h3dnet.py index 281a56d3b6..bbae09a8df 100644 --- a/mmdet3d/models/detectors/h3dnet.py +++ b/mmdet3d/models/detectors/h3dnet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet3d.core import merge_aug_bboxes_3d diff --git a/mmdet3d/models/detectors/imvotenet.py b/mmdet3d/models/detectors/imvotenet.py index 3f375eafc5..fdec49e225 100644 --- a/mmdet3d/models/detectors/imvotenet.py +++ b/mmdet3d/models/detectors/imvotenet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch import warnings diff --git a/mmdet3d/models/detectors/imvoxelnet.py b/mmdet3d/models/detectors/imvoxelnet.py index 2b70be7a54..fd5bb7c3f9 100644 --- a/mmdet3d/models/detectors/imvoxelnet.py +++ b/mmdet3d/models/detectors/imvoxelnet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet3d.core import bbox3d2result, build_anchor_generator diff --git a/mmdet3d/models/detectors/mvx_faster_rcnn.py b/mmdet3d/models/detectors/mvx_faster_rcnn.py index 40d6ade489..1e6f9a1a7e 100644 --- a/mmdet3d/models/detectors/mvx_faster_rcnn.py +++ b/mmdet3d/models/detectors/mvx_faster_rcnn.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.runner import force_fp32 from torch.nn import functional as F diff --git a/mmdet3d/models/detectors/mvx_two_stage.py b/mmdet3d/models/detectors/mvx_two_stage.py index 726a145618..0ca8b9686b 100644 --- a/mmdet3d/models/detectors/mvx_two_stage.py +++ b/mmdet3d/models/detectors/mvx_two_stage.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import torch import warnings diff --git a/mmdet3d/models/detectors/parta2.py b/mmdet3d/models/detectors/parta2.py index 8c907d2653..d39798b289 100644 --- a/mmdet3d/models/detectors/parta2.py +++ b/mmdet3d/models/detectors/parta2.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from torch.nn import functional as F diff --git a/mmdet3d/models/detectors/single_stage.py b/mmdet3d/models/detectors/single_stage.py index 3d78ed5115..a6c3d6baef 100644 --- a/mmdet3d/models/detectors/single_stage.py +++ b/mmdet3d/models/detectors/single_stage.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.models import DETECTORS, build_backbone, build_head, build_neck from .base import Base3DDetector diff --git a/mmdet3d/models/detectors/single_stage_mono3d.py b/mmdet3d/models/detectors/single_stage_mono3d.py index 3219154579..205091146e 100644 --- a/mmdet3d/models/detectors/single_stage_mono3d.py +++ b/mmdet3d/models/detectors/single_stage_mono3d.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import torch diff --git a/mmdet3d/models/detectors/ssd3dnet.py b/mmdet3d/models/detectors/ssd3dnet.py index 93d42d41e9..ef5191d9a5 100644 --- a/mmdet3d/models/detectors/ssd3dnet.py +++ b/mmdet3d/models/detectors/ssd3dnet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.models import DETECTORS from .votenet import VoteNet diff --git a/mmdet3d/models/detectors/two_stage.py b/mmdet3d/models/detectors/two_stage.py index fe8760d0e9..f5e14540ad 100644 --- a/mmdet3d/models/detectors/two_stage.py +++ b/mmdet3d/models/detectors/two_stage.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.models import DETECTORS, TwoStageDetector from .base import Base3DDetector diff --git a/mmdet3d/models/detectors/votenet.py b/mmdet3d/models/detectors/votenet.py index 384204d4ed..4ba0caa88f 100644 --- a/mmdet3d/models/detectors/votenet.py +++ b/mmdet3d/models/detectors/votenet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet3d.core import bbox3d2result, merge_aug_bboxes_3d diff --git a/mmdet3d/models/detectors/voxelnet.py b/mmdet3d/models/detectors/voxelnet.py index a556df7cd9..86c3239d3c 100644 --- a/mmdet3d/models/detectors/voxelnet.py +++ b/mmdet3d/models/detectors/voxelnet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.runner import force_fp32 from torch.nn import functional as F diff --git a/mmdet3d/models/fusion_layers/__init__.py b/mmdet3d/models/fusion_layers/__init__.py index 2ea683efa5..6df4741d78 100644 --- a/mmdet3d/models/fusion_layers/__init__.py +++ b/mmdet3d/models/fusion_layers/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .coord_transform import (apply_3d_transformation, bbox_2d_transform, coord_2d_transform) from .point_fusion import PointFusion diff --git a/mmdet3d/models/fusion_layers/coord_transform.py b/mmdet3d/models/fusion_layers/coord_transform.py index b1805f6b53..63e06599a4 100644 --- a/mmdet3d/models/fusion_layers/coord_transform.py +++ b/mmdet3d/models/fusion_layers/coord_transform.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from functools import partial diff --git a/mmdet3d/models/fusion_layers/point_fusion.py b/mmdet3d/models/fusion_layers/point_fusion.py index a6f4297d6c..6500280729 100644 --- a/mmdet3d/models/fusion_layers/point_fusion.py +++ b/mmdet3d/models/fusion_layers/point_fusion.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.cnn import ConvModule from mmcv.runner import BaseModule diff --git a/mmdet3d/models/fusion_layers/vote_fusion.py b/mmdet3d/models/fusion_layers/vote_fusion.py index ad65fdb7a9..3633e4d205 100644 --- a/mmdet3d/models/fusion_layers/vote_fusion.py +++ b/mmdet3d/models/fusion_layers/vote_fusion.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from torch import nn as nn diff --git a/mmdet3d/models/losses/__init__.py b/mmdet3d/models/losses/__init__.py index d14a0ca7df..7d4703aef9 100644 --- a/mmdet3d/models/losses/__init__.py +++ b/mmdet3d/models/losses/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.losses import FocalLoss, SmoothL1Loss, binary_cross_entropy from .axis_aligned_iou_loss import AxisAlignedIoULoss, axis_aligned_iou_loss from .chamfer_distance import ChamferDistance, chamfer_distance diff --git a/mmdet3d/models/losses/axis_aligned_iou_loss.py b/mmdet3d/models/losses/axis_aligned_iou_loss.py index a09acf2a9f..5ccef45012 100644 --- a/mmdet3d/models/losses/axis_aligned_iou_loss.py +++ b/mmdet3d/models/losses/axis_aligned_iou_loss.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from torch import nn as nn diff --git a/mmdet3d/models/losses/chamfer_distance.py b/mmdet3d/models/losses/chamfer_distance.py index 14e9a35f93..a0caebad78 100644 --- a/mmdet3d/models/losses/chamfer_distance.py +++ b/mmdet3d/models/losses/chamfer_distance.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from torch import nn as nn from torch.nn.functional import l1_loss, mse_loss, smooth_l1_loss diff --git a/mmdet3d/models/losses/paconv_regularization_loss.py b/mmdet3d/models/losses/paconv_regularization_loss.py index f2e3f2650f..cac96395e9 100644 --- a/mmdet3d/models/losses/paconv_regularization_loss.py +++ b/mmdet3d/models/losses/paconv_regularization_loss.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from torch import nn as nn diff --git a/mmdet3d/models/middle_encoders/__init__.py b/mmdet3d/models/middle_encoders/__init__.py index 93e53158d5..b3a7bdee97 100644 --- a/mmdet3d/models/middle_encoders/__init__.py +++ b/mmdet3d/models/middle_encoders/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .pillar_scatter import PointPillarsScatter from .sparse_encoder import SparseEncoder from .sparse_unet import SparseUNet diff --git a/mmdet3d/models/middle_encoders/pillar_scatter.py b/mmdet3d/models/middle_encoders/pillar_scatter.py index 3faff25901..9b8b727cb4 100644 --- a/mmdet3d/models/middle_encoders/pillar_scatter.py +++ b/mmdet3d/models/middle_encoders/pillar_scatter.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.runner import auto_fp16 from torch import nn diff --git a/mmdet3d/models/middle_encoders/sparse_encoder.py b/mmdet3d/models/middle_encoders/sparse_encoder.py index b406b0d41e..8b296c1a47 100644 --- a/mmdet3d/models/middle_encoders/sparse_encoder.py +++ b/mmdet3d/models/middle_encoders/sparse_encoder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.runner import auto_fp16 from torch import nn as nn diff --git a/mmdet3d/models/middle_encoders/sparse_unet.py b/mmdet3d/models/middle_encoders/sparse_unet.py index 34e1b2b6d5..d81fc50d02 100644 --- a/mmdet3d/models/middle_encoders/sparse_unet.py +++ b/mmdet3d/models/middle_encoders/sparse_unet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.runner import BaseModule, auto_fp16 diff --git a/mmdet3d/models/model_utils/__init__.py b/mmdet3d/models/model_utils/__init__.py index 87f73d27a9..83d4c4a8f6 100644 --- a/mmdet3d/models/model_utils/__init__.py +++ b/mmdet3d/models/model_utils/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .transformer import GroupFree3DMHA from .vote_module import VoteModule diff --git a/mmdet3d/models/model_utils/transformer.py b/mmdet3d/models/model_utils/transformer.py index c58cd7bf3d..3d4878dafb 100644 --- a/mmdet3d/models/model_utils/transformer.py +++ b/mmdet3d/models/model_utils/transformer.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.cnn.bricks.registry import ATTENTION from mmcv.cnn.bricks.transformer import POSITIONAL_ENCODING, MultiheadAttention from torch import nn as nn diff --git a/mmdet3d/models/model_utils/vote_module.py b/mmdet3d/models/model_utils/vote_module.py index f5e0f520d5..5cc52ad9d4 100644 --- a/mmdet3d/models/model_utils/vote_module.py +++ b/mmdet3d/models/model_utils/vote_module.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv import is_tuple_of from mmcv.cnn import ConvModule diff --git a/mmdet3d/models/necks/__init__.py b/mmdet3d/models/necks/__init__.py index 583b2521ca..9752a8b490 100644 --- a/mmdet3d/models/necks/__init__.py +++ b/mmdet3d/models/necks/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.necks.fpn import FPN from .imvoxel_neck import OutdoorImVoxelNeck from .second_fpn import SECONDFPN diff --git a/mmdet3d/models/necks/imvoxel_neck.py b/mmdet3d/models/necks/imvoxel_neck.py index 50be9cebca..de6799e02c 100644 --- a/mmdet3d/models/necks/imvoxel_neck.py +++ b/mmdet3d/models/necks/imvoxel_neck.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.cnn import ConvModule from torch import nn diff --git a/mmdet3d/models/necks/second_fpn.py b/mmdet3d/models/necks/second_fpn.py index d3f30e2a06..c5253617f8 100644 --- a/mmdet3d/models/necks/second_fpn.py +++ b/mmdet3d/models/necks/second_fpn.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from mmcv.cnn import build_conv_layer, build_norm_layer, build_upsample_layer diff --git a/mmdet3d/models/roi_heads/__init__.py b/mmdet3d/models/roi_heads/__init__.py index c93a3e3481..509c9ccb61 100644 --- a/mmdet3d/models/roi_heads/__init__.py +++ b/mmdet3d/models/roi_heads/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .base_3droi_head import Base3DRoIHead from .bbox_heads import PartA2BboxHead from .h3d_roi_head import H3DRoIHead diff --git a/mmdet3d/models/roi_heads/base_3droi_head.py b/mmdet3d/models/roi_heads/base_3droi_head.py index 3ab0645cce..b19b5a96b2 100644 --- a/mmdet3d/models/roi_heads/base_3droi_head.py +++ b/mmdet3d/models/roi_heads/base_3droi_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod from mmcv.runner import BaseModule diff --git a/mmdet3d/models/roi_heads/bbox_heads/__init__.py b/mmdet3d/models/roi_heads/bbox_heads/__init__.py index 1005cf614c..6294f52f4c 100644 --- a/mmdet3d/models/roi_heads/bbox_heads/__init__.py +++ b/mmdet3d/models/roi_heads/bbox_heads/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.roi_heads.bbox_heads import (BBoxHead, ConvFCBBoxHead, DoubleConvFCBBoxHead, Shared2FCBBoxHead, diff --git a/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py b/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py index f242e2a5aa..890d5b3d40 100644 --- a/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py +++ b/mmdet3d/models/roi_heads/bbox_heads/h3d_bbox_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.cnn import ConvModule from mmcv.runner import BaseModule diff --git a/mmdet3d/models/roi_heads/bbox_heads/parta2_bbox_head.py b/mmdet3d/models/roi_heads/bbox_heads/parta2_bbox_head.py index 2c2a6c3067..0d9f043f72 100644 --- a/mmdet3d/models/roi_heads/bbox_heads/parta2_bbox_head.py +++ b/mmdet3d/models/roi_heads/bbox_heads/parta2_bbox_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from mmcv.cnn import ConvModule, normal_init diff --git a/mmdet3d/models/roi_heads/h3d_roi_head.py b/mmdet3d/models/roi_heads/h3d_roi_head.py index c579cbd6cd..4bf8cf36b6 100644 --- a/mmdet3d/models/roi_heads/h3d_roi_head.py +++ b/mmdet3d/models/roi_heads/h3d_roi_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet3d.core.bbox import bbox3d2result from mmdet.models import HEADS from ..builder import build_head diff --git a/mmdet3d/models/roi_heads/mask_heads/__init__.py b/mmdet3d/models/roi_heads/mask_heads/__init__.py index ecc8a118a5..0aa11569ae 100644 --- a/mmdet3d/models/roi_heads/mask_heads/__init__.py +++ b/mmdet3d/models/roi_heads/mask_heads/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .pointwise_semantic_head import PointwiseSemanticHead from .primitive_head import PrimitiveHead diff --git a/mmdet3d/models/roi_heads/mask_heads/pointwise_semantic_head.py b/mmdet3d/models/roi_heads/mask_heads/pointwise_semantic_head.py index 41e4fae282..fbdd2f0da3 100644 --- a/mmdet3d/models/roi_heads/mask_heads/pointwise_semantic_head.py +++ b/mmdet3d/models/roi_heads/mask_heads/pointwise_semantic_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.runner import BaseModule from torch import nn as nn diff --git a/mmdet3d/models/roi_heads/mask_heads/primitive_head.py b/mmdet3d/models/roi_heads/mask_heads/primitive_head.py index ee95e3c601..c930ee9a98 100644 --- a/mmdet3d/models/roi_heads/mask_heads/primitive_head.py +++ b/mmdet3d/models/roi_heads/mask_heads/primitive_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.cnn import ConvModule from mmcv.runner import BaseModule diff --git a/mmdet3d/models/roi_heads/part_aggregation_roi_head.py b/mmdet3d/models/roi_heads/part_aggregation_roi_head.py index 2cc3e33687..7b6e2f656d 100644 --- a/mmdet3d/models/roi_heads/part_aggregation_roi_head.py +++ b/mmdet3d/models/roi_heads/part_aggregation_roi_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import warnings from torch.nn import functional as F diff --git a/mmdet3d/models/roi_heads/roi_extractors/__init__.py b/mmdet3d/models/roi_heads/roi_extractors/__init__.py index 24a4991047..70c28812bb 100644 --- a/mmdet3d/models/roi_heads/roi_extractors/__init__.py +++ b/mmdet3d/models/roi_heads/roi_extractors/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.roi_heads.roi_extractors import SingleRoIExtractor from .single_roiaware_extractor import Single3DRoIAwareExtractor from .single_roipoint_extractor import Single3DRoIPointExtractor diff --git a/mmdet3d/models/roi_heads/roi_extractors/single_roiaware_extractor.py b/mmdet3d/models/roi_heads/roi_extractors/single_roiaware_extractor.py index a859e8b30f..caa5a5bb94 100644 --- a/mmdet3d/models/roi_heads/roi_extractors/single_roiaware_extractor.py +++ b/mmdet3d/models/roi_heads/roi_extractors/single_roiaware_extractor.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.runner import BaseModule diff --git a/mmdet3d/models/segmentors/__init__.py b/mmdet3d/models/segmentors/__init__.py index e424e03bbb..29fbc33e6a 100644 --- a/mmdet3d/models/segmentors/__init__.py +++ b/mmdet3d/models/segmentors/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .base import Base3DSegmentor from .encoder_decoder import EncoderDecoder3D diff --git a/mmdet3d/models/segmentors/base.py b/mmdet3d/models/segmentors/base.py index eaa7acd02c..6b233c2555 100644 --- a/mmdet3d/models/segmentors/base.py +++ b/mmdet3d/models/segmentors/base.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import torch diff --git a/mmdet3d/models/segmentors/encoder_decoder.py b/mmdet3d/models/segmentors/encoder_decoder.py index 81ef29392e..196904ad1e 100644 --- a/mmdet3d/models/segmentors/encoder_decoder.py +++ b/mmdet3d/models/segmentors/encoder_decoder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from torch import nn as nn diff --git a/mmdet3d/models/utils/__init__.py b/mmdet3d/models/utils/__init__.py index 94aa1923e1..d0639301d0 100644 --- a/mmdet3d/models/utils/__init__.py +++ b/mmdet3d/models/utils/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .clip_sigmoid import clip_sigmoid from .mlp import MLP diff --git a/mmdet3d/models/utils/clip_sigmoid.py b/mmdet3d/models/utils/clip_sigmoid.py index aaed03f624..3afd4edbef 100644 --- a/mmdet3d/models/utils/clip_sigmoid.py +++ b/mmdet3d/models/utils/clip_sigmoid.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch diff --git a/mmdet3d/models/utils/mlp.py b/mmdet3d/models/utils/mlp.py index 48c73facd4..0b499bb46f 100644 --- a/mmdet3d/models/utils/mlp.py +++ b/mmdet3d/models/utils/mlp.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.cnn import ConvModule from mmcv.runner import BaseModule from torch import nn as nn diff --git a/mmdet3d/models/voxel_encoders/__init__.py b/mmdet3d/models/voxel_encoders/__init__.py index 5eb4bde4e1..d6cc629b5e 100644 --- a/mmdet3d/models/voxel_encoders/__init__.py +++ b/mmdet3d/models/voxel_encoders/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .pillar_encoder import PillarFeatureNet from .voxel_encoder import DynamicSimpleVFE, DynamicVFE, HardSimpleVFE, HardVFE diff --git a/mmdet3d/models/voxel_encoders/pillar_encoder.py b/mmdet3d/models/voxel_encoders/pillar_encoder.py index bfb5c5fe47..ddba9efbf7 100644 --- a/mmdet3d/models/voxel_encoders/pillar_encoder.py +++ b/mmdet3d/models/voxel_encoders/pillar_encoder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.cnn import build_norm_layer from mmcv.runner import force_fp32 diff --git a/mmdet3d/models/voxel_encoders/utils.py b/mmdet3d/models/voxel_encoders/utils.py index 2da7ffaedc..8c54fc2d16 100644 --- a/mmdet3d/models/voxel_encoders/utils.py +++ b/mmdet3d/models/voxel_encoders/utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.cnn import build_norm_layer from mmcv.runner import auto_fp16 diff --git a/mmdet3d/models/voxel_encoders/voxel_encoder.py b/mmdet3d/models/voxel_encoders/voxel_encoder.py index dbd67127f5..45be8b6c1c 100644 --- a/mmdet3d/models/voxel_encoders/voxel_encoder.py +++ b/mmdet3d/models/voxel_encoders/voxel_encoder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.cnn import build_norm_layer from mmcv.runner import force_fp32 diff --git a/mmdet3d/ops/__init__.py b/mmdet3d/ops/__init__.py index 8ce119f2e3..38e2ea7367 100644 --- a/mmdet3d/ops/__init__.py +++ b/mmdet3d/ops/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.ops import (RoIAlign, SigmoidFocalLoss, get_compiler_version, get_compiling_cuda_version, nms, roi_align, sigmoid_focal_loss) diff --git a/mmdet3d/ops/pointnet_modules/__init__.py b/mmdet3d/ops/pointnet_modules/__init__.py index 7b803ea0c8..99b08eb889 100644 --- a/mmdet3d/ops/pointnet_modules/__init__.py +++ b/mmdet3d/ops/pointnet_modules/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .builder import build_sa_module from .paconv_sa_module import (PAConvCUDASAModule, PAConvCUDASAModuleMSG, PAConvSAModule, PAConvSAModuleMSG) diff --git a/mmdet3d/ops/pointnet_modules/builder.py b/mmdet3d/ops/pointnet_modules/builder.py index e27ef0da36..6631cb4248 100644 --- a/mmdet3d/ops/pointnet_modules/builder.py +++ b/mmdet3d/ops/pointnet_modules/builder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.utils import Registry SA_MODULES = Registry('point_sa_module') diff --git a/mmdet3d/ops/pointnet_modules/paconv_sa_module.py b/mmdet3d/ops/pointnet_modules/paconv_sa_module.py index 19040504b8..6a4308cfdd 100644 --- a/mmdet3d/ops/pointnet_modules/paconv_sa_module.py +++ b/mmdet3d/ops/pointnet_modules/paconv_sa_module.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from torch import nn as nn diff --git a/mmdet3d/ops/pointnet_modules/point_fp_module.py b/mmdet3d/ops/pointnet_modules/point_fp_module.py index 59161d14fa..d97d5ffe31 100644 --- a/mmdet3d/ops/pointnet_modules/point_fp_module.py +++ b/mmdet3d/ops/pointnet_modules/point_fp_module.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.cnn import ConvModule from mmcv.runner import BaseModule, force_fp32 diff --git a/mmdet3d/ops/pointnet_modules/point_sa_module.py b/mmdet3d/ops/pointnet_modules/point_sa_module.py index f640b3d386..293d6242d7 100644 --- a/mmdet3d/ops/pointnet_modules/point_sa_module.py +++ b/mmdet3d/ops/pointnet_modules/point_sa_module.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.cnn import ConvModule from torch import nn as nn diff --git a/mmdet3d/ops/sparse_block.py b/mmdet3d/ops/sparse_block.py index c89cd53db5..0539736342 100644 --- a/mmdet3d/ops/sparse_block.py +++ b/mmdet3d/ops/sparse_block.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.cnn import build_conv_layer, build_norm_layer from torch import nn diff --git a/mmdet3d/utils/__init__.py b/mmdet3d/utils/__init__.py index bed830a93a..e9958abbee 100644 --- a/mmdet3d/utils/__init__.py +++ b/mmdet3d/utils/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.utils import Registry, build_from_cfg, print_log from .collect_env import collect_env diff --git a/mmdet3d/utils/collect_env.py b/mmdet3d/utils/collect_env.py index 25c4f00705..ccd96b10d0 100644 --- a/mmdet3d/utils/collect_env.py +++ b/mmdet3d/utils/collect_env.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.utils import collect_env as collect_base_env from mmcv.utils import get_git_hash diff --git a/mmdet3d/utils/logger.py b/mmdet3d/utils/logger.py index ee10ee55ad..d9f08918c9 100644 --- a/mmdet3d/utils/logger.py +++ b/mmdet3d/utils/logger.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import logging from mmcv.utils import get_logger diff --git a/tests/test_data/test_datasets/test_dataset_wrappers.py b/tests/test_data/test_datasets/test_dataset_wrappers.py index aa87f0bda1..bcf183f41d 100644 --- a/tests/test_data/test_datasets/test_dataset_wrappers.py +++ b/tests/test_data/test_datasets/test_dataset_wrappers.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/tests/test_data/test_datasets/test_kitti_dataset.py b/tests/test_data/test_datasets/test_kitti_dataset.py index 19dd58cf6b..599f707c8f 100644 --- a/tests/test_data/test_datasets/test_kitti_dataset.py +++ b/tests/test_data/test_datasets/test_kitti_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import math import numpy as np import os diff --git a/tests/test_data/test_datasets/test_kitti_mono_dataset.py b/tests/test_data/test_datasets/test_kitti_mono_dataset.py index 23ab8dd62c..2884809697 100644 --- a/tests/test_data/test_datasets/test_kitti_mono_dataset.py +++ b/tests/test_data/test_datasets/test_kitti_mono_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import pytest diff --git a/tests/test_data/test_datasets/test_lyft_dataset.py b/tests/test_data/test_datasets/test_lyft_dataset.py index 4bdfd1234e..f7a07552fd 100644 --- a/tests/test_data/test_datasets/test_lyft_dataset.py +++ b/tests/test_data/test_datasets/test_lyft_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import tempfile diff --git a/tests/test_data/test_datasets/test_nuscene_dataset.py b/tests/test_data/test_datasets/test_nuscene_dataset.py index ccb6cf0ccb..e2e8b55fd9 100644 --- a/tests/test_data/test_datasets/test_nuscene_dataset.py +++ b/tests/test_data/test_datasets/test_nuscene_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import tempfile import torch diff --git a/tests/test_data/test_datasets/test_nuscenes_mono_dataset.py b/tests/test_data/test_datasets/test_nuscenes_mono_dataset.py index e0843c9176..f88775d2ef 100644 --- a/tests/test_data/test_datasets/test_nuscenes_mono_dataset.py +++ b/tests/test_data/test_datasets/test_nuscenes_mono_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import pytest diff --git a/tests/test_data/test_datasets/test_s3dis_dataset.py b/tests/test_data/test_datasets/test_s3dis_dataset.py index dd037d25c0..6d254bf473 100644 --- a/tests/test_data/test_datasets/test_s3dis_dataset.py +++ b/tests/test_data/test_datasets/test_s3dis_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_data/test_datasets/test_scannet_dataset.py b/tests/test_data/test_datasets/test_scannet_dataset.py index f098d2d983..e66f2b0020 100644 --- a/tests/test_data/test_datasets/test_scannet_dataset.py +++ b/tests/test_data/test_datasets/test_scannet_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import numpy as np import pytest diff --git a/tests/test_data/test_datasets/test_semantickitti_dataset.py b/tests/test_data/test_datasets/test_semantickitti_dataset.py index a6e31e63d9..908363a02c 100644 --- a/tests/test_data/test_datasets/test_semantickitti_dataset.py +++ b/tests/test_data/test_datasets/test_semantickitti_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np from mmdet3d.datasets import SemanticKITTIDataset diff --git a/tests/test_data/test_datasets/test_sunrgbd_dataset.py b/tests/test_data/test_datasets/test_sunrgbd_dataset.py index ea0156ecf8..6eb429abf1 100644 --- a/tests/test_data/test_datasets/test_sunrgbd_dataset.py +++ b/tests/test_data/test_datasets/test_sunrgbd_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_data/test_datasets/test_waymo_dataset.py b/tests/test_data/test_datasets/test_waymo_dataset.py index 41dbe3138f..6ff4c1bcf4 100644 --- a/tests/test_data/test_datasets/test_waymo_dataset.py +++ b/tests/test_data/test_datasets/test_waymo_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import tempfile diff --git a/tests/test_data/test_pipelines/test_augmentations/test_data_augment_utils.py b/tests/test_data/test_pipelines/test_augmentations/test_data_augment_utils.py index 843861f0d8..60dca9e1d9 100644 --- a/tests/test_data/test_pipelines/test_augmentations/test_data_augment_utils.py +++ b/tests/test_data/test_pipelines/test_augmentations/test_data_augment_utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np diff --git a/tests/test_data/test_pipelines/test_augmentations/test_test_augment_utils.py b/tests/test_data/test_pipelines/test_augmentations/test_test_augment_utils.py index 946c52998c..4e621d3b20 100644 --- a/tests/test_data/test_pipelines/test_augmentations/test_test_augment_utils.py +++ b/tests/test_data/test_pipelines/test_augmentations/test_test_augment_utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py b/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py index 985295aeef..5ac7aeda3e 100644 --- a/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py +++ b/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import pytest diff --git a/tests/test_data/test_pipelines/test_indoor_pipeline.py b/tests/test_data/test_pipelines/test_indoor_pipeline.py index ec2e9e430a..6fd9ea4ef5 100644 --- a/tests/test_data/test_pipelines/test_indoor_pipeline.py +++ b/tests/test_data/test_pipelines/test_indoor_pipeline.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import torch diff --git a/tests/test_data/test_pipelines/test_indoor_sample.py b/tests/test_data/test_pipelines/test_indoor_sample.py index b1144d19c2..d8ce10006e 100644 --- a/tests/test_data/test_pipelines/test_indoor_sample.py +++ b/tests/test_data/test_pipelines/test_indoor_sample.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np from mmdet3d.core.points import DepthPoints diff --git a/tests/test_data/test_pipelines/test_loadings/test_load_images_from_multi_views.py b/tests/test_data/test_pipelines/test_loadings/test_load_images_from_multi_views.py index 9c227160e9..925c9497d5 100644 --- a/tests/test_data/test_pipelines/test_loadings/test_load_images_from_multi_views.py +++ b/tests/test_data/test_pipelines/test_loadings/test_load_images_from_multi_views.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from mmcv.parallel import DataContainer diff --git a/tests/test_data/test_pipelines/test_loadings/test_load_points_from_multi_sweeps.py b/tests/test_data/test_pipelines/test_loadings/test_load_points_from_multi_sweeps.py index 9bcbeac6d2..88d8b32773 100644 --- a/tests/test_data/test_pipelines/test_loadings/test_load_points_from_multi_sweeps.py +++ b/tests/test_data/test_pipelines/test_loadings/test_load_points_from_multi_sweeps.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np from mmdet3d.core.points import LiDARPoints diff --git a/tests/test_data/test_pipelines/test_loadings/test_loading.py b/tests/test_data/test_pipelines/test_loadings/test_loading.py index 8ff429aaad..f360da2748 100644 --- a/tests/test_data/test_pipelines/test_loadings/test_loading.py +++ b/tests/test_data/test_pipelines/test_loadings/test_loading.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import pytest diff --git a/tests/test_data/test_pipelines/test_outdoor_pipeline.py b/tests/test_data/test_pipelines/test_outdoor_pipeline.py index 90037ceff0..9d5a591893 100644 --- a/tests/test_data/test_pipelines/test_outdoor_pipeline.py +++ b/tests/test_data/test_pipelines/test_outdoor_pipeline.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/tests/test_metrics/test_indoor_eval.py b/tests/test_metrics/test_indoor_eval.py index 2c6defcf74..2842c588b6 100644 --- a/tests/test_metrics/test_indoor_eval.py +++ b/tests/test_metrics/test_indoor_eval.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_metrics/test_kitti_eval.py b/tests/test_metrics/test_kitti_eval.py index e671cabd48..7405e6b0a5 100644 --- a/tests/test_metrics/test_kitti_eval.py +++ b/tests/test_metrics/test_kitti_eval.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_metrics/test_losses.py b/tests/test_metrics/test_losses.py index ef24128c9d..08cdb6275c 100644 --- a/tests/test_metrics/test_losses.py +++ b/tests/test_metrics/test_losses.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from torch import nn as nn diff --git a/tests/test_metrics/test_seg_eval.py b/tests/test_metrics/test_seg_eval.py index f921c9e3e1..193fc89ade 100644 --- a/tests/test_metrics/test_seg_eval.py +++ b/tests/test_metrics/test_seg_eval.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_models/test_backbones.py b/tests/test_models/test_backbones.py index b8a13959e0..6a3d2cb422 100644 --- a/tests/test_models/test_backbones.py +++ b/tests/test_models/test_backbones.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_models/test_common_modules/test_middle_encoders.py b/tests/test_models/test_common_modules/test_middle_encoders.py index 28dfc77e02..df9bbf2adc 100644 --- a/tests/test_models/test_common_modules/test_middle_encoders.py +++ b/tests/test_models/test_common_modules/test_middle_encoders.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import pytest import torch diff --git a/tests/test_models/test_common_modules/test_paconv_modules.py b/tests/test_models/test_common_modules/test_paconv_modules.py index cb69bac8b8..2ca12744ef 100644 --- a/tests/test_models/test_common_modules/test_paconv_modules.py +++ b/tests/test_models/test_common_modules/test_paconv_modules.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_models/test_common_modules/test_paconv_ops.py b/tests/test_models/test_common_modules/test_paconv_ops.py index 6add7359db..9c2d7f420e 100644 --- a/tests/test_models/test_common_modules/test_paconv_ops.py +++ b/tests/test_models/test_common_modules/test_paconv_ops.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import pytest import torch diff --git a/tests/test_models/test_common_modules/test_pointnet_modules.py b/tests/test_models/test_common_modules/test_pointnet_modules.py index 929893b354..a3a96cbea7 100644 --- a/tests/test_models/test_common_modules/test_pointnet_modules.py +++ b/tests/test_models/test_common_modules/test_pointnet_modules.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_models/test_common_modules/test_pointnet_ops.py b/tests/test_models/test_common_modules/test_pointnet_ops.py index a8f4ef106a..095c2d01ca 100644 --- a/tests/test_models/test_common_modules/test_pointnet_ops.py +++ b/tests/test_models/test_common_modules/test_pointnet_ops.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import pytest import torch diff --git a/tests/test_models/test_common_modules/test_roiaware_pool3d.py b/tests/test_models/test_common_modules/test_roiaware_pool3d.py index c6d104cd35..90559dc976 100644 --- a/tests/test_models/test_common_modules/test_roiaware_pool3d.py +++ b/tests/test_models/test_common_modules/test_roiaware_pool3d.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_models/test_common_modules/test_sparse_unet.py b/tests/test_models/test_common_modules/test_sparse_unet.py index 1a2d219b59..5685782949 100644 --- a/tests/test_models/test_common_modules/test_sparse_unet.py +++ b/tests/test_models/test_common_modules/test_sparse_unet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet3d.ops import SparseBasicBlock diff --git a/tests/test_models/test_common_modules/test_vote_module.py b/tests/test_models/test_common_modules/test_vote_module.py index bf64866a80..30e6b93ca2 100644 --- a/tests/test_models/test_common_modules/test_vote_module.py +++ b/tests/test_models/test_common_modules/test_vote_module.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch diff --git a/tests/test_models/test_detectors.py b/tests/test_models/test_detectors.py index 204264e803..dc7a70812b 100644 --- a/tests/test_models/test_detectors.py +++ b/tests/test_models/test_detectors.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import numpy as np import pytest diff --git a/tests/test_models/test_forward.py b/tests/test_models/test_forward.py index 5f83722cef..107f199241 100644 --- a/tests/test_models/test_forward.py +++ b/tests/test_models/test_forward.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. """Test model forward process. CommandLine: diff --git a/tests/test_models/test_fusion/test_fusion_coord_trans.py b/tests/test_models/test_fusion/test_fusion_coord_trans.py index 88c1cae5c1..4df63825d3 100644 --- a/tests/test_models/test_fusion/test_fusion_coord_trans.py +++ b/tests/test_models/test_fusion/test_fusion_coord_trans.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. """Tests coords transformation in fusion modules. CommandLine: diff --git a/tests/test_models/test_fusion/test_point_fusion.py b/tests/test_models/test_fusion/test_point_fusion.py index 31911b0ba1..d976a74d6b 100644 --- a/tests/test_models/test_fusion/test_point_fusion.py +++ b/tests/test_models/test_fusion/test_point_fusion.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. """Tests the core function of point fusion. CommandLine: diff --git a/tests/test_models/test_fusion/test_vote_fusion.py b/tests/test_models/test_fusion/test_vote_fusion.py index 47fa297e11..a4c2e05f25 100644 --- a/tests/test_models/test_fusion/test_vote_fusion.py +++ b/tests/test_models/test_fusion/test_vote_fusion.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. """Tests the core function of vote fusion. CommandLine: diff --git a/tests/test_models/test_heads/test_heads.py b/tests/test_models/test_heads/test_heads.py index 3e89eaf07d..63cd1f8346 100644 --- a/tests/test_models/test_heads/test_heads.py +++ b/tests/test_models/test_heads/test_heads.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import numpy as np import pytest diff --git a/tests/test_models/test_heads/test_paconv_decode_head.py b/tests/test_models/test_heads/test_paconv_decode_head.py index 4b152100e2..e5e57c82ec 100644 --- a/tests/test_models/test_heads/test_paconv_decode_head.py +++ b/tests/test_models/test_heads/test_paconv_decode_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_models/test_heads/test_parta2_bbox_head.py b/tests/test_models/test_heads/test_parta2_bbox_head.py index 27174d30ea..140e72e087 100644 --- a/tests/test_models/test_heads/test_parta2_bbox_head.py +++ b/tests/test_models/test_heads/test_parta2_bbox_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmcv import Config diff --git a/tests/test_models/test_heads/test_pointnet2_decode_head.py b/tests/test_models/test_heads/test_pointnet2_decode_head.py index cc4b5afd6a..5e6e40f462 100644 --- a/tests/test_models/test_heads/test_pointnet2_decode_head.py +++ b/tests/test_models/test_heads/test_pointnet2_decode_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_models/test_heads/test_roi_extractors.py b/tests/test_models/test_heads/test_roi_extractors.py index 8ff941443d..13296316ed 100644 --- a/tests/test_models/test_heads/test_roi_extractors.py +++ b/tests/test_models/test_heads/test_roi_extractors.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_models/test_heads/test_semantic_heads.py b/tests/test_models/test_heads/test_semantic_heads.py index ac0e13f4a8..e259ecbff7 100644 --- a/tests/test_models/test_heads/test_semantic_heads.py +++ b/tests/test_models/test_heads/test_semantic_heads.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import pytest import torch diff --git a/tests/test_models/test_necks/test_fpn.py b/tests/test_models/test_necks/test_fpn.py index 1b089a1ddb..a623ebf999 100644 --- a/tests/test_models/test_necks/test_fpn.py +++ b/tests/test_models/test_necks/test_fpn.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import pytest diff --git a/tests/test_models/test_necks/test_necks.py b/tests/test_models/test_necks/test_necks.py index 960742ef68..1e48493cb8 100644 --- a/tests/test_models/test_necks/test_necks.py +++ b/tests/test_models/test_necks/test_necks.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import pytest import torch diff --git a/tests/test_models/test_segmentors.py b/tests/test_models/test_segmentors.py index 3b46a86a4d..faff3f9515 100644 --- a/tests/test_models/test_segmentors.py +++ b/tests/test_models/test_segmentors.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import numpy as np import pytest diff --git a/tests/test_models/test_voxel_encoder/test_dynamic_scatter.py b/tests/test_models/test_voxel_encoder/test_dynamic_scatter.py index dfb7c417a4..928c627998 100644 --- a/tests/test_models/test_voxel_encoder/test_dynamic_scatter.py +++ b/tests/test_models/test_voxel_encoder/test_dynamic_scatter.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from torch.autograd import gradcheck diff --git a/tests/test_models/test_voxel_encoder/test_voxel_encoders.py b/tests/test_models/test_voxel_encoder/test_voxel_encoders.py index f7503a8c34..9bee9c9fd9 100644 --- a/tests/test_models/test_voxel_encoder/test_voxel_encoders.py +++ b/tests/test_models/test_voxel_encoder/test_voxel_encoders.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet3d.models.builder import build_voxel_encoder diff --git a/tests/test_models/test_voxel_encoder/test_voxel_generator.py b/tests/test_models/test_voxel_encoder/test_voxel_generator.py index a49caece5c..2f89aa90b7 100644 --- a/tests/test_models/test_voxel_encoder/test_voxel_generator.py +++ b/tests/test_models/test_voxel_encoder/test_voxel_generator.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np from mmdet3d.core.voxel.voxel_generator import VoxelGenerator diff --git a/tests/test_models/test_voxel_encoder/test_voxelize.py b/tests/test_models/test_voxel_encoder/test_voxelize.py index 17ad127e9b..55899c55a9 100644 --- a/tests/test_models/test_voxel_encoder/test_voxelize.py +++ b/tests/test_models/test_voxel_encoder/test_voxelize.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_runtime/test_apis.py b/tests/test_runtime/test_apis.py index dc5e2c2dd5..be1f555457 100644 --- a/tests/test_runtime/test_apis.py +++ b/tests/test_runtime/test_apis.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import os import pytest diff --git a/tests/test_runtime/test_config.py b/tests/test_runtime/test_config.py index e1ce39a11d..a1950e283b 100644 --- a/tests/test_runtime/test_config.py +++ b/tests/test_runtime/test_config.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from os.path import dirname, exists, join, relpath diff --git a/tests/test_utils/test_anchors.py b/tests/test_utils/test_anchors.py index e052e5ed86..d11c2616e2 100644 --- a/tests/test_utils/test_anchors.py +++ b/tests/test_utils/test_anchors.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. """ CommandLine: pytest tests/test_utils/test_anchor.py diff --git a/tests/test_utils/test_assigners.py b/tests/test_utils/test_assigners.py index 38e58bfe47..a2e910edeb 100644 --- a/tests/test_utils/test_assigners.py +++ b/tests/test_utils/test_assigners.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. """Tests the Assigner objects. CommandLine: diff --git a/tests/test_utils/test_bbox_coders.py b/tests/test_utils/test_bbox_coders.py index ceae5409ed..1e9e236d81 100644 --- a/tests/test_utils/test_bbox_coders.py +++ b/tests/test_utils/test_bbox_coders.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet3d.core.bbox import DepthInstance3DBoxes, LiDARInstance3DBoxes diff --git a/tests/test_utils/test_box3d.py b/tests/test_utils/test_box3d.py index 2520ce9aca..974f22caec 100644 --- a/tests/test_utils/test_box3d.py +++ b/tests/test_utils/test_box3d.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_utils/test_box_np_ops.py b/tests/test_utils/test_box_np_ops.py index 4a4ccc14cf..1c6275de52 100644 --- a/tests/test_utils/test_box_np_ops.py +++ b/tests/test_utils/test_box_np_ops.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np diff --git a/tests/test_utils/test_coord_3d_mode.py b/tests/test_utils/test_coord_3d_mode.py index 6a18120b77..24f0e192c8 100644 --- a/tests/test_utils/test_coord_3d_mode.py +++ b/tests/test_utils/test_coord_3d_mode.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/tests/test_utils/test_merge_augs.py b/tests/test_utils/test_merge_augs.py index 555fe283db..79ae16c7ba 100644 --- a/tests/test_utils/test_merge_augs.py +++ b/tests/test_utils/test_merge_augs.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import pytest import torch diff --git a/tests/test_utils/test_nms.py b/tests/test_utils/test_nms.py index 6802bcc2b2..aca33f87f1 100644 --- a/tests/test_utils/test_nms.py +++ b/tests/test_utils/test_nms.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch diff --git a/tests/test_utils/test_points.py b/tests/test_utils/test_points.py index e63936b3bd..20af27fc24 100644 --- a/tests/test_utils/test_points.py +++ b/tests/test_utils/test_points.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tests/test_utils/test_samplers.py b/tests/test_utils/test_samplers.py index a628221746..236a6bd4fe 100644 --- a/tests/test_utils/test_samplers.py +++ b/tests/test_utils/test_samplers.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import pytest import torch diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 00a313018f..0aa9732c90 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch diff --git a/tools/analysis_tools/analyze_logs.py b/tools/analysis_tools/analyze_logs.py index 4d4ea7c4cb..806175f34c 100644 --- a/tools/analysis_tools/analyze_logs.py +++ b/tools/analysis_tools/analyze_logs.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import json import numpy as np diff --git a/tools/analysis_tools/benchmark.py b/tools/analysis_tools/benchmark.py index 748fe48537..17c9dd35f1 100644 --- a/tools/analysis_tools/benchmark.py +++ b/tools/analysis_tools/benchmark.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import time import torch diff --git a/tools/analysis_tools/get_flops.py b/tools/analysis_tools/get_flops.py index 3b096b0901..de364980c3 100644 --- a/tools/analysis_tools/get_flops.py +++ b/tools/analysis_tools/get_flops.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import torch from mmcv import Config, DictAction diff --git a/tools/create_data.py b/tools/create_data.py index 8bb4bed5e5..798e7dfc78 100644 --- a/tools/create_data.py +++ b/tools/create_data.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse from os import path as osp diff --git a/tools/data_converter/__init__.py b/tools/data_converter/__init__.py index e69de29bb2..ef101fec61 100644 --- a/tools/data_converter/__init__.py +++ b/tools/data_converter/__init__.py @@ -0,0 +1 @@ +# Copyright (c) OpenMMLab. All rights reserved. diff --git a/tools/data_converter/create_gt_database.py b/tools/data_converter/create_gt_database.py index 86040e1800..3e62977160 100644 --- a/tools/data_converter/create_gt_database.py +++ b/tools/data_converter/create_gt_database.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import pickle diff --git a/tools/data_converter/indoor_converter.py b/tools/data_converter/indoor_converter.py index 66761b638c..1178f3c1ea 100644 --- a/tools/data_converter/indoor_converter.py +++ b/tools/data_converter/indoor_converter.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import os diff --git a/tools/data_converter/kitti_converter.py b/tools/data_converter/kitti_converter.py index 7784a6d203..cc470b6996 100644 --- a/tools/data_converter/kitti_converter.py +++ b/tools/data_converter/kitti_converter.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np from collections import OrderedDict diff --git a/tools/data_converter/kitti_data_utils.py b/tools/data_converter/kitti_data_utils.py index e456260f2d..01538e065b 100644 --- a/tools/data_converter/kitti_data_utils.py +++ b/tools/data_converter/kitti_data_utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import numpy as np from collections import OrderedDict from concurrent import futures as futures diff --git a/tools/data_converter/lyft_converter.py b/tools/data_converter/lyft_converter.py index a31989809c..ba35dea239 100644 --- a/tools/data_converter/lyft_converter.py +++ b/tools/data_converter/lyft_converter.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import os diff --git a/tools/data_converter/lyft_data_fixer.py b/tools/data_converter/lyft_data_fixer.py index 629c8e075e..42070490ca 100644 --- a/tools/data_converter/lyft_data_fixer.py +++ b/tools/data_converter/lyft_data_fixer.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import numpy as np import os diff --git a/tools/data_converter/nuimage_converter.py b/tools/data_converter/nuimage_converter.py index 561eecfd8b..92be1de3db 100644 --- a/tools/data_converter/nuimage_converter.py +++ b/tools/data_converter/nuimage_converter.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import base64 import mmcv diff --git a/tools/data_converter/nuscenes_converter.py b/tools/data_converter/nuscenes_converter.py index 0ae8c22a55..627212a601 100644 --- a/tools/data_converter/nuscenes_converter.py +++ b/tools/data_converter/nuscenes_converter.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import os diff --git a/tools/data_converter/s3dis_data_utils.py b/tools/data_converter/s3dis_data_utils.py index e356b0aef6..bc473fbd93 100644 --- a/tools/data_converter/s3dis_data_utils.py +++ b/tools/data_converter/s3dis_data_utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import os diff --git a/tools/data_converter/scannet_data_utils.py b/tools/data_converter/scannet_data_utils.py index 4570889b8b..4f341edb36 100644 --- a/tools/data_converter/scannet_data_utils.py +++ b/tools/data_converter/scannet_data_utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np import os diff --git a/tools/data_converter/sunrgbd_data_utils.py b/tools/data_converter/sunrgbd_data_utils.py index 48ab46333b..931d2a3eaa 100644 --- a/tools/data_converter/sunrgbd_data_utils.py +++ b/tools/data_converter/sunrgbd_data_utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import mmcv import numpy as np from concurrent import futures as futures diff --git a/tools/data_converter/waymo_converter.py b/tools/data_converter/waymo_converter.py index 272d293334..0642d0f714 100644 --- a/tools/data_converter/waymo_converter.py +++ b/tools/data_converter/waymo_converter.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. r"""Adapted from `Waymo to KITTI converter `_. """ diff --git a/tools/misc/browse_dataset.py b/tools/misc/browse_dataset.py index a7bbf476e0..e3419f66df 100644 --- a/tools/misc/browse_dataset.py +++ b/tools/misc/browse_dataset.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import numpy as np import warnings diff --git a/tools/misc/fuse_conv_bn.py b/tools/misc/fuse_conv_bn.py index 8ebfc837e8..d4e22018d6 100644 --- a/tools/misc/fuse_conv_bn.py +++ b/tools/misc/fuse_conv_bn.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import torch from mmcv.runner import save_checkpoint diff --git a/tools/misc/print_config.py b/tools/misc/print_config.py index dc1015c331..3100fc324b 100644 --- a/tools/misc/print_config.py +++ b/tools/misc/print_config.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse from mmcv import Config, DictAction diff --git a/tools/misc/visualize_results.py b/tools/misc/visualize_results.py index cd820dd33b..302adc50ec 100644 --- a/tools/misc/visualize_results.py +++ b/tools/misc/visualize_results.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import mmcv from mmcv import Config diff --git a/tools/model_converters/convert_votenet_checkpoints.py b/tools/model_converters/convert_votenet_checkpoints.py index 15db9ac02d..33792b00dd 100644 --- a/tools/model_converters/convert_votenet_checkpoints.py +++ b/tools/model_converters/convert_votenet_checkpoints.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import tempfile import torch diff --git a/tools/model_converters/publish_model.py b/tools/model_converters/publish_model.py index 39795f14a5..318fd46a65 100644 --- a/tools/model_converters/publish_model.py +++ b/tools/model_converters/publish_model.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import subprocess import torch diff --git a/tools/model_converters/regnet2mmdet.py b/tools/model_converters/regnet2mmdet.py index 51f77ff7fc..9dee3c878a 100644 --- a/tools/model_converters/regnet2mmdet.py +++ b/tools/model_converters/regnet2mmdet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import torch from collections import OrderedDict diff --git a/tools/test.py b/tools/test.py index 0815be97c8..a8eca66200 100644 --- a/tools/test.py +++ b/tools/test.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import argparse import mmcv import os diff --git a/tools/train.py b/tools/train.py index 622dbf6781..ec16c0004d 100644 --- a/tools/train.py +++ b/tools/train.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from __future__ import division import argparse From 63fd399ae2c1c03b21ca811d0fc2f42bf63bb7b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E6=81=A9=E6=B3=BD?= Date: Thu, 19 Aug 2021 14:23:50 +0800 Subject: [PATCH 14/51] [fix] fix typos (#872) --- docs_zh-CN/tutorials/customize_dataset.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs_zh-CN/tutorials/customize_dataset.md b/docs_zh-CN/tutorials/customize_dataset.md index ca4328ba74..33b2a9a938 100644 --- a/docs_zh-CN/tutorials/customize_dataset.md +++ b/docs_zh-CN/tutorials/customize_dataset.md @@ -353,6 +353,6 @@ data = dict( **注意** (与 MMDetection 相关)： -- 在 MMDetection v2.5.0 之前，一旦设置了上述的 classes，数据集将会自动的过滤没有真实标注狂的图像，然而却无法通过调整配置文件的方式来取消该行为，这会引起一定的疑惑：当没有设置 classes 的情况下，只有当选项中同时出现 `filter_empty_gt=True` 和 `test_mode=False` 时才会对数据集中没有真实标注框的图像进行过滤。在 MMDetection v2.5.0 之后，我们对图像过滤过程和类别修改过程进行分离，例如：不管配置文件中是否 classes 进行设置，数据集只会在设置 `filter_empty_gt=True` 和 `test_mode=False` 时对没有真实标注框的图像进行过滤。 因此，设置 classes 仅会影响用于训练的类别标注信息，用户可以自行决定是否需要对没有真实标注框的图像进行过滤。 +- 在 MMDetection v2.5.0 之前，一旦设置了上述的 classes，数据集将会自动的过滤没有真实标注框的图像，然而却无法通过调整配置文件的方式来取消该行为，这会引起一定的疑惑：当没有设置 classes 的情况下，只有当选项中同时出现 `filter_empty_gt=True` 和 `test_mode=False` 时才会对数据集中没有真实标注框的图像进行过滤。在 MMDetection v2.5.0 之后，我们对图像过滤过程和类别修改过程进行分离，例如：不管配置文件中是否 classes 进行设置，数据集只会在设置 `filter_empty_gt=True` 和 `test_mode=False` 时对没有真实标注框的图像进行过滤。 因此，设置 classes 仅会影响用于训练的类别标注信息，用户可以自行决定是否需要对没有真实标注框的图像进行过滤。 - 因为数据集的中间格式仅包含标注框的标签信息，并不包含类别名，因此在使用 `CustomDataset` 时，用户只能够通过离线的方式来过滤没有真实标注框的图像，而无法通过配置文件来实现过滤。 - 设置数据集类别和数据集过滤的特征将在之后进行重构，使得对应的特征更加便于使用。 From 884b5938c910c9e986c54b1cdaffe1bca568ecad Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Tue, 24 Aug 2021 14:49:01 +0800 Subject: [PATCH 15/51] Fix 3 unworking configs (#882) --- ...xel_second_secfpn_dcn_4x8_cyclic_tta_20e_nus.py | 2 -- ...sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py | 14 +++++++------- ...sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py | 14 +++++++------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/configs/centerpoint/centerpoint_0075voxel_second_secfpn_dcn_4x8_cyclic_tta_20e_nus.py b/configs/centerpoint/centerpoint_0075voxel_second_secfpn_dcn_4x8_cyclic_tta_20e_nus.py index 8ddae97166..770a11c68c 100644 --- a/configs/centerpoint/centerpoint_0075voxel_second_secfpn_dcn_4x8_cyclic_tta_20e_nus.py +++ b/configs/centerpoint/centerpoint_0075voxel_second_secfpn_dcn_4x8_cyclic_tta_20e_nus.py @@ -1,7 +1,5 @@ _base_ = './centerpoint_0075voxel_second_secfpn_dcn_4x8_cyclic_20e_nus.py' -test_cfg = dict(pts=dict(use_rotate_nms=True, max_num=500)) - point_cloud_range = [-54, -54, -5.0, 54, 54, 3.0] file_client_args = dict(backend='disk') class_names = [ diff --git a/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py b/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py index 624ae79f65..f4bfdc9894 100644 --- a/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py +++ b/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py @@ -24,16 +24,16 @@ 'car', 'truck', 'trailer', 'bus', 'construction_vehicle', 'bicycle', 'motorcycle', 'pedestrian', 'traffic_cone', 'barrier' ] -# file_client_args = dict(backend='disk') +file_client_args = dict(backend='disk') # Uncomment the following if use ceph or other file clients. # See https://mmcv.readthedocs.io/en/latest/api.html#mmcv.fileio.FileClient # for more details. -file_client_args = dict( - backend='petrel', - path_mapping=dict({ - './data/nuscenes/': 's3://nuscenes/nuscenes/', - 'data/nuscenes/': 's3://nuscenes/nuscenes/' - })) +# file_client_args = dict( +# backend='petrel', +# path_mapping=dict({ +# './data/nuscenes/': 's3://nuscenes/nuscenes/', +# 'data/nuscenes/': 's3://nuscenes/nuscenes/' +# })) train_pipeline = [ dict( type='LoadPointsFromFile', diff --git a/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py b/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py index 64fd332268..f2931a727a 100644 --- a/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py +++ b/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py @@ -24,16 +24,16 @@ 'car', 'truck', 'trailer', 'bus', 'construction_vehicle', 'bicycle', 'motorcycle', 'pedestrian', 'traffic_cone', 'barrier' ] -# file_client_args = dict(backend='disk') +file_client_args = dict(backend='disk') # Uncomment the following if use ceph or other file clients. # See https://mmcv.readthedocs.io/en/latest/api.html#mmcv.fileio.FileClient # for more details. -file_client_args = dict( - backend='petrel', - path_mapping=dict({ - './data/nuscenes/': 's3://nuscenes/nuscenes/', - 'data/nuscenes/': 's3://nuscenes/nuscenes/' - })) +# file_client_args = dict( +# backend='petrel', +# path_mapping=dict({ +# './data/nuscenes/': 's3://nuscenes/nuscenes/', +# 'data/nuscenes/': 's3://nuscenes/nuscenes/' +# })) train_pipeline = [ dict( type='LoadPointsFromFile', From d6881605192bb0f8699895e8f2a227c77a60539f Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Tue, 24 Aug 2021 14:58:53 +0800 Subject: [PATCH 16/51] [Fix] Fix `index.rst` for Chinese docs (#873) * Fix index.rst for zh docs * Change switch language --- docs/index.rst | 2 +- docs_zh-CN/index.rst | 24 +++++++++++++++++++----- docs_zh-CN/tutorials/index.rst | 1 - 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 8b3e565b05..dff69a5eed 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -55,7 +55,7 @@ Welcome to MMDetection3D's documentation! api.rst .. toctree:: - :caption: 语言切换 + :caption: Switch Language switch_language.md diff --git a/docs_zh-CN/index.rst b/docs_zh-CN/index.rst index 20cce8f252..0a4718b944 100644 --- a/docs_zh-CN/index.rst +++ b/docs_zh-CN/index.rst @@ -6,6 +6,7 @@ Welcome to MMDetection3D's documentation! :caption: 开始你的第一步 getting_started.md + demo.md model_zoo.md data_preparation.md @@ -13,10 +14,21 @@ Welcome to MMDetection3D's documentation! :maxdepth: 2 :caption: 快速启动 - 0_demo.md 1_exist_data_model.md 2_new_data_model.md +.. toctree:: + :maxdepth: 2 + :caption: 支持的任务 + + supported_tasks/index.rst + +.. toctree:: + :maxdepth: 2 + :caption: 数据集介绍 + + datasets/index.rst + .. toctree:: :maxdepth: 2 :caption: 教程 @@ -33,17 +45,19 @@ Welcome to MMDetection3D's documentation! :maxdepth: 2 :caption: 说明 + benchmarks.md faq.md + compatibility.md .. toctree:: - :caption: 语言切换 + :caption: 接口文档（英文） - switch_language.md + api.rst .. toctree:: - :caption: 接口文档（英文） + :caption: 语言切换 - api.rst + switch_language.md Indices and tables ================== diff --git a/docs_zh-CN/tutorials/index.rst b/docs_zh-CN/tutorials/index.rst index b5e1137a67..dec6ee8133 100644 --- a/docs_zh-CN/tutorials/index.rst +++ b/docs_zh-CN/tutorials/index.rst @@ -6,4 +6,3 @@ data_pipeline.md customize_models.md customize_runtime.md - waymo.md From a000db59d0d06823d4f79e727e4d35dfaaa82f87 Mon Sep 17 00:00:00 2001 From: Robin Karlsson <34254153+robin-karlsson0@users.noreply.github.com> Date: Wed, 25 Aug 2021 15:54:23 +0900 Subject: [PATCH 17/51] [Fix] Centerpoint head nested list transpose (#879) * FIX Transpose nested lists without Numpy * Removed unused Numpy import --- .../models/dense_heads/centerpoint_head.py | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/mmdet3d/models/dense_heads/centerpoint_head.py b/mmdet3d/models/dense_heads/centerpoint_head.py index b11dee9b4c..5189002b5e 100644 --- a/mmdet3d/models/dense_heads/centerpoint_head.py +++ b/mmdet3d/models/dense_heads/centerpoint_head.py @@ -1,6 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. import copy -import numpy as np import torch from mmcv.cnn import ConvModule, build_conv_layer from mmcv.runner import BaseModule, force_fp32 @@ -386,6 +385,17 @@ def _gather_feat(self, feat, ind, mask=None): def get_targets(self, gt_bboxes_3d, gt_labels_3d): """Generate targets. + How each output is transformed: + + Each nested list is transposed so that all same-index elements in + each sub-list (1, ..., N) become the new sub-lists. + [ [a0, a1, a2, ... ], [b0, b1, b2, ... ], ... ] + ==> [ [a0, b0, ... ], [a1, b1, ... ], [a2, b2, ... ] ] + + The new transposed nested list is converted into a list of N + tensors generated by concatenating tensors in the new sub-lists. + [ tensor0, tensor1, tensor2, ... ] + Args: gt_bboxes_3d (list[:obj:`LiDARInstance3DBoxes`]): Ground truth gt boxes. @@ -405,18 +415,17 @@ def get_targets(self, gt_bboxes_3d, gt_labels_3d): """ heatmaps, anno_boxes, inds, masks = multi_apply( self.get_targets_single, gt_bboxes_3d, gt_labels_3d) - # transpose heatmaps, because the dimension of tensors in each task is - # different, we have to use numpy instead of torch to do the transpose. - heatmaps = np.array(heatmaps).transpose(1, 0).tolist() + # Transpose heatmaps + heatmaps = list(map(list, zip(*heatmaps))) heatmaps = [torch.stack(hms_) for hms_ in heatmaps] - # transpose anno_boxes - anno_boxes = np.array(anno_boxes).transpose(1, 0).tolist() + # Transpose anno_boxes + anno_boxes = list(map(list, zip(*anno_boxes))) anno_boxes = [torch.stack(anno_boxes_) for anno_boxes_ in anno_boxes] - # transpose inds - inds = np.array(inds).transpose(1, 0).tolist() + # Transpose inds + inds = list(map(list, zip(*inds))) inds = [torch.stack(inds_) for inds_ in inds] - # transpose inds - masks = np.array(masks).transpose(1, 0).tolist() + # Transpose inds + masks = list(map(list, zip(*masks))) masks = [torch.stack(masks_) for masks_ in masks] return heatmaps, anno_boxes, inds, masks From 4e9c99269262f1fca33d372050f30cafbe7c4a61 Mon Sep 17 00:00:00 2001 From: Danila Rukhovich Date: Wed, 25 Aug 2021 16:30:00 +0300 Subject: [PATCH 18/51] [Enhance] Update PointFusion (#791) * update point fusion * remove LIDAR hardcode * move get_proj_mat_by_coord_type to utils * fix lint * remove todo * fix lint --- mmdet3d/core/bbox/structures/__init__.py | 7 +- mmdet3d/core/bbox/structures/utils.py | 17 +++++ mmdet3d/models/detectors/imvoxelnet.py | 3 +- .../models/fusion_layers/coord_transform.py | 6 +- mmdet3d/models/fusion_layers/point_fusion.py | 66 +++++++++---------- 5 files changed, 59 insertions(+), 40 deletions(-) diff --git a/mmdet3d/core/bbox/structures/__init__.py b/mmdet3d/core/bbox/structures/__init__.py index 8b010cf72d..58c111ef43 100644 --- a/mmdet3d/core/bbox/structures/__init__.py +++ b/mmdet3d/core/bbox/structures/__init__.py @@ -5,12 +5,13 @@ from .coord_3d_mode import Coord3DMode from .depth_box3d import DepthInstance3DBoxes from .lidar_box3d import LiDARInstance3DBoxes -from .utils import (get_box_type, limit_period, mono_cam_box2vis, - points_cam2img, rotation_3d_in_axis, xywhr2xyxyr) +from .utils import (get_box_type, get_proj_mat_by_coord_type, limit_period, + mono_cam_box2vis, points_cam2img, rotation_3d_in_axis, + xywhr2xyxyr) __all__ = [ 'Box3DMode', 'BaseInstance3DBoxes', 'LiDARInstance3DBoxes', 'CameraInstance3DBoxes', 'DepthInstance3DBoxes', 'xywhr2xyxyr', 'get_box_type', 'rotation_3d_in_axis', 'limit_period', 'points_cam2img', - 'Coord3DMode', 'mono_cam_box2vis' + 'Coord3DMode', 'mono_cam_box2vis', 'get_proj_mat_by_coord_type' ] diff --git a/mmdet3d/core/bbox/structures/utils.py b/mmdet3d/core/bbox/structures/utils.py index 2511fdb2a3..9ad4a6f6df 100644 --- a/mmdet3d/core/bbox/structures/utils.py +++ b/mmdet3d/core/bbox/structures/utils.py @@ -257,3 +257,20 @@ def mono_cam_box2vis(cam_box): cam_box, box_dim=cam_box.shape[-1], origin=(0.5, 0.5, 0.5)) return cam_box + + +def get_proj_mat_by_coord_type(img_meta, coord_type): + """Obtain image features using points. + + Args: + img_meta (dict): Meta info. + coord_type (str): 'DEPTH' or 'CAMERA' or 'LIDAR'. + Can be case-insensitive. + + Returns: + torch.Tensor: transformation matrix. + """ + coord_type = coord_type.upper() + mapping = {'LIDAR': 'lidar2img', 'DEPTH': 'depth2img', 'CAMERA': 'cam2img'} + assert coord_type in mapping.keys() + return img_meta[mapping[coord_type]] diff --git a/mmdet3d/models/detectors/imvoxelnet.py b/mmdet3d/models/detectors/imvoxelnet.py index fd5bb7c3f9..651d8d7a7e 100644 --- a/mmdet3d/models/detectors/imvoxelnet.py +++ b/mmdet3d/models/detectors/imvoxelnet.py @@ -61,7 +61,8 @@ def extract_feat(self, img, img_metas): img_meta, img_features=feature[None, ...], points=points, - lidar2img_rt=points.new_tensor(img_meta['lidar2img']), + proj_mat=points.new_tensor(img_meta['lidar2img']), + coord_type='LIDAR', img_scale_factor=img_scale_factor, img_crop_offset=img_crop_offset, img_flip=img_flip, diff --git a/mmdet3d/models/fusion_layers/coord_transform.py b/mmdet3d/models/fusion_layers/coord_transform.py index 63e06599a4..b3ad7297a0 100644 --- a/mmdet3d/models/fusion_layers/coord_transform.py +++ b/mmdet3d/models/fusion_layers/coord_transform.py @@ -5,12 +5,12 @@ from mmdet3d.core.points import get_points_type -def apply_3d_transformation(pcd, coords_type, img_meta, reverse=False): +def apply_3d_transformation(pcd, coord_type, img_meta, reverse=False): """Apply transformation to input point cloud. Args: pcd (torch.Tensor): The point cloud to be transformed. - coords_type (str): 'DEPTH' or 'CAMERA' or 'LIDAR' + coord_type (str): 'DEPTH' or 'CAMERA' or 'LIDAR'. img_meta(dict): Meta info regarding data transformation. reverse (bool): Reversed transformation or not. @@ -54,7 +54,7 @@ def apply_3d_transformation(pcd, coords_type, img_meta, reverse=False): if 'transformation_3d_flow' in img_meta else [] pcd = pcd.clone() # prevent inplace modification - pcd = get_points_type(coords_type)(pcd) + pcd = get_points_type(coord_type)(pcd) horizontal_flip_func = partial(pcd.flip, bev_direction='horizontal') \ if pcd_horizontal_flip else lambda: None diff --git a/mmdet3d/models/fusion_layers/point_fusion.py b/mmdet3d/models/fusion_layers/point_fusion.py index 6500280729..97b4177763 100644 --- a/mmdet3d/models/fusion_layers/point_fusion.py +++ b/mmdet3d/models/fusion_layers/point_fusion.py @@ -5,31 +5,33 @@ from torch import nn as nn from torch.nn import functional as F +from mmdet3d.core.bbox.structures import (get_proj_mat_by_coord_type, + points_cam2img) from ..builder import FUSION_LAYERS from . import apply_3d_transformation -def point_sample( - img_meta, - img_features, - points, - lidar2img_rt, - img_scale_factor, - img_crop_offset, - img_flip, - img_pad_shape, - img_shape, - aligned=True, - padding_mode='zeros', - align_corners=True, -): +def point_sample(img_meta, + img_features, + points, + proj_mat, + coord_type, + img_scale_factor, + img_crop_offset, + img_flip, + img_pad_shape, + img_shape, + aligned=True, + padding_mode='zeros', + align_corners=True): """Obtain image features using points. Args: img_meta (dict): Meta info. img_features (torch.Tensor): 1 x C x H x W image features. points (torch.Tensor): Nx3 point cloud in LiDAR coordinates. - lidar2img_rt (torch.Tensor): 4x4 transformation matrix. + proj_mat (torch.Tensor): 4x4 transformation matrix. + coord_type (str): 'DEPTH' or 'CAMERA' or 'LIDAR'. img_scale_factor (torch.Tensor): Scale factor with shape of (w_scale, h_scale). img_crop_offset (torch.Tensor): Crop offset used to crop @@ -51,19 +53,11 @@ def point_sample( """ # apply transformation based on info in img_meta - points = apply_3d_transformation(points, 'LIDAR', img_meta, reverse=True) - - # project points from velo coordinate to camera coordinate - num_points = points.shape[0] - pts_4d = torch.cat([points, points.new_ones(size=(num_points, 1))], dim=-1) - pts_2d = pts_4d @ lidar2img_rt.t() - - # cam_points is Tensor of Nx4 whose last column is 1 - # transform camera coordinate to image coordinate + points = apply_3d_transformation( + points, coord_type, img_meta, reverse=True) - pts_2d[:, 2] = torch.clamp(pts_2d[:, 2], min=1e-5) - pts_2d[:, 0] /= pts_2d[:, 2] - pts_2d[:, 1] /= pts_2d[:, 2] + # project points to camera coordinate + pts_2d = points_cam2img(points, proj_mat) # img transformation: scale -> crop -> flip # the image is resized by img_scale_factor @@ -108,6 +102,8 @@ class PointFusion(BaseModule): mid_channels (int): Channels of middle layers out_channels (int): Channels of output fused features img_levels (int, optional): Number of image levels. Defaults to 3. + coord_type (str): 'DEPTH' or 'CAMERA' or 'LIDAR'. + Defaults to 'LIDAR'. conv_cfg (dict, optional): Dict config of conv layers of middle layers. Defaults to None. norm_cfg (dict, optional): Dict config of norm layers of middle @@ -137,6 +133,7 @@ def __init__(self, mid_channels, out_channels, img_levels=3, + coord_type='LIDAR', conv_cfg=None, norm_cfg=None, act_cfg=None, @@ -158,6 +155,7 @@ def __init__(self, assert len(img_channels) == len(img_levels) self.img_levels = img_levels + self.coord_type = coord_type self.act_cfg = act_cfg self.activate_out = activate_out self.fuse_out = fuse_out @@ -289,13 +287,15 @@ def sample_single(self, img_feats, pts, img_meta): img_crop_offset = ( pts.new_tensor(img_meta['img_crop_offset']) if 'img_crop_offset' in img_meta.keys() else 0) + proj_mat = get_proj_mat_by_coord_type(img_meta, self.coord_type) img_pts = point_sample( - img_meta, - img_feats, - pts, - pts.new_tensor(img_meta['lidar2img']), - img_scale_factor, - img_crop_offset, + img_meta=img_meta, + img_features=img_feats, + points=pts, + proj_mat=pts.new_tensor(proj_mat), + coord_type=self.coord_type, + img_scale_factor=img_scale_factor, + img_crop_offset=img_crop_offset, img_flip=img_flip, img_pad_shape=img_meta['input_shape'][:2], img_shape=img_meta['img_shape'][:2], From 08dae0405be79f9bfcb3f073dfe3c6b605245851 Mon Sep 17 00:00:00 2001 From: ChaimZhu Date: Thu, 26 Aug 2021 10:11:23 +0800 Subject: [PATCH 19/51] [Doc] Add nuscenes_det.md Chinese version (#854) * add nus chinese doc * add nuScenes Chinese doc * fix typo * fix typo * fix typo * fix typo * fix typo --- docs_zh-CN/datasets/nuscenes_det.md | 259 ++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) diff --git a/docs_zh-CN/datasets/nuscenes_det.md b/docs_zh-CN/datasets/nuscenes_det.md index d1f6c7026a..be08b211b5 100644 --- a/docs_zh-CN/datasets/nuscenes_det.md +++ b/docs_zh-CN/datasets/nuscenes_det.md @@ -1 +1,260 @@ # 3D 目标检测 NuScenes 数据集 + +本页提供了有关在 MMDetection3D 中使用 nuScenes 数据集的具体教程。 + +## 准备之前 + +您可以在[这里](https://www.nuscenes.org/download)下载 nuScenes 3D 检测数据并解压缩所有 zip 文件。 + +像准备数据集的一般方法一样，建议将数据集根目录链接到 `$MMDETECTION3D/data`。 + +在我们处理之前，文件夹结构应按如下方式组织。 + +``` +mmdetection3d +├── mmdet3d +├── tools +├── configs +├── data +│ ├── nuscenes +│ │ ├── maps +│ │ ├── samples +│ │ ├── sweeps +│ │ ├── v1.0-test +| | ├── v1.0-trainval +``` + +## 数据准备 + +我们通常需要通过特定样式来使用 .pkl 或 .json 文件组织有用的数据信息，例如用于组织图像及其标注的 coco 样式。 +要为 nuScenes 准备这些文件，请运行以下命令： + +```bash +python tools/create_data.py nuscenes --root-path ./data/nuscenes --out-dir ./data/nuscenes --extra-tag nuscenes +``` + +处理后的文件夹结构应该如下 + +``` +mmdetection3d +├── mmdet3d +├── tools +├── configs +├── data +│ ├── nuscenes +│ │ ├── maps +│ │ ├── samples +│ │ ├── sweeps +│ │ ├── v1.0-test +| | ├── v1.0-trainval +│ │ ├── nuscenes_database +│ │ ├── nuscenes_infos_train.pkl +│ │ ├── nuscenes_infos_trainval.pkl +│ │ ├── nuscenes_infos_val.pkl +│ │ ├── nuscenes_infos_test.pkl +│ │ ├── nuscenes_dbinfos_train.pkl +│ │ ├── nuscenes_infos_train_mono3d.coco.json +│ │ ├── nuscenes_infos_trainval_mono3d.coco.json +│ │ ├── nuscenes_infos_val_mono3d.coco.json +│ │ ├── nuscenes_infos_test_mono3d.coco.json +``` + +这里，.pkl 文件一般用于涉及点云的方法，coco 风格的 .json 文件更适合基于图像的方法，例如基于图像的 2D 和 3D 检测。 +接下来，我们将详细说明这些信息文件中记录的细节。 + +- `nuscenes_database/xxxxx.bin`：训练数据集的每个 3D 包围框中包含的点云数据。 +- `nuscenes_infos_train.pkl`：训练数据集信息，每帧信息有两个键值： `metadata` 和 `infos`。 `metadata` 包含数据集本身的基本信息，例如 `{'version': 'v1.0-trainval'}`，而 `infos` 包含详细信息如下： + - info['lidar_path']：激光雷达点云数据的文件路径。 + - info['token']：样本数据标记。 + - info['sweeps']：扫描信息（nuScenes 中的 `sweeps` 是指没有标注的中间帧，而 `samples` 是指那些带有标注的关键帧）。 + - info['sweeps'][i]['data_path']：第 i 次扫描的数据路径。 + - info['sweeps'][i]['type']：扫描数据类型，例如“激光雷达”。 + - info['sweeps'][i]['sample_data_token']：扫描样本数据标记。 + - info['sweeps'][i]['sensor2ego_translation']：从当前传感器（用于收集扫描数据）到自车（包含感知周围环境传感器的车辆，车辆坐标系固连在自车上）的转换（1x3 列表）。 + - info['sweeps'][i]['sensor2ego_rotation']：从当前传感器（用于收集扫描数据）到自车的旋转（四元数格式的 1x4 列表）。 + - info['sweeps'][i]['ego2global_translation']：从自车到全局坐标的转换（1x3 列表）。 + - info['sweeps'][i]['ego2global_rotation']：从自车到全局坐标的旋转（四元数格式的 1x4 列表）。 + - info['sweeps'][i]['timestamp']：扫描数据的时间戳。 + - info['sweeps'][i]['sensor2lidar_translation']：从当前传感器（用于收集扫描数据）到激光雷达的转换（1x3 列表）。 + - info['sweeps'][i]['sensor2lidar_rotation']：从当前传感器（用于收集扫描数据）到激光雷达的旋转（四元数格式的 1x4 列表）。 + - info['cams']：相机校准信息。它包含与每个摄像头对应的六个键值： `'CAM_FRONT'`, `'CAM_FRONT_RIGHT'`, `'CAM_FRONT_LEFT'`, `'CAM_BACK'`, `'CAM_BACK_LEFT'`, `'CAM_BACK_RIGHT'`。 + 每个字典包含每个扫描数据按照上述方式的详细信息（每个信息的关键字与上述相同）。 + - info['lidar2ego_translation']：从激光雷达到自车的转换（1x3 列表）。 + - info['lidar2ego_rotation']：从激光雷达到自车的旋转（四元数格式的 1x4 列表）。 + - info['ego2global_translation']：从自车到全局坐标的转换（1x3 列表）。 + - info['ego2global_rotation']：从自我车辆到全局坐标的旋转（四元数格式的 1x4 列表）。 + - info['timestamp']：样本数据的时间戳。 + - info['gt_boxes']：7 个自由度的 3D 包围框，一个 Nx7 数组。 + - info['gt_names']：3D 包围框的类别，一个 1xN 数组。 + - info['gt_velocity']：3D 包围框的速度（由于不准确，没有垂直测量），一个 Nx2 数组。 + - info['num_lidar_pts']：每个 3D 包围框中包含的激光雷达点数。 + - info['num_radar_pts']：每个 3D 包围框中包含的雷达点数。 + - info['valid_flag']：每个包围框是否有效。一般情况下，我们只将包含至少一个激光雷达或雷达点的 3D 框作为有效框。 +- `nuscenes_infos_train_mono3d.coco.json`：训练数据集 coco 风格的信息。该文件将基于图像的数据组织为三类（键值）：`'categories'`, `'images'`, `'annotations'`。 + - info['categories']：包含所有类别名称的列表。每个元素都遵循字典格式并由两个键值组成：`'id'` 和 `'name'`。 + - info['images']：包含所有图像信息的列表。 + - info['images'][i]['file_name']：第 i 张图像的文件名。 + - info['images'][i]['id']：第 i 张图像的样本数据标记。 + - info['images'][i]['token']：与该帧对应的样本标记。 + - info['images'][i]['cam2ego_rotation']：从相机到自车的旋转（四元数格式的 1x4 列表）。 + - info['images'][i]['cam2ego_translation']：从相机到自车的转换（1x3 列表）。 + - info['images'][i]['ego2global_rotation'']：从自车到全局坐标的旋转（四元数格式的 1x4 列表）。 + - info['images'][i]['ego2global_translation']：从自车到全局坐标的转换（1x3 列表）。 + - info['images'][i]['cam_intrinsic']: 相机内参矩阵（3x3 列表）。 + - info['images'][i]['width']：图片宽度， nuScenes 中默认为 1600。 + - info['images'][i]['height']：图像高度， nuScenes 中默认为 900。 + - info['annotations']: 包含所有标注信息的列表。 + - info['annotations'][i]['file_name']：对应图像的文件名。 + - info['annotations'][i]['image_id']：对应图像的图像 ID （标记）。 + - info['annotations'][i]['area']：2D 包围框的面积。 + - info['annotations'][i]['category_name']：类别名称。 + - info['annotations'][i]['category_id']：类别 id。 + - info['annotations'][i]['bbox']：2D 包围框标注（3D 投影框的外部矩形），1x4 列表跟随 [x1, y1, x2-x1, y2-y1]。x1/y1 是沿图像水平/垂直方向的最小坐标。 + - info['annotations'][i]['iscrowd']：该区域是否拥挤。默认为 0。 + - info['annotations'][i]['bbox_cam3d']：3D 包围框（重力）中心位置（3）、大小（3）、（全局）偏航角（1）、1x7 列表。 + - info['annotations'][i]['velo_cam3d']：3D 包围框的速度（由于不准确，没有垂直测量），一个 Nx2 数组。 + - info['annotations'][i]['center2d']：包含 2.5D 信息的投影 3D 中心：图像上的投影中心位置（2）和深度（1），1x3 列表。 + - info['annotations'][i]['attribute_name']：属性名称。 + - info['annotations'][i]['attribute_id']：属性 ID。 + 我们为属性分类维护了一个属性集合和映射。更多的细节请参考[这里](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/datasets/nuscenes_mono_dataset.py#L53)。 + - info['annotations'][i]['id']：标注 ID。默认为 `i`。 + +这里我们只解释训练信息文件中记录的数据。这同样适用于验证和测试集。 +获取 `nuscenes_infos_xxx.pkl` 和 `nuscenes_infos_xxx_mono3d.coco.json` 的核心函数分别为 [\_fill_trainval_infos](https://github.com/open-mmlab/mmdetection3d/blob/master/tools/data_converter/nuscenes_converter.py#L143) 和 [get_2d_boxes](https://github.com/open-mmlab/mmdetection3d/blob/master/tools/data_converter/nuscenes_converter.py#L397)。更多细节请参考 [nuscenes_converter.py](https://github.com/open-mmlab/mmdetection3d/blob/master/tools/data_converter/nuscenes_converter.py)。 + +## 训练流水线 + +### 基于 LiDAR 的方法 + +nuScenes 上基于 LiDAR 的 3D 检测（包括多模态方法）的典型训练流水线如下。 + +```python +train_pipeline = [ + dict( + type='LoadPointsFromFile', + coord_type='LIDAR', + load_dim=5, + use_dim=5, + file_client_args=file_client_args), + dict( + type='LoadPointsFromMultiSweeps', + sweeps_num=10, + file_client_args=file_client_args), + dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True), + dict( + type='GlobalRotScaleTrans', + rot_range=[-0.3925, 0.3925], + scale_ratio_range=[0.95, 1.05], + translation_std=[0, 0, 0]), + dict(type='RandomFlip3D', flip_ratio_bev_horizontal=0.5), + dict(type='PointsRangeFilter', point_cloud_range=point_cloud_range), + dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range), + dict(type='ObjectNameFilter', classes=class_names), + dict(type='PointShuffle'), + dict(type='DefaultFormatBundle3D', class_names=class_names), + dict(type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d']) +] +``` + +与一般情况相比，nuScenes 有一个特定的 `'LoadPointsFromMultiSweeps'` 流水线来从连续帧加载点云。这是此设置中使用的常见做法。 +更多细节请参考 nuScenes [原始论文](https://arxiv.org/abs/1903.11027)。 +`'LoadPointsFromMultiSweeps'` 中的默认 `use_dim` 是 `[0, 1, 2, 4]`，其中前 3 个维度是指点坐标，最后一个是指时间戳差异。 +由于在拼接来自不同帧的点时使用点云的强度信息会产生噪声，因此默认情况下不使用点云的强度信息。 + +### 基于视觉的方法 + +nuScenes 上基于图像的 3D 检测的典型训练流水线如下。 + +```python +train_pipeline = [ + dict(type='LoadImageFromFileMono3D'), + dict( + type='LoadAnnotations3D', + with_bbox=True, + with_label=True, + with_attr_label=True, + with_bbox_3d=True, + with_label_3d=True, + with_bbox_depth=True), + dict(type='Resize', img_scale=(1600, 900), keep_ratio=True), + dict(type='RandomFlip3D', flip_ratio_bev_horizontal=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle3D', class_names=class_names), + dict( + type='Collect3D', + keys=[ + 'img', 'gt_bboxes', 'gt_labels', 'attr_labels', 'gt_bboxes_3d', + 'gt_labels_3d', 'centers2d', 'depths' + ]), +] +``` + +它遵循 2D 检测的一般流水线，但在一些细节上有所不同： +- 它使用单目流水线加载图像，其中包括额外的必需信息，如相机内参矩阵。 +- 它需要加载 3D 标注。 +- 一些数据增强技术需要调整，例如`RandomFlip3D`。 +目前我们不支持更多的增强方法，因为如何迁移和应用其他技术仍在探索中。 + +## 评估 + +使用 8 个 GPU 以及 nuScenes 指标评估的 PointPillars 的示例如下 + +```shell +bash ./tools/dist_test.sh configs/pointpillars/hv_pointpillars_fpn_sbn-all_4x8_2x_nus-3d.py checkpoints/hv_pointpillars_fpn_sbn-all_4x8_2x_nus-3d_20200620_230405-2fa62f3d.pth 8 --eval bbox +``` + +## 指标 + +NuScenes 提出了一个综合指标，即 nuScenes 检测分数（NDS），以评估不同的方法并设置基准测试。 +它由平均精度（mAP）、平均平移误差（ATE）、平均尺度误差（ASE）、平均方向误差（AOE）、平均速度误差（AVE）和平均属性误差（AAE）组成。 +更多细节请参考其[官方网站](https://www.nuscenes.org/object-detection?externalData=all&mapData=all&modalities=Any)。 + +我们也采用这种方法对 nuScenes 进行评估。打印的评估结果示例如下： + +``` +mAP: 0.3197 +mATE: 0.7595 +mASE: 0.2700 +mAOE: 0.4918 +mAVE: 1.3307 +mAAE: 0.1724 +NDS: 0.3905 +Eval time: 170.8s + +Per-class results: +Object Class AP ATE ASE AOE AVE AAE +car 0.503 0.577 0.152 0.111 2.096 0.136 +truck 0.223 0.857 0.224 0.220 1.389 0.179 +bus 0.294 0.855 0.204 0.190 2.689 0.283 +trailer 0.081 1.094 0.243 0.553 0.742 0.167 +construction_vehicle 0.058 1.017 0.450 1.019 0.137 0.341 +pedestrian 0.392 0.687 0.284 0.694 0.876 0.158 +motorcycle 0.317 0.737 0.265 0.580 2.033 0.104 +bicycle 0.308 0.704 0.299 0.892 0.683 0.010 +traffic_cone 0.555 0.486 0.309 nan nan nan +barrier 0.466 0.581 0.269 0.169 nan nan +``` + +## 测试和提交 + +使用 8 个 GPU 在 kitti 上测试 PointPillars 并生成对排行榜的提交的示例如下 + +```shell +./tools/dist_test.sh configs/pointpillars/hv_pointpillars_fpn_sbn-all_4x8_2x_nus-3d.py work_dirs/hv_pointpillars_secfpn_6x8_160e_kitti-3d-3class/latest.pth 8 --out work_dirs/pp-nus/results_eval.pkl --format-only --eval-options 'jsonfile_prefix=work_dirs/pp-nus/results_eval' +``` + +请注意，在[这里](https://github.com/open-mmlab/mmdetection3d/blob/master/configs/_base_/datasets/nus-3d.py#L132)测试信息应更改为测试集而不是验证集。 + +生成 `work_dirs/pp-nus/results_eval.json` 后，您可以压缩并提交给 nuScenes 基准测试。更多信息请参考 [nuScenes 官方网站](https://www.nuscenes.org/object-detection?externalData=all&mapData=all&modalities=Any)。 + +我们还可以使用我们开发的可视化工具将预测结果可视化。更多细节请参考[可视化文档](https://mmdetection3d.readthedocs.io/en/latest/useful_tools.html#visualization)。 + +## Notes + +### `NuScenesBox` 和我们的 `CameraInstanceBoxes` 之间的转换。 + +总的来说，`NuScenesBox` 和我们的 `CameraInstanceBoxes` 的主要区别主要体现在转向角（yaw）定义上。 `NuScenesBox` 定义了一个四元数或三个欧拉角的旋转，而我们的由于实际情况只定义了一个转向角（yaw），它需要我们在预处理和后处理中手动添加一些额外的旋转，例如[这里](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/datasets/nuscenes_mono_dataset.py#L673)。 + +另外，请注意，角点和位置的定义在 `NuScenesBox` 中是分离的。例如，在单目 3D 检测中，框位置的定义在其相机坐标中（有关汽车设置，请参阅其官方[插图](https://www.nuscenes.org/nuscenes#data-collection)），即与[我们的](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/bbox/structures/cam_box3d.py)一致。相比之下，它的角点是通过[惯例](https://github.com/nutonomy/nuscenes-devkit/blob/02e9200218977193a1058dd7234f935834378319/python-sdk/nuscenes/utils/data_classes.py#L527) 定义的，“x 向前， y 向左， z 向上”。它导致了与我们的 `CameraInstanceBoxes` 不同的维度和旋转定义理念。一个移除相似冲突的例子是 PR [#744](https://github.com/open-mmlab/mmdetection3d/pull/744)。同样的问题也存在于 LiDAR 系统中。为了解决它们，我们通常会在预处理和后处理中添加一些转换，以保证在整个训练和推理过程中框都在我们的坐标系系统里。 + From 93de7c2ba3b33670e9d13c97dec496d0539c49d6 Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Fri, 27 Aug 2021 17:00:24 +0800 Subject: [PATCH 20/51] [Fix] Fix RegNet pretrained weight loading (#889) * Fix regnet pretrained weight loading * Remove unused file --- ...lars_regnet-1.6gf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py | 3 ++- ...-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py | 3 ++- ...lars_regnet-3.2gf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py | 3 ++- ...-3.2gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py | 3 ++- ...lars_regnet-400mf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py | 3 ++- .../hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py | 3 ++- ...hv_pointpillars_regnet-400mf_fpn_sbn-all_2x8_2x_lyft-3d.py | 3 ++- .../hv_pointpillars_regnet-400mf_fpn_sbn-all_4x8_2x_nus-3d.py | 3 ++- ...illars_regnet-400mf_fpn_sbn-all_range100_2x8_2x_lyft-3d.py | 3 ++- .../ssn/hv_ssn_regnet-400mf_secfpn_sbn-all_1x16_2x_lyft-3d.py | 3 ++- .../ssn/hv_ssn_regnet-400mf_secfpn_sbn-all_2x16_2x_nus-3d.py | 3 ++- mmdet3d/models/detectors/mvx_two_stage.py | 4 ++-- 12 files changed, 24 insertions(+), 13 deletions(-) diff --git a/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py b/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py index e6b5ee2ebc..ef740a8acf 100644 --- a/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py +++ b/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py @@ -1,11 +1,12 @@ _base_ = './hv_pointpillars_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py' model = dict( - pretrained=dict(pts='open-mmlab://regnetx_1.6gf'), pts_backbone=dict( _delete_=True, type='NoStemRegNet', arch='regnetx_1.6gf', + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://regnetx_1.6gf'), out_indices=(1, 2, 3), frozen_stages=-1, strides=(1, 2, 2, 2), diff --git a/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py b/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py index f4bfdc9894..d4e48d367f 100644 --- a/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py +++ b/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py @@ -1,11 +1,12 @@ _base_ = './hv_pointpillars_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py' model = dict( - pretrained=dict(pts='open-mmlab://regnetx_1.6gf'), pts_backbone=dict( _delete_=True, type='NoStemRegNet', arch='regnetx_1.6gf', + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://regnetx_1.6gf'), out_indices=(1, 2, 3), frozen_stages=-1, strides=(1, 2, 2, 2), diff --git a/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py b/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py index 354d1d65a2..13bc0d681d 100644 --- a/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py +++ b/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py @@ -1,11 +1,12 @@ _base_ = './hv_pointpillars_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py' model = dict( - pretrained=dict(pts='open-mmlab://regnetx_3.2gf'), pts_backbone=dict( _delete_=True, type='NoStemRegNet', arch='regnetx_3.2gf', + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://regnetx_3.2gf'), out_indices=(1, 2, 3), frozen_stages=-1, strides=(1, 2, 2, 2), diff --git a/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py b/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py index f2931a727a..6fbce89bda 100644 --- a/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py +++ b/configs/free_anchor/hv_pointpillars_regnet-3.2gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py @@ -1,11 +1,12 @@ _base_ = './hv_pointpillars_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py' model = dict( - pretrained=dict(pts='open-mmlab://regnetx_3.2gf'), pts_backbone=dict( _delete_=True, type='NoStemRegNet', arch='regnetx_3.2gf', + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://regnetx_3.2gf'), out_indices=(1, 2, 3), frozen_stages=-1, strides=(1, 2, 2, 2), diff --git a/configs/free_anchor/hv_pointpillars_regnet-400mf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py b/configs/free_anchor/hv_pointpillars_regnet-400mf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py index e5d06bb2cf..2b5f254b1d 100644 --- a/configs/free_anchor/hv_pointpillars_regnet-400mf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py +++ b/configs/free_anchor/hv_pointpillars_regnet-400mf_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py @@ -1,11 +1,12 @@ _base_ = './hv_pointpillars_fpn_sbn-all_free-anchor_4x8_2x_nus-3d.py' model = dict( - pretrained=dict(pts='open-mmlab://regnetx_400mf'), pts_backbone=dict( _delete_=True, type='NoStemRegNet', arch='regnetx_400mf', + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://regnetx_400mf'), out_indices=(1, 2, 3), frozen_stages=-1, strides=(1, 2, 2, 2), diff --git a/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py b/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py index 3c14120330..0574be5761 100644 --- a/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py +++ b/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py @@ -7,11 +7,12 @@ # model settings model = dict( type='MVXFasterRCNN', - pretrained=dict(pts='open-mmlab://regnetx_1.6gf'), pts_backbone=dict( _delete_=True, type='NoStemRegNet', arch='regnetx_1.6gf', + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://regnetx_1.6gf'), out_indices=(1, 2, 3), frozen_stages=-1, strides=(1, 2, 2, 2), diff --git a/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_2x8_2x_lyft-3d.py b/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_2x8_2x_lyft-3d.py index 8462733b53..1f391a328a 100644 --- a/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_2x8_2x_lyft-3d.py +++ b/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_2x8_2x_lyft-3d.py @@ -7,11 +7,12 @@ # model settings model = dict( type='MVXFasterRCNN', - pretrained=dict(pts='open-mmlab://regnetx_400mf'), pts_backbone=dict( _delete_=True, type='NoStemRegNet', arch=dict(w0=24, wa=24.48, wm=2.54, group_w=16, depth=22, bot_mul=1.0), + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://regnetx_400mf'), out_indices=(1, 2, 3), frozen_stages=-1, strides=(1, 2, 2, 2), diff --git a/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_4x8_2x_nus-3d.py b/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_4x8_2x_nus-3d.py index 9dc4124a76..884729cc90 100644 --- a/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_4x8_2x_nus-3d.py +++ b/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_4x8_2x_nus-3d.py @@ -7,11 +7,12 @@ # model settings model = dict( type='MVXFasterRCNN', - pretrained=dict(pts='open-mmlab://regnetx_400mf'), pts_backbone=dict( _delete_=True, type='NoStemRegNet', arch=dict(w0=24, wa=24.48, wm=2.54, group_w=16, depth=22, bot_mul=1.0), + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://regnetx_400mf'), out_indices=(1, 2, 3), frozen_stages=-1, strides=(1, 2, 2, 2), diff --git a/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_range100_2x8_2x_lyft-3d.py b/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_range100_2x8_2x_lyft-3d.py index 2250385787..fef308dfc9 100644 --- a/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_range100_2x8_2x_lyft-3d.py +++ b/configs/regnet/hv_pointpillars_regnet-400mf_fpn_sbn-all_range100_2x8_2x_lyft-3d.py @@ -7,11 +7,12 @@ # model settings model = dict( type='MVXFasterRCNN', - pretrained=dict(pts='open-mmlab://regnetx_400mf'), pts_backbone=dict( _delete_=True, type='NoStemRegNet', arch=dict(w0=24, wa=24.48, wm=2.54, group_w=16, depth=22, bot_mul=1.0), + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://regnetx_400mf'), out_indices=(1, 2, 3), frozen_stages=-1, strides=(1, 2, 2, 2), diff --git a/configs/ssn/hv_ssn_regnet-400mf_secfpn_sbn-all_1x16_2x_lyft-3d.py b/configs/ssn/hv_ssn_regnet-400mf_secfpn_sbn-all_1x16_2x_lyft-3d.py index b28a5d5883..1103bcf12a 100644 --- a/configs/ssn/hv_ssn_regnet-400mf_secfpn_sbn-all_1x16_2x_lyft-3d.py +++ b/configs/ssn/hv_ssn_regnet-400mf_secfpn_sbn-all_1x16_2x_lyft-3d.py @@ -2,11 +2,12 @@ # model settings model = dict( type='MVXFasterRCNN', - pretrained=dict(pts='open-mmlab://regnetx_400mf'), pts_backbone=dict( _delete_=True, type='NoStemRegNet', arch=dict(w0=24, wa=24.48, wm=2.54, group_w=16, depth=22, bot_mul=1.0), + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://regnetx_400mf'), out_indices=(1, 2, 3), frozen_stages=-1, strides=(1, 2, 2, 2), diff --git a/configs/ssn/hv_ssn_regnet-400mf_secfpn_sbn-all_2x16_2x_nus-3d.py b/configs/ssn/hv_ssn_regnet-400mf_secfpn_sbn-all_2x16_2x_nus-3d.py index 4475168665..fb9ef31603 100644 --- a/configs/ssn/hv_ssn_regnet-400mf_secfpn_sbn-all_2x16_2x_nus-3d.py +++ b/configs/ssn/hv_ssn_regnet-400mf_secfpn_sbn-all_2x16_2x_nus-3d.py @@ -2,11 +2,12 @@ # model settings model = dict( type='MVXFasterRCNN', - pretrained=dict(pts='open-mmlab://regnetx_400mf'), pts_backbone=dict( _delete_=True, type='NoStemRegNet', arch=dict(w0=24, wa=24.48, wm=2.54, group_w=16, depth=22, bot_mul=1.0), + init_cfg=dict( + type='Pretrained', checkpoint='open-mmlab://regnetx_400mf'), out_indices=(1, 2, 3), frozen_stages=-1, strides=(1, 2, 2, 2), diff --git a/mmdet3d/models/detectors/mvx_two_stage.py b/mmdet3d/models/detectors/mvx_two_stage.py index 0ca8b9686b..63d1d33dbe 100644 --- a/mmdet3d/models/detectors/mvx_two_stage.py +++ b/mmdet3d/models/detectors/mvx_two_stage.py @@ -96,9 +96,9 @@ def __init__(self, type='Pretrained', checkpoint=img_pretrained) if self.with_pts_backbone: - if img_pretrained is not None: + if pts_pretrained is not None: warnings.warn('DeprecationWarning: pretrained is a deprecated ' - 'key, please consider using init_cfg.') + 'key, please consider using init_cfg') self.pts_backbone.init_cfg = dict( type='Pretrained', checkpoint=pts_pretrained) From 0eb7e718e675f5ca33951b99aba278805d526817 Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Mon, 30 Aug 2021 10:29:20 +0800 Subject: [PATCH 21/51] Fix centerpoint tta (#892) --- ..._0075voxel_second_secfpn_dcn_4x8_cyclic_tta_20e_nus.py | 2 ++ mmdet3d/models/detectors/centerpoint.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/configs/centerpoint/centerpoint_0075voxel_second_secfpn_dcn_4x8_cyclic_tta_20e_nus.py b/configs/centerpoint/centerpoint_0075voxel_second_secfpn_dcn_4x8_cyclic_tta_20e_nus.py index 770a11c68c..cdbdf0600f 100644 --- a/configs/centerpoint/centerpoint_0075voxel_second_secfpn_dcn_4x8_cyclic_tta_20e_nus.py +++ b/configs/centerpoint/centerpoint_0075voxel_second_secfpn_dcn_4x8_cyclic_tta_20e_nus.py @@ -1,5 +1,7 @@ _base_ = './centerpoint_0075voxel_second_secfpn_dcn_4x8_cyclic_20e_nus.py' +model = dict(test_cfg=dict(pts=dict(use_rotate_nms=True, max_num=500))) + point_cloud_range = [-54, -54, -5.0, 54, 54, 3.0] file_client_args = dict(backend='disk') class_names = [ diff --git a/mmdet3d/models/detectors/centerpoint.py b/mmdet3d/models/detectors/centerpoint.py index d6e971d2b9..ef34810d19 100644 --- a/mmdet3d/models/detectors/centerpoint.py +++ b/mmdet3d/models/detectors/centerpoint.py @@ -122,8 +122,8 @@ def aug_test_pts(self, feats, img_metas, rescale=False): task_id][0][key][:, 1, ...] elif key == 'rot': outs[task_id][0][ - key][:, 1, - ...] = -outs[task_id][0][key][:, 1, ...] + key][:, 0, + ...] = -outs[task_id][0][key][:, 0, ...] elif key == 'vel': outs[task_id][0][ key][:, 1, @@ -136,8 +136,8 @@ def aug_test_pts(self, feats, img_metas, rescale=False): task_id][0][key][:, 0, ...] elif key == 'rot': outs[task_id][0][ - key][:, 0, - ...] = -outs[task_id][0][key][:, 0, ...] + key][:, 1, + ...] = -outs[task_id][0][key][:, 1, ...] elif key == 'vel': outs[task_id][0][ key][:, 0, From 00c037a87064d2ab126f28189855d37698262428 Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Mon, 30 Aug 2021 10:30:55 +0800 Subject: [PATCH 22/51] [Enhance] Add benchmark regression script (#808) --- .dev_scripts/gen_benchmark_script.py | 193 ++++++++++++++++++ .dev_scripts/test_benchmark.sh | 128 ++++++++++++ .dev_scripts/train_benchmark.sh | 128 ++++++++++++ ...-car.py => imvoxelnet_4x8_kitti-3d-car.py} | 0 4 files changed, 449 insertions(+) create mode 100644 .dev_scripts/gen_benchmark_script.py create mode 100644 .dev_scripts/test_benchmark.sh create mode 100644 .dev_scripts/train_benchmark.sh rename configs/imvoxelnet/{imvoxelnet_kitti-3d-car.py => imvoxelnet_4x8_kitti-3d-car.py} (100%) diff --git a/.dev_scripts/gen_benchmark_script.py b/.dev_scripts/gen_benchmark_script.py new file mode 100644 index 0000000000..3ae1128029 --- /dev/null +++ b/.dev_scripts/gen_benchmark_script.py @@ -0,0 +1,193 @@ +import argparse +import re +from os import path as osp + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Generate benchmark training/testing scripts') + parser.add_argument( + '--input_file', + required=False, + type=str, + help='Input file containing the paths ' + 'of configs to be trained/tested.') + parser.add_argument( + '--output_file', + required=True, + type=str, + help='Output file containing the ' + 'commands to train/test selected models.') + parser.add_argument( + '--gpus_per_node', + type=int, + default=8, + help='GPUs per node config for slurm, ' + 'should be set according to your slurm environment') + parser.add_argument( + '--cpus_per_task', + type=int, + default=5, + help='CPUs per task config for slurm, ' + 'should be set according to your slurm environment') + parser.add_argument( + '--gpus', + type=int, + default=8, + help='Totally used num of GPUs config for slurm (in testing), ' + 'should be set according to your slurm environment') + parser.add_argument( + '--mode', type=str, default='train', help='Train or test') + parser.add_argument( + '--long_work_dir', + action='store_true', + help='Whether use full relative path of config as work dir') + parser.add_argument( + '--max_keep_ckpts', + type=int, + default=1, + help='The max number of checkpoints saved in training') + parser.add_argument( + '--full_log', + action='store_true', + help='Whether save full log in a file') + + args = parser.parse_args() + return args + + +args = parse_args() +assert args.mode in ['train', 'test'], 'Currently we only support ' \ + 'automatically generating training or testing scripts.' + +config_paths = [] + +if args.input_file is not None: + with open(args.input_file, 'r') as fi: + config_paths = fi.read().strip().split('\n') +else: + while True: + print('Please type a config path and ' + 'press enter (press enter directly to exit):') + config_path = input() + if config_path != '': + config_paths.append(config_path) + else: + break + +script = '''PARTITION=$1 +CHECKPOINT_DIR=$2 + +''' + +if args.mode == 'train': + for i, config_path in enumerate(config_paths): + root_dir = osp.dirname(osp.dirname(osp.abspath(__file__))) + if not osp.exists(osp.join(root_dir, config_path)): + print(f'Invalid config path (does not exist):\n{config_path}') + continue + + config_name = config_path.split('/')[-1][:-3] + match_obj = re.match(r'^.*_[0-9]+x([0-9]+)_.*$', config_name) + if match_obj is None: + print(f'Invalid config path (no GPU num in ' + f'config name):\n{config_path}') + continue + + gpu_num = int(match_obj.group(1)) + work_dir_name = config_path if args.long_work_dir else config_name + + script += f"echo '{config_path}' &\n" + if args.full_log: + script += f'mkdir -p $CHECKPOINT_DIR/{work_dir_name}\n' + + # training commands + script += f'GPUS={gpu_num} GPUS_PER_NODE={args.gpus_per_node} ' \ + f'CPUS_PER_TASK={args.cpus_per_task} ' \ + f'./tools/slurm_train.sh $PARTITION {config_name} ' \ + f'{config_path} \\\n' + script += f'$CHECKPOINT_DIR/{work_dir_name} --cfg-options ' \ + f'checkpoint_config.max_keep_ckpts=' \ + f'{args.max_keep_ckpts} \\\n' \ + + # if output full log, redirect stdout and stderr to + # another log file in work dir + if args.full_log: + script += f'2>&1|tee $CHECKPOINT_DIR/{work_dir_name}' \ + f'/FULL_LOG.txt &\n' + else: + script += '>/dev/null &\n' + + if i != len(config_paths) - 1: + script += '\n' + + print(f'Successfully generated script for {config_name}') + + with open(args.output_file, 'w') as fo: + fo.write(script) + +elif args.mode == 'test': + for i, config_path in enumerate(config_paths): + root_dir = osp.dirname(osp.dirname(osp.abspath(__file__))) + if not osp.exists(osp.join(root_dir, config_path)): + print(f'Invalid config path (does not exist):\n{config_path}') + continue + + config_name = config_path.split('/')[-1][:-3] + + tasks = { + 'scannet_seg', 'scannet', 's3dis_seg', 'sunrgbd', 'kitti', 'nus', + 'lyft', 'waymo' + } + eval_option = None + for task in tasks: + if task in config_name: + eval_option = task + break + if eval_option is None: + print(f'Invalid config path (invalid task):\n{config_path}') + continue + + work_dir_name = config_path if args.long_work_dir else config_name + + script += f"echo '{config_path}' &\n" + if args.full_log: + script += f'mkdir -p $CHECKPOINT_DIR/{work_dir_name}\n' + + # training commands + script += f'GPUS={args.gpus} GPUS_PER_NODE={args.gpus_per_node} ' \ + f'CPUS_PER_TASK={args.cpus_per_task} ' \ + f'./tools/slurm_test.sh $PARTITION {config_name} ' \ + f'{config_path} \\\n' + script += f'$CHECKPOINT_DIR/{work_dir_name}/latest.pth ' \ + + if eval_option in ['scannet_seg', 's3dis_seg']: + script += '--eval mIoU \\\n' + elif eval_option in ['scannet', 'sunrgbd', 'kitti', 'nus']: + script += '--eval map \\\n' + elif eval_option in ['lyft']: + script += f'--format-only --eval-options jsonfile_prefix=' \ + f'$CHECKPOINT_DIR/{work_dir_name}/results_challenge ' \ + f'csv_savepath=$CHECKPOINT_DIR/{work_dir_name}/' \ + f'results_challenge.csv \\\n' + elif eval_option in ['waymo']: + script += f'--eval waymo --eval-options pklfile_prefix=' \ + f'$CHECKPOINT_DIR/{work_dir_name}/kitti_results ' \ + f'submission_prefix=$CHECKPOINT_DIR/{work_dir_name}/' \ + f'kitti_results \\\n' + + # if output full log, redirect stdout and stderr to + # another log file in work dir + if args.full_log: + script += f'2>&1|tee $CHECKPOINT_DIR/{work_dir_name}' \ + f'/FULL_LOG.txt &\n' + else: + script += '>/dev/null &\n' + + if i != len(config_paths) - 1: + script += '\n' + + print(f'Successfully generated script for {config_name}') + + with open(args.output_file, 'w') as fo: + fo.write(script) diff --git a/.dev_scripts/test_benchmark.sh b/.dev_scripts/test_benchmark.sh new file mode 100644 index 0000000000..f020d7ff9e --- /dev/null +++ b/.dev_scripts/test_benchmark.sh @@ -0,0 +1,128 @@ +PARTITION=$1 +CHECKPOINT_DIR=$2 + +echo 'configs/3dssd/3dssd_4x4_kitti-3d-car.py' & +mkdir -p $CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION 3dssd_4x4_kitti-3d-car configs/3dssd/3dssd_4x4_kitti-3d-car.py \ +$CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py/FULL_LOG.txt & + +echo 'configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py' & +mkdir -p $CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py \ +$CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py/FULL_LOG.txt & + +echo 'configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py \ +$CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py/FULL_LOG.txt & + +echo 'configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py \ +$CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py/FULL_LOG.txt & + +echo 'configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py' & +mkdir -p $CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION groupfree3d_8x4_scannet-3d-18class-L6-O256 configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py \ +$CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py/FULL_LOG.txt & + +echo 'configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py' & +mkdir -p $CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION h3dnet_3x8_scannet-3d-18class configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py \ +$CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py/FULL_LOG.txt & + +echo 'configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py \ +$CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py/FULL_LOG.txt & + +echo 'configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION imvotenet_stage2_16x8_sunrgbd-3d-10class configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py \ +$CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py/FULL_LOG.txt & + +echo 'configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION imvoxelnet_4x8_kitti-3d-car configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py \ +$CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py/FULL_LOG.txt & + +echo 'configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py \ +$CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py/latest.pth --eval mIoU \ +2>&1|tee $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py/FULL_LOG.txt & + +echo 'configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py \ +$CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py/FULL_LOG.txt & + +echo 'configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py \ +$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/latest.pth --format-only --eval-options jsonfile_prefix=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/results_challenge csv_savepath=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/results_challenge.csv \ +2>&1|tee $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/FULL_LOG.txt & + +echo 'configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py \ +$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/latest.pth --eval waymo --eval-options pklfile_prefix=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/kitti_results submission_prefix=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/kitti_results \ +2>&1|tee $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/FULL_LOG.txt & + +echo 'configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py \ +$CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py/FULL_LOG.txt & + +echo 'configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_second_secfpn_6x8_80e_kitti-3d-3class configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py \ +$CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/latest.pth --format-only --eval-options jsonfile_prefix=$CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/results_challenge csv_savepath=$CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/results_challenge.csv \ +2>&1|tee $CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/FULL_LOG.txt & + +echo 'configs/votenet/votenet_8x8_scannet-3d-18class.py' & +mkdir -p $CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION votenet_8x8_scannet-3d-18class configs/votenet/votenet_8x8_scannet-3d-18class.py \ +$CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py/FULL_LOG.txt & diff --git a/.dev_scripts/train_benchmark.sh b/.dev_scripts/train_benchmark.sh new file mode 100644 index 0000000000..7655ab174e --- /dev/null +++ b/.dev_scripts/train_benchmark.sh @@ -0,0 +1,128 @@ +PARTITION=$1 +CHECKPOINT_DIR=$2 + +echo 'configs/3dssd/3dssd_4x4_kitti-3d-car.py' & +mkdir -p $CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py +GPUS=4 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION 3dssd_4x4_kitti-3d-car configs/3dssd/3dssd_4x4_kitti-3d-car.py \ +$CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py/FULL_LOG.txt & + +echo 'configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py' & +mkdir -p $CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py \ +$CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py/FULL_LOG.txt & + +echo 'configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py \ +$CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py/FULL_LOG.txt & + +echo 'configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py \ +$CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py/FULL_LOG.txt & + +echo 'configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py' & +mkdir -p $CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py +GPUS=4 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION groupfree3d_8x4_scannet-3d-18class-L6-O256 configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py \ +$CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py/FULL_LOG.txt & + +echo 'configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py' & +mkdir -p $CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION h3dnet_3x8_scannet-3d-18class configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py \ +$CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py/FULL_LOG.txt & + +echo 'configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py +GPUS=4 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py \ +$CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py/FULL_LOG.txt & + +echo 'configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION imvotenet_stage2_16x8_sunrgbd-3d-10class configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py \ +$CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py/FULL_LOG.txt & + +echo 'configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION imvoxelnet_4x8_kitti-3d-car configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py \ +$CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py/FULL_LOG.txt & + +echo 'configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py +GPUS=2 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py \ +$CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py/FULL_LOG.txt & + +echo 'configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py +GPUS=2 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py \ +$CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py/FULL_LOG.txt & + +echo 'configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py \ +$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/FULL_LOG.txt & + +echo 'configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py +GPUS=16 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py \ +$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/FULL_LOG.txt & + +echo 'configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py \ +$CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py/FULL_LOG.txt & + +echo 'configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_second_secfpn_6x8_80e_kitti-3d-3class configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py +GPUS=16 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py \ +$CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/FULL_LOG.txt & + +echo 'configs/votenet/votenet_8x8_scannet-3d-18class.py' & +mkdir -p $CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION votenet_8x8_scannet-3d-18class configs/votenet/votenet_8x8_scannet-3d-18class.py \ +$CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py/FULL_LOG.txt & diff --git a/configs/imvoxelnet/imvoxelnet_kitti-3d-car.py b/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py similarity index 100% rename from configs/imvoxelnet/imvoxelnet_kitti-3d-car.py rename to configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py From 1e6cdea27b471d6e1dc67511303f312de77b7c96 Mon Sep 17 00:00:00 2001 From: THU17cyz Date: Wed, 1 Sep 2021 14:20:59 +0800 Subject: [PATCH 23/51] Initial commit --- docs/tutorials/coord_sys_tutorial.md | 242 +++++++++++++++++++++ docs/tutorials/index.rst | 1 + docs_zh-CN/tutorials/coord_sys_tutorial.md | 1 + docs_zh-CN/tutorials/index.rst | 1 + 4 files changed, 245 insertions(+) create mode 100644 docs/tutorials/coord_sys_tutorial.md create mode 100644 docs_zh-CN/tutorials/coord_sys_tutorial.md diff --git a/docs/tutorials/coord_sys_tutorial.md b/docs/tutorials/coord_sys_tutorial.md new file mode 100644 index 0000000000..a7e7474736 --- /dev/null +++ b/docs/tutorials/coord_sys_tutorial.md @@ -0,0 +1,242 @@ +# Coordinate System Tutorial + +## Overview + +MMDetection3D uses three different coordinate systems. The existence of different coordinate systems in the society of 3D object detection is necessary, because for various 3D data collection devices, such as LiDAR, depth camera, etc., the coordinate systems are not consistent, and different 3D datasets also follow different data formats. Early works, such as SECOND, VoteNet, convert the raw data to another format, forming conventions that some later works also follow, making the conversion between coordinate systems even more complicated. + +Despite the variety of datasets and equipments, by summarizing the line of works on 3D object detection we can roughly categorize coordinate systems into three: + +- Camera coordinate system -- the coordinate system of most cameras, in which the positive direction of the y-axis points to the ground, the positive direction of the x-axis points to the right, and the positive direction of the z-axis points to the front. + ``` + up z front + | ^ + | / + | / + | / + |/ + left ------ 0 ------> x right + | + | + | + | + v + y down + ``` +- LiDAR coordinate system -- the coordinate system of many LiDARs, in which the negative direction of the z-axis points to the ground, the positive direction of the x-axis points to the front, and the positive direction of the y-axis points to the left. + ``` + z up x front + ^ ^ + | / + | / + | / + |/ + y left <------ 0 ------ right + ``` +- Depth coordinate system, the coordinate system used by VoteNet, H3DNet, etc., in which the negative direction of the z-axis points to the ground, the positive direction of the x-axis points to the right, and the positive direction of the y-axis points to the front. + ``` + z up y front + ^ ^ + | / + | / + | / + |/ + left ------ 0 ------> x right + ``` + +The definition of coordinate systems in this tutorial is actually **more than just defining the three axes**. For a box in the form of , our coordinate systems also define how to interpret the box dimensions and the yaw angle . + +The illustration of the three coordinate systems is shown below: + +![coord_sys_all](https://github.com/open-mmlab/mmdetection3d/blob/master/resources/coord_sys_all.png) + +The three figures above are the 3D coordinate systems while the three figures below are the bird's eye view. + +We will stick to the three coordinate systems defined in this tutorial in the future. + +## Definition of the yaw angle + +Please refer to [wikipedia](https://en.wikipedia.org/wiki/Euler_angles#Tait%E2%80%93Bryan_angles) for the standard definition of the yaw angle. In object detection, we choose an axis as the gravity axis, and a reference direction on the plane perpendicular to the gravity axis, then the reference direction has a yaw angle of 0, and other directions on have non-zero yaw angles depending on its angle with the reference direction. + +Currently, for all supported datasets, annotations do not include pitch angle and roll angle, which means we need only consider the yaw angle when predicting boxes and calculating overlap between boxes. + +In MMDetection3D, all three coordinate systems are right-handed coordinate systems, which means the ascending direction of the yaw angle is counter-clockwise if viewed from the negative direction of the gravity axis (the axis is pointing at one's eyes). + +The figure below shows that, in this right-handed coordinate system, if we set the positive direction of the x-axis as a reference direction, then the positive direction of the y-axis has a yaw angle of . + +``` + z up y front (yaw=0.5*pi) + ^ ^ + | / + | / + | / + |/ +left (yaw=pi) ------ 0 ------> x right (yaw=0) +``` + +For a box, the value of its yaw angle equals its direction minus a reference direction. In all three coordinate systems in MMDetection3D, the reference direction is always the positive direction of the x-axis, while the direction of a box is defined to be parallel with the x-axis if its yaw angle is 0. The definition of the yaw angle of a box is illustrated in the figure below. + +``` +y front + ^ box direction (yaw=0.5*pi) + /|\ ^ + | /|\ + | ____|____ + | | | | + | | | | +__|____|____|____|______\ x right + | | | | / + | | | | + | |____|____| + | +``` + +## Definition of the box dimensions + +The definition of the box dimensions cannot be disentangled with the definition of the yaw angle. In the previous section, we said that the direction of a box is defined to be parallel with the x-axis if its yaw angle is 0. Then naturally, the dimension of a box which corresponds to the x-axis should be . However, this is not always the case in some datasets (we will address that later). + +The following figures show the meaning of the correspondence between the x-axis and , and between the y-axis and . + +``` +y front + ^ box direction (yaw=0.5*pi) + /|\ ^ + | /|\ + | ____|____ + | | | | + | | | | dx +__|____|____|____|______\ x right + | | | | / + | | | | + | |____|____| + | dy +``` + +Note that the box direction is always parallel with the edge . + +``` +y front + ^ _________ + /|\ | | | + | | | | + | | | | dy + | |____|____|____\ box direction (yaw=0) + | | | | / +__|____|____|____|_________\ x right + | | | | / + | |____|____| + | dx + | +``` + +## Relation with raw coordinate systems of supported datasets + +### KITTI + +The raw annotation of KITTI is under Camera coordinate system, see [get_label_anno](https://github.com/open-mmlab/mmdetection3d/blob/master/tools/data_converter/kitti_data_utils.py). In MMDetection3D, to train models on KITTI, the data is first converted from camera to LiDAR, see [get_ann_info](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/datasets/kitti_dataset.py). + +In SECOND, the LiDAR coordinate system for a box is defined as follows (a bird's eye view): + +![lidar](https://raw.githubusercontent.com/traveller59/second.pytorch/master/images/kittibox.png) + + + +For each box, the dimensions are , and the reference direction for the yaw angle is the positive direction of the y axis. For more details, refer to the [repo](https://github.com/traveller59/second.pytorch#concepts). + +Our LiDAR coordinate system has two changes: + +- The yaw angle is defined to be right-handed instead of left-handed for consistency; +- The box dimensions are instead of , since corresponds to and corresponds to in KITTI. + +### Waymo + +We use the KITTI-format data of Waymo dataset. Therefore, KITTI and Waymo also share the same coordinate system in our implementation. + +### NuScenes + +NuScenes provides a toolkit for evaluation, in which each box is wrapped into a `Box` instance. The coordinate system of `Box` is different from our LiDAR coordinate system in that the first two elements of the box dimension correspond to , or , respectively, instead of the reverse. For more details, please refer to the NuScenes [tutorial](https://github.com/open-mmlab/mmdetection3d/blob/master/docs/datasets/nuscenes_det.md#notes). + +Readers may refer to the [NuScenes development kit](https://github.com/nutonomy/nuscenes-devkit/tree/master/python-sdk/nuscenes/eval/detection) for the definition of a [NuScenes box](https://github.com/nutonomy/nuscenes-devkit/blob/2c6a752319f23910d5f55cc995abc547a9e54142/python-sdk/nuscenes/utils/data_classes.py#L457) and implementation of [NuScenes evaluation](https://github.com/nutonomy/nuscenes-devkit/blob/master/python-sdk/nuscenes/eval/detection/evaluate.py). + +### Lyft + +Lyft shares the same data format with NuScenes as far as coordinate system is involved. + +Please refer to the [official website](https://www.kaggle.com/c/3d-object-detection-for-autonomous-vehicles/data) for more information. + +### ScanNet + +The raw data of ScanNet is not point cloud but mesh. The sampled point cloud data is under our depth coordinate system. For ScanNet detection task, the box annotations are axis-aligned, and the yaw angle is always zero. Therefore the direction of the yaw angle in our depth coordinate system makes no difference regarding ScanNet. + +### SUN RGB-D + +The raw data of SUN RGB-D is not point cloud but RGB-D image. By back projection, we obtain the corresponding point cloud for each image, which is under our Depth coordinate system. However, the annotation is not under our system and thus needs conversion. + +For the conversion from raw annotation to annotation under our Depth coordinate system, please refer to [sunrgbd_data_utils.py](https://github.com/open-mmlab/mmdetection3d/blob/master/tools/data_converter/sunrgbd_data_utils.py). + +### S3DIS + +S3DIS shares the same coordinate system as ScanNet in our implementation. However, S3DIS is a segmentation-task-only dataset, and thus no annotation is coordinate system sensitive. + +## Examples + +### Box conversion (between different coordinate systems) + +Take the conversion between our Camera coordinate system and LiDAR coordinate system as an example: + +First, for points and box centers, the coordinates before and after the conversion satisfy the following relationship: + +- +- +- + +Then, the box dimensions before and after the conversion satisfy the following relationship: + +- +- +- + +Finally, the yaw angle should also be converted: + +- + +See the code [here](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/bbox/structures/box_3d_mode.py) for more details. + +### Bird's Eye View + +The BEV of a camera coordinate system box is if the 3D box is . The inversion of the sign of the yaw angle is because the positive direction of the gravity axis of the Camera coordinate system points to the ground. + +See the code [here](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/bbox/structures/cam_box3d.py) for more details. + +### Rotation of boxes + +We set the rotation of all kinds of boxes to be counter-clockwise about the gravity axis. Therefore, to rotate a 3D box we first calculate the new box center, and then we add the rotation angle to the yaw angle. + +See the code [here](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/bbox/structures/cam_box3d.py) for more details. + +## Common FAQ + +#### Q1: Are the box related ops universal to all coordinate system types? + +No. For example, the ops under [this folder](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/ops/roiaware_pool3d) are applicable to boxes under Depth or LiDAR coordinate system only. The evaluation functions for KITTI dataset [here](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/evaluation/kitti_utils) are only applicable to boxes under Camera coordinate system since the rotation is clockwise if viewed from above. + +For each box related op, we have marked the type of boxes to which we can apply the op. + +#### Q2: In every coordinate system, do the three axes point exactly to the right, the front, and the ground, respectively? + +No. For example, in KITTI, we need a calibration matrix when converting from Camera coordinate system to LiDAR coordinate system. + +#### Q3: How does a phase difference of in the yaw angle of a box affect evaluation? + +For IoU calculation, a phase difference of in the yaw angle will result in the same box, thus not affecting evaluation. + +For angle prediction evaluation such as the NDS metric in NuScenes and the AOS metric in KITTI, the angle of predicted boxes will be first standardized, so the phase difference will not change the result. + +#### Q4: How does a phase difference of in the yaw angle of a box affect evaluation? + +For IoU calculation, a phase difference of in the yaw angle will result in the same box, thus not affecting evaluation. + +However, for angle prediction evaluation, this will result in the exact opposite direction. + +Just think about a car. The yaw angle is the angle between the direction of the car front and the positive direction of the x-axis. If we add to this angle, the car front will become the car rear. + +For categories such as barrier, the front and the rear have no difference, therefore a phase difference of will not affect the angle prediction score. diff --git a/docs/tutorials/index.rst b/docs/tutorials/index.rst index dec6ee8133..5ecd9f6e48 100644 --- a/docs/tutorials/index.rst +++ b/docs/tutorials/index.rst @@ -6,3 +6,4 @@ data_pipeline.md customize_models.md customize_runtime.md + coord_sys_tutorial.md diff --git a/docs_zh-CN/tutorials/coord_sys_tutorial.md b/docs_zh-CN/tutorials/coord_sys_tutorial.md new file mode 100644 index 0000000000..96c163a67d --- /dev/null +++ b/docs_zh-CN/tutorials/coord_sys_tutorial.md @@ -0,0 +1 @@ +# 坐标系教程 diff --git a/docs_zh-CN/tutorials/index.rst b/docs_zh-CN/tutorials/index.rst index dec6ee8133..5ecd9f6e48 100644 --- a/docs_zh-CN/tutorials/index.rst +++ b/docs_zh-CN/tutorials/index.rst @@ -6,3 +6,4 @@ data_pipeline.md customize_models.md customize_runtime.md + coord_sys_tutorial.md From f095eb6b4e2f1a6bf198d42d5c66afe53c5fefff Mon Sep 17 00:00:00 2001 From: dingchang Date: Fri, 3 Sep 2021 13:01:42 +0800 Subject: [PATCH 24/51] [Feature] Support DGCNN (v1.0.0.dev0) (#896) * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * fix typo * fix typo * fix typo * del gf&fa registry (wo reuse pointnet module) * fix typo * add benchmark and add copyright header (for DGCNN only) * fix typo * fix typo * fix typo * fix typo * fix typo * support dgcnn --- .dev_scripts/gather_models.py | 1 + README.md | 41 ++-- README_zh-CN.md | 41 ++-- configs/_base_/models/dgcnn.py | 28 +++ configs/_base_/schedules/seg_cosine_100e.py | 8 + configs/dgcnn/README.md | 43 ++++ ...n_32x4_cosine_100e_s3dis_seg-3d-13class.py | 24 ++ configs/dgcnn/metafile.yml | 24 ++ docs/model_zoo.md | 4 + docs_zh-CN/model_zoo.md | 8 + mmdet3d/models/backbones/__init__.py | 4 +- mmdet3d/models/backbones/dgcnn.py | 98 ++++++++ mmdet3d/models/decode_heads/__init__.py | 3 +- mmdet3d/models/decode_heads/dgcnn_head.py | 67 ++++++ mmdet3d/ops/__init__.py | 10 +- mmdet3d/ops/dgcnn_modules/__init__.py | 6 + mmdet3d/ops/dgcnn_modules/dgcnn_fa_module.py | 68 ++++++ mmdet3d/ops/dgcnn_modules/dgcnn_fp_module.py | 59 +++++ mmdet3d/ops/dgcnn_modules/dgcnn_gf_module.py | 222 ++++++++++++++++++ tests/test_models/test_backbones.py | 33 +++ .../test_common_modules/test_dgcnn_modules.py | 92 ++++++++ .../test_heads/test_dgcnn_decode_head.py | 68 ++++++ tests/test_models/test_segmentors.py | 45 ++++ 23 files changed, 953 insertions(+), 44 deletions(-) create mode 100644 configs/_base_/models/dgcnn.py create mode 100644 configs/_base_/schedules/seg_cosine_100e.py create mode 100644 configs/dgcnn/README.md create mode 100644 configs/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py create mode 100644 configs/dgcnn/metafile.yml create mode 100644 mmdet3d/models/backbones/dgcnn.py create mode 100644 mmdet3d/models/decode_heads/dgcnn_head.py create mode 100644 mmdet3d/ops/dgcnn_modules/__init__.py create mode 100644 mmdet3d/ops/dgcnn_modules/dgcnn_fa_module.py create mode 100644 mmdet3d/ops/dgcnn_modules/dgcnn_fp_module.py create mode 100644 mmdet3d/ops/dgcnn_modules/dgcnn_gf_module.py create mode 100644 tests/test_models/test_common_modules/test_dgcnn_modules.py create mode 100644 tests/test_models/test_heads/test_dgcnn_decode_head.py diff --git a/.dev_scripts/gather_models.py b/.dev_scripts/gather_models.py index 58919fd444..38270a73b8 100644 --- a/.dev_scripts/gather_models.py +++ b/.dev_scripts/gather_models.py @@ -25,6 +25,7 @@ '_6x_': 73, '_50e_': 50, '_80e_': 80, + '_100e_': 100, '_150e_': 150, '_200e_': 200, '_250e_': 250, diff --git a/README.md b/README.md index 4d0c0a98e2..8a2075919a 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ Support backbones: - [x] PointNet (CVPR'2017) - [x] PointNet++ (NeurIPS'2017) - [x] RegNet (CVPR'2020) +- [x] DGCNN (TOG'2019) Support methods @@ -94,25 +95,27 @@ Support methods - [x] [Group-Free-3D (Arxiv'2021)](configs/groupfree3d/README.md) - [x] [ImVoxelNet (Arxiv'2021)](configs/imvoxelnet/README.md) - [x] [PAConv (CVPR'2021)](configs/paconv/README.md) - -| | ResNet | ResNeXt | SENet |PointNet++ | HRNet | RegNetX | Res2Net | -|--------------------|:--------:|:--------:|:--------:|:---------:|:-----:|:--------:|:-----:| -| SECOND | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| PointPillars | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| FreeAnchor | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| VoteNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| H3DNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| 3DSSD | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| Part-A2 | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| MVXNet | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| CenterPoint | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| SSN | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| ImVoteNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| FCOS3D | ✓ | ☐ | ☐ | ✗ | ☐ | ☐ | ☐ | -| PointNet++ | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| Group-Free-3D | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| ImVoxelNet | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | -| PAConv | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | +- [x] [DGCNN (TOG'2019)](configs/dgcnn/README.md) + +| | ResNet | ResNeXt | SENet |PointNet++ |DGCNN | HRNet | RegNetX | Res2Net | +|--------------------|:--------:|:--------:|:--------:|:---------:|:---------:|:-----:|:--------:|:-----:| +| SECOND | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| PointPillars | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| FreeAnchor | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| VoteNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| H3DNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| 3DSSD | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| Part-A2 | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| MVXNet | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| CenterPoint | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| SSN | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| ImVoteNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| FCOS3D | ✓ | ☐ | ☐ | ✗ | ✗ | ☐ | ☐ | ☐ | +| PointNet++ | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| Group-Free-3D | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| ImVoxelNet | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| PAConv | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| DGCNN | ✗ | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | Other features - [x] [Dynamic Voxelization](configs/dynamic_voxelization/README.md) diff --git a/README_zh-CN.md b/README_zh-CN.md index 30c0d4c561..4fbf874a93 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -74,6 +74,7 @@ MMDetection3D 是一个基于 PyTorch 的目标检测开源工具箱, 下一代 - [x] PointNet (CVPR'2017) - [x] PointNet++ (NeurIPS'2017) - [x] RegNet (CVPR'2020) +- [x] DGCNN (TOG'2019) 已支持的算法： @@ -93,25 +94,27 @@ MMDetection3D 是一个基于 PyTorch 的目标检测开源工具箱, 下一代 - [x] [Group-Free-3D (Arxiv'2021)](configs/groupfree3d/README.md) - [x] [ImVoxelNet (Arxiv'2021)](configs/imvoxelnet/README.md) - [x] [PAConv (CVPR'2021)](configs/paconv/README.md) - -| | ResNet | ResNeXt | SENet |PointNet++ | HRNet | RegNetX | Res2Net | -|--------------------|:--------:|:--------:|:--------:|:---------:|:-----:|:--------:|:-----:| -| SECOND | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| PointPillars | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| FreeAnchor | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| VoteNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| H3DNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| 3DSSD | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| Part-A2 | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| MVXNet | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| CenterPoint | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| SSN | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ | -| ImVoteNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| FCOS3D | ✓ | ☐ | ☐ | ✗ | ☐ | ☐ | ☐ | -| PointNet++ | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| Group-Free-3D | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | -| ImVoxelNet | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | -| PAConv | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | +- [x] [DGCNN (TOG'2019)](configs/dgcnn/README.md) + +| | ResNet | ResNeXt | SENet |PointNet++ |DGCNN | HRNet | RegNetX | Res2Net | +|--------------------|:--------:|:--------:|:--------:|:---------:|:---------:|:-----:|:--------:|:-----:| +| SECOND | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| PointPillars | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| FreeAnchor | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| VoteNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| H3DNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| 3DSSD | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| Part-A2 | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| MVXNet | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| CenterPoint | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| SSN | ☐ | ☐ | ☐ | ✗ | ✗ | ☐ | ✓ | ☐ | +| ImVoteNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| FCOS3D | ✓ | ☐ | ☐ | ✗ | ✗ | ☐ | ☐ | ☐ | +| PointNet++ | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| Group-Free-3D | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| ImVoxelNet | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| PAConv | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | +| DGCNN | ✗ | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | 其他特性 - [x] [Dynamic Voxelization](configs/dynamic_voxelization/README.md) diff --git a/configs/_base_/models/dgcnn.py b/configs/_base_/models/dgcnn.py new file mode 100644 index 0000000000..61e7272692 --- /dev/null +++ b/configs/_base_/models/dgcnn.py @@ -0,0 +1,28 @@ +# model settings +model = dict( + type='EncoderDecoder3D', + backbone=dict( + type='DGCNNBackbone', + in_channels=9, # [xyz, rgb, normal_xyz], modified with dataset + num_samples=(20, 20, 20), + knn_modes=('D-KNN', 'F-KNN', 'F-KNN'), + radius=(None, None, None), + gf_channels=((64, 64), (64, 64), (64, )), + fa_channels=(1024, ), + act_cfg=dict(type='LeakyReLU', negative_slope=0.2)), + decode_head=dict( + type='DGCNNHead', + fp_channels=(1216, 512), + channels=256, + dropout_ratio=0.5, + conv_cfg=dict(type='Conv1d'), + norm_cfg=dict(type='BN1d'), + act_cfg=dict(type='LeakyReLU', negative_slope=0.2), + loss_decode=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + class_weight=None, # modified with dataset + loss_weight=1.0)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='slide')) diff --git a/configs/_base_/schedules/seg_cosine_100e.py b/configs/_base_/schedules/seg_cosine_100e.py new file mode 100644 index 0000000000..3b75932b3a --- /dev/null +++ b/configs/_base_/schedules/seg_cosine_100e.py @@ -0,0 +1,8 @@ +# optimizer +# This schedule is mainly used on S3DIS dataset in segmentation task +optimizer = dict(type='SGD', lr=0.1, momentum=0.9, weight_decay=0.0001) +optimizer_config = dict(grad_clip=None) +lr_config = dict(policy='CosineAnnealing', warmup=None, min_lr=1e-5) + +# runtime settings +runner = dict(type='EpochBasedRunner', max_epochs=100) diff --git a/configs/dgcnn/README.md b/configs/dgcnn/README.md new file mode 100644 index 0000000000..fa31e43f34 --- /dev/null +++ b/configs/dgcnn/README.md @@ -0,0 +1,43 @@ +# Dynamic Graph CNN for Learning on Point Clouds + +## Introduction + + + +We implement DGCNN and provide the results and checkpoints on S3DIS dataset. + +``` +@article{dgcnn, + title={Dynamic Graph CNN for Learning on Point Clouds}, + author={Wang, Yue and Sun, Yongbin and Liu, Ziwei and Sarma, Sanjay E. and Bronstein, Michael M. and Solomon, Justin M.}, + journal={ACM Transactions on Graphics (TOG)}, + year={2019} +} +``` + +**Notice**: We follow the implementations in the original DGCNN paper and a PyTorch implementation of DGCNN [code](https://github.com/AnTao97/dgcnn.pytorch). + +## Results + +### S3DIS + +| Method | Split | Lr schd | Mem (GB) | Inf time (fps) | mIoU (Val set) | Download | +| :-------------------------------------------------------------------------: | :----: | :--------: | :------: | :------------: | :------------: | :----------------------: | +| [DGCNN](./dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py) | Area_1 | cosine 100e | 13.1 | | 68.33 | [model](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area1/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210731_000734-39658f14.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area1/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210731_000734.log.json) | +| [DGCNN](./dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py) | Area_2 | cosine 100e | 13.1 | | 40.68 | [model](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area2/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210731_144648-aea9ecb6.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area2/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210731_144648.log.json) | +| [DGCNN](./dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py) | Area_3 | cosine 100e | 13.1 | | 69.38 | [model](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area3/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210801_154629-2ff50ee0.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area3/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210801_154629.log.json) | +| [DGCNN](./dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py) | Area_4 | cosine 100e | 13.1 | | 50.07 | [model](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area4/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210802_073551-dffab9cd.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area4/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210802_073551.log.json) | +| [DGCNN](./dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py) | Area_5 | cosine 100e | 13.1 | | 50.59 | [model](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area5/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210730_235824-f277e0c5.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area5/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210730_235824.log.json) | +| [DGCNN](./dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py) | Area_6 | cosine 100e | 13.1 | | 77.94 | [model](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area6/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210802_154317-e3511b32.pth) | [log](https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area6/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210802_154317.log.json) | +| [DGCNN](./dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py) | 6-fold | | | | 59.43 | | + +**Notes:** + +- We use XYZ+Color+Normalized_XYZ as input in all the experiments on S3DIS datasets. +- `Area_5` Split means training the model on Area_1, 2, 3, 4, 6 and testing on Area_5. +- `6-fold` Split means the overall result of 6 different splits (Area_1, Area_2, Area_3, Area_4, Area_5 and Area_6 Splits). +- Users need to modify `train_area` and `test_area` in the S3DIS dataset's [config](./configs/_base_/datasets/s3dis_seg-3d-13class.py) to set the training and testing areas, respectively. + +## Indeterminism + +Since DGCNN testing adopts sliding patch inference which involves random point sampling, and the test script uses fixed random seeds while the random seeds of validation in training are not fixed, the test results may be slightly different from the results reported above. diff --git a/configs/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py b/configs/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py new file mode 100644 index 0000000000..6f1b5822a2 --- /dev/null +++ b/configs/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py @@ -0,0 +1,24 @@ +_base_ = [ + '../_base_/datasets/s3dis_seg-3d-13class.py', '../_base_/models/dgcnn.py', + '../_base_/schedules/seg_cosine_100e.py', '../_base_/default_runtime.py' +] + +# data settings +data = dict(samples_per_gpu=32) +evaluation = dict(interval=2) + +# model settings +model = dict( + backbone=dict(in_channels=9), # [xyz, rgb, normalized_xyz] + decode_head=dict( + num_classes=13, ignore_index=13, + loss_decode=dict(class_weight=None)), # S3DIS doesn't use class_weight + test_cfg=dict( + num_points=4096, + block_size=1.0, + sample_rate=0.5, + use_normalized_coord=True, + batch_size=24)) + +# runtime settings +checkpoint_config = dict(interval=2) diff --git a/configs/dgcnn/metafile.yml b/configs/dgcnn/metafile.yml new file mode 100644 index 0000000000..87ff9156bc --- /dev/null +++ b/configs/dgcnn/metafile.yml @@ -0,0 +1,24 @@ +Collections: + - Name: DGCNN + Metadata: + Training Techniques: + - SGD + Training Resources: 4x Titan XP GPUs + Architecture: + - DGCNN + Paper: https://arxiv.org/abs/1801.07829 + README: configs/dgcnn/README.md + +Models: + - Name: dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py + In Collection: DGCNN + Config: configs/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py + Metadata: + Training Data: S3DIS + Training Memory (GB): 13.3 + Results: + - Task: 3D Semantic Segmentation + Dataset: S3DIS + Metrics: + mIoU: 50.59 + Weights: https://download.openmmlab.com/mmdetection3d/v0.17.0_models/dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class/area5/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class_20210730_235824-f277e0c5.pth diff --git a/docs/model_zoo.md b/docs/model_zoo.md index 9192855959..3ebae29c58 100644 --- a/docs/model_zoo.md +++ b/docs/model_zoo.md @@ -77,3 +77,7 @@ Please refer to [ImVoxelNet](https://github.com/open-mmlab/mmdetection3d/blob/ma ### PAConv Please refer to [PAConv](https://github.com/open-mmlab/mmdetection3d/blob/master/configs/paconv) for details. We provide PAConv baselines on S3DIS dataset. + +### DGCNN + +Please refer to [DGCNN](https://github.com/open-mmlab/mmdetection3d/blob/master/configs/dgcnn) for details. We provide DGCNN baselines on S3DIS dataset. diff --git a/docs_zh-CN/model_zoo.md b/docs_zh-CN/model_zoo.md index d897bf9dfd..7a7589e0cd 100644 --- a/docs_zh-CN/model_zoo.md +++ b/docs_zh-CN/model_zoo.md @@ -75,3 +75,11 @@ ### ImVoxelNet 请参考 [ImVoxelNet](https://github.com/open-mmlab/mmdetection3d/blob/master/configs/imvoxelnet) 获取更多细节，我们在 KITTI 数据集上给出了相应的结果。 + +### PAConv + +请参考 [PAConv](https://github.com/open-mmlab/mmdetection3d/blob/master/configs/paconv) 获取更多细节，我们在 S3DIS 数据集上给出了相应的结果. + +### DGCNN + +请参考 [DGCNN](https://github.com/open-mmlab/mmdetection3d/blob/master/configs/dgcnn) 获取更多细节，我们在 S3DIS 数据集上给出了相应的结果. diff --git a/mmdet3d/models/backbones/__init__.py b/mmdet3d/models/backbones/__init__.py index 0251a10456..26c432ccfc 100644 --- a/mmdet3d/models/backbones/__init__.py +++ b/mmdet3d/models/backbones/__init__.py @@ -1,5 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.backbones import SSDVGG, HRNet, ResNet, ResNetV1d, ResNeXt +from .dgcnn import DGCNNBackbone from .multi_backbone import MultiBackbone from .nostem_regnet import NoStemRegNet from .pointnet2_sa_msg import PointNet2SAMSG @@ -8,5 +9,6 @@ __all__ = [ 'ResNet', 'ResNetV1d', 'ResNeXt', 'SSDVGG', 'HRNet', 'NoStemRegNet', - 'SECOND', 'PointNet2SASSG', 'PointNet2SAMSG', 'MultiBackbone' + 'SECOND', 'DGCNNBackbone', 'PointNet2SASSG', 'PointNet2SAMSG', + 'MultiBackbone' ] diff --git a/mmdet3d/models/backbones/dgcnn.py b/mmdet3d/models/backbones/dgcnn.py new file mode 100644 index 0000000000..fe369890e0 --- /dev/null +++ b/mmdet3d/models/backbones/dgcnn.py @@ -0,0 +1,98 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from mmcv.runner import BaseModule, auto_fp16 +from torch import nn as nn + +from mmdet3d.ops import DGCNNFAModule, DGCNNGFModule +from mmdet.models import BACKBONES + + +@BACKBONES.register_module() +class DGCNNBackbone(BaseModule): + """Backbone network for DGCNN. + + Args: + in_channels (int): Input channels of point cloud. + num_samples (tuple[int], optional): The number of samples for knn or + ball query in each graph feature (GF) module. + Defaults to (20, 20, 20). + knn_modes (tuple[str], optional): Mode of KNN of each knn module. + Defaults to ('D-KNN', 'F-KNN', 'F-KNN'). + radius (tuple[float], optional): Sampling radii of each GF module. + Defaults to (None, None, None). + gf_channels (tuple[tuple[int]], optional): Out channels of each mlp in + GF module. Defaults to ((64, 64), (64, 64), (64, )). + fa_channels (tuple[int], optional): Out channels of each mlp in FA + module. Defaults to (1024, ). + act_cfg (dict, optional): Config of activation layer. + Defaults to dict(type='ReLU'). + init_cfg (dict, optional): Initialization config. + Defaults to None. + """ + + def __init__(self, + in_channels, + num_samples=(20, 20, 20), + knn_modes=('D-KNN', 'F-KNN', 'F-KNN'), + radius=(None, None, None), + gf_channels=((64, 64), (64, 64), (64, )), + fa_channels=(1024, ), + act_cfg=dict(type='ReLU'), + init_cfg=None): + super().__init__(init_cfg=init_cfg) + self.num_gf = len(gf_channels) + + assert len(num_samples) == len(knn_modes) == len(radius) == len( + gf_channels), 'Num_samples, knn_modes, radius and gf_channels \ + should have the same length.' + + self.GF_modules = nn.ModuleList() + gf_in_channel = in_channels * 2 + skip_channel_list = [gf_in_channel] # input channel list + + for gf_index in range(self.num_gf): + cur_gf_mlps = list(gf_channels[gf_index]) + cur_gf_mlps = [gf_in_channel] + cur_gf_mlps + gf_out_channel = cur_gf_mlps[-1] + + self.GF_modules.append( + DGCNNGFModule( + mlp_channels=cur_gf_mlps, + num_sample=num_samples[gf_index], + knn_mode=knn_modes[gf_index], + radius=radius[gf_index], + act_cfg=act_cfg)) + skip_channel_list.append(gf_out_channel) + gf_in_channel = gf_out_channel * 2 + + fa_in_channel = sum(skip_channel_list[1:]) + cur_fa_mlps = list(fa_channels) + cur_fa_mlps = [fa_in_channel] + cur_fa_mlps + + self.FA_module = DGCNNFAModule( + mlp_channels=cur_fa_mlps, act_cfg=act_cfg) + + @auto_fp16(apply_to=('points', )) + def forward(self, points): + """Forward pass. + + Args: + points (torch.Tensor): point coordinates with features, + with shape (B, N, in_channels). + + Returns: + dict[str, list[torch.Tensor]]: Outputs after graph feature (GF) and + feature aggregation (FA) modules. + + - gf_points (list[torch.Tensor]): Outputs after each GF module. + - fa_points (torch.Tensor): Outputs after FA module. + """ + gf_points = [points] + + for i in range(self.num_gf): + cur_points = self.GF_modules[i](gf_points[i]) + gf_points.append(cur_points) + + fa_points = self.FA_module(gf_points) + + out = dict(gf_points=gf_points, fa_points=fa_points) + return out diff --git a/mmdet3d/models/decode_heads/__init__.py b/mmdet3d/models/decode_heads/__init__.py index e17d91da0c..2e86c7c8a9 100644 --- a/mmdet3d/models/decode_heads/__init__.py +++ b/mmdet3d/models/decode_heads/__init__.py @@ -1,5 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. +from .dgcnn_head import DGCNNHead from .paconv_head import PAConvHead from .pointnet2_head import PointNet2Head -__all__ = ['PointNet2Head', 'PAConvHead'] +__all__ = ['PointNet2Head', 'DGCNNHead', 'PAConvHead'] diff --git a/mmdet3d/models/decode_heads/dgcnn_head.py b/mmdet3d/models/decode_heads/dgcnn_head.py new file mode 100644 index 0000000000..4d4e1887bc --- /dev/null +++ b/mmdet3d/models/decode_heads/dgcnn_head.py @@ -0,0 +1,67 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from mmcv.cnn.bricks import ConvModule + +from mmdet3d.ops import DGCNNFPModule +from mmdet.models import HEADS +from .decode_head import Base3DDecodeHead + + +@HEADS.register_module() +class DGCNNHead(Base3DDecodeHead): + r"""DGCNN decoder head. + + Decoder head used in `DGCNN `_. + Refer to the + `reimplementation code `_. + + Args: + fp_channels (tuple[int], optional): Tuple of mlp channels in feature + propagation (FP) modules. Defaults to (1216, 512). + """ + + def __init__(self, fp_channels=(1216, 512), **kwargs): + super(DGCNNHead, self).__init__(**kwargs) + + self.FP_module = DGCNNFPModule( + mlp_channels=fp_channels, act_cfg=self.act_cfg) + + # https://github.com/charlesq34/pointnet2/blob/master/models/pointnet2_sem_seg.py#L40 + self.pre_seg_conv = ConvModule( + fp_channels[-1], + self.channels, + kernel_size=1, + bias=False, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def _extract_input(self, feat_dict): + """Extract inputs from features dictionary. + + Args: + feat_dict (dict): Feature dict from backbone. + + Returns: + torch.Tensor: points for decoder. + """ + fa_points = feat_dict['fa_points'] + + return fa_points + + def forward(self, feat_dict): + """Forward pass. + + Args: + feat_dict (dict): Feature dict from backbone. + + Returns: + torch.Tensor: Segmentation map of shape [B, num_classes, N]. + """ + fa_points = self._extract_input(feat_dict) + + fp_points = self.FP_module(fa_points) + fp_points = fp_points.transpose(1, 2).contiguous() + output = self.pre_seg_conv(fp_points) + output = self.cls_seg(output) + + return output diff --git a/mmdet3d/ops/__init__.py b/mmdet3d/ops/__init__.py index 38e2ea7367..1dafa428ac 100644 --- a/mmdet3d/ops/__init__.py +++ b/mmdet3d/ops/__init__.py @@ -4,6 +4,7 @@ sigmoid_focal_loss) from .ball_query import ball_query +from .dgcnn_modules import DGCNNFAModule, DGCNNFPModule, DGCNNGFModule from .furthest_point_sample import (Points_Sampler, furthest_point_sample, furthest_point_sample_with_dist) from .gather_points import gather_points @@ -34,8 +35,9 @@ 'furthest_point_sample_with_dist', 'three_interpolate', 'three_nn', 'gather_points', 'grouping_operation', 'group_points', 'GroupAll', 'QueryAndGroup', 'PointSAModule', 'PointSAModuleMSG', 'PointFPModule', - 'points_in_boxes_all', 'get_compiler_version', 'assign_score_withk', - 'get_compiling_cuda_version', 'Points_Sampler', 'build_sa_module', - 'PAConv', 'PAConvCUDA', 'PAConvSAModuleMSG', 'PAConvSAModule', - 'PAConvCUDASAModule', 'PAConvCUDASAModuleMSG' + 'DGCNNFPModule', 'DGCNNGFModule', 'DGCNNFAModule', 'points_in_boxes_all', + 'get_compiler_version', 'assign_score_withk', 'get_compiling_cuda_version', + 'Points_Sampler', 'build_sa_module', 'PAConv', 'PAConvCUDA', + 'PAConvSAModuleMSG', 'PAConvSAModule', 'PAConvCUDASAModule', + 'PAConvCUDASAModuleMSG' ] diff --git a/mmdet3d/ops/dgcnn_modules/__init__.py b/mmdet3d/ops/dgcnn_modules/__init__.py new file mode 100644 index 0000000000..67beb0907f --- /dev/null +++ b/mmdet3d/ops/dgcnn_modules/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .dgcnn_fa_module import DGCNNFAModule +from .dgcnn_fp_module import DGCNNFPModule +from .dgcnn_gf_module import DGCNNGFModule + +__all__ = ['DGCNNFAModule', 'DGCNNFPModule', 'DGCNNGFModule'] diff --git a/mmdet3d/ops/dgcnn_modules/dgcnn_fa_module.py b/mmdet3d/ops/dgcnn_modules/dgcnn_fa_module.py new file mode 100644 index 0000000000..b0975e691b --- /dev/null +++ b/mmdet3d/ops/dgcnn_modules/dgcnn_fa_module.py @@ -0,0 +1,68 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from mmcv.cnn import ConvModule +from mmcv.runner import BaseModule, force_fp32 +from torch import nn as nn + + +class DGCNNFAModule(BaseModule): + """Point feature aggregation module used in DGCNN. + + Aggregate all the features of points. + + Args: + mlp_channels (list[int]): List of mlp channels. + norm_cfg (dict, optional): Type of normalization method. + Defaults to dict(type='BN1d'). + act_cfg (dict, optional): Type of activation method. + Defaults to dict(type='ReLU'). + init_cfg (dict, optional): Initialization config. Defaults to None. + """ + + def __init__(self, + mlp_channels, + norm_cfg=dict(type='BN1d'), + act_cfg=dict(type='ReLU'), + init_cfg=None): + super().__init__(init_cfg=init_cfg) + self.fp16_enabled = False + self.mlps = nn.Sequential() + for i in range(len(mlp_channels) - 1): + self.mlps.add_module( + f'layer{i}', + ConvModule( + mlp_channels[i], + mlp_channels[i + 1], + kernel_size=(1, ), + stride=(1, ), + conv_cfg=dict(type='Conv1d'), + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + + @force_fp32() + def forward(self, points): + """forward. + + Args: + points (List[Tensor]): tensor of the features to be aggregated. + + Returns: + Tensor: (B, N, M) M = mlp[-1], tensor of the output points. + """ + + if len(points) > 1: + new_points = torch.cat(points[1:], dim=-1) + new_points = new_points.transpose(1, 2).contiguous() # (B, C, N) + new_points_copy = new_points + + new_points = self.mlps(new_points) + + new_fa_points = new_points.max(dim=-1, keepdim=True)[0] + new_fa_points = new_fa_points.repeat(1, 1, new_points.shape[-1]) + + new_points = torch.cat([new_fa_points, new_points_copy], dim=1) + new_points = new_points.transpose(1, 2).contiguous() + else: + new_points = points + + return new_points diff --git a/mmdet3d/ops/dgcnn_modules/dgcnn_fp_module.py b/mmdet3d/ops/dgcnn_modules/dgcnn_fp_module.py new file mode 100644 index 0000000000..c871721bc1 --- /dev/null +++ b/mmdet3d/ops/dgcnn_modules/dgcnn_fp_module.py @@ -0,0 +1,59 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from mmcv.cnn import ConvModule +from mmcv.runner import BaseModule, force_fp32 +from torch import nn as nn + + +class DGCNNFPModule(BaseModule): + """Point feature propagation module used in DGCNN. + + Propagate the features from one set to another. + + Args: + mlp_channels (list[int]): List of mlp channels. + norm_cfg (dict, optional): Type of activation method. + Defaults to dict(type='BN1d'). + act_cfg (dict, optional): Type of activation method. + Defaults to dict(type='ReLU'). + init_cfg (dict, optional): Initialization config. Defaults to None. + """ + + def __init__(self, + mlp_channels, + norm_cfg=dict(type='BN1d'), + act_cfg=dict(type='ReLU'), + init_cfg=None): + super().__init__(init_cfg=init_cfg) + self.fp16_enabled = False + self.mlps = nn.Sequential() + for i in range(len(mlp_channels) - 1): + self.mlps.add_module( + f'layer{i}', + ConvModule( + mlp_channels[i], + mlp_channels[i + 1], + kernel_size=(1, ), + stride=(1, ), + conv_cfg=dict(type='Conv1d'), + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + + @force_fp32() + def forward(self, points): + """forward. + + Args: + points (Tensor): (B, N, C) tensor of the input points. + + Returns: + Tensor: (B, N, M) M = mlp[-1], tensor of the new points. + """ + + if points is not None: + new_points = points.transpose(1, 2).contiguous() # (B, C, N) + new_points = self.mlps(new_points) + new_points = new_points.transpose(1, 2).contiguous() + else: + new_points = points + + return new_points diff --git a/mmdet3d/ops/dgcnn_modules/dgcnn_gf_module.py b/mmdet3d/ops/dgcnn_modules/dgcnn_gf_module.py new file mode 100644 index 0000000000..e317ccd086 --- /dev/null +++ b/mmdet3d/ops/dgcnn_modules/dgcnn_gf_module.py @@ -0,0 +1,222 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from mmcv.cnn import ConvModule +from torch import nn as nn +from torch.nn import functional as F + +from ..group_points import GroupAll, QueryAndGroup, grouping_operation + + +class BaseDGCNNGFModule(nn.Module): + """Base module for point graph feature module used in DGCNN. + + Args: + radii (list[float]): List of radius in each knn or ball query. + sample_nums (list[int]): Number of samples in each knn or ball query. + mlp_channels (list[list[int]]): Specify of the dgcnn before + the global pooling for each graph feature module. + knn_modes (list[str], optional): Type of KNN method, valid mode + ['F-KNN', 'D-KNN'], Defaults to ['F-KNN']. + dilated_group (bool, optional): Whether to use dilated ball query. + Defaults to False. + use_xyz (bool, optional): Whether to use xyz as point features. + Defaults to True. + pool_mode (str, optional): Type of pooling method. Defaults to 'max'. + normalize_xyz (bool, optional): If ball query, whether to normalize + local XYZ with radius. Defaults to False. + grouper_return_grouped_xyz (bool, optional): Whether to return grouped + xyz in `QueryAndGroup`. Defaults to False. + grouper_return_grouped_idx (bool, optional): Whether to return grouped + idx in `QueryAndGroup`. Defaults to False. + """ + + def __init__(self, + radii, + sample_nums, + mlp_channels, + knn_modes=['F-KNN'], + dilated_group=False, + use_xyz=True, + pool_mode='max', + normalize_xyz=False, + grouper_return_grouped_xyz=False, + grouper_return_grouped_idx=False): + super(BaseDGCNNGFModule, self).__init__() + + assert len(sample_nums) == len( + mlp_channels + ), 'Num_samples and mlp_channels should have the same length.' + assert pool_mode in ['max', 'avg' + ], "Pool_mode should be one of ['max', 'avg']." + assert isinstance(knn_modes, list) or isinstance( + knn_modes, tuple), 'The type of knn_modes should be list or tuple.' + + if isinstance(mlp_channels, tuple): + mlp_channels = list(map(list, mlp_channels)) + self.mlp_channels = mlp_channels + + self.pool_mode = pool_mode + self.groupers = nn.ModuleList() + self.mlps = nn.ModuleList() + self.knn_modes = knn_modes + + for i in range(len(sample_nums)): + sample_num = sample_nums[i] + if sample_num is not None: + if self.knn_modes[i] == 'D-KNN': + grouper = QueryAndGroup( + radii[i], + sample_num, + use_xyz=use_xyz, + normalize_xyz=normalize_xyz, + return_grouped_xyz=grouper_return_grouped_xyz, + return_grouped_idx=True) + else: + grouper = QueryAndGroup( + radii[i], + sample_num, + use_xyz=use_xyz, + normalize_xyz=normalize_xyz, + return_grouped_xyz=grouper_return_grouped_xyz, + return_grouped_idx=grouper_return_grouped_idx) + else: + grouper = GroupAll(use_xyz) + self.groupers.append(grouper) + + def _pool_features(self, features): + """Perform feature aggregation using pooling operation. + + Args: + features (torch.Tensor): (B, C, N, K) + Features of locally grouped points before pooling. + + Returns: + torch.Tensor: (B, C, N) + Pooled features aggregating local information. + """ + if self.pool_mode == 'max': + # (B, C, N, 1) + new_features = F.max_pool2d( + features, kernel_size=[1, features.size(3)]) + elif self.pool_mode == 'avg': + # (B, C, N, 1) + new_features = F.avg_pool2d( + features, kernel_size=[1, features.size(3)]) + else: + raise NotImplementedError + + return new_features.squeeze(-1).contiguous() + + def forward(self, points): + """forward. + + Args: + points (Tensor): (B, N, C) input points. + + Returns: + List[Tensor]: (B, N, C1) new points generated from each graph + feature module. + """ + new_points_list = [points] + + for i in range(len(self.groupers)): + + new_points = new_points_list[i] + new_points_trans = new_points.transpose( + 1, 2).contiguous() # (B, C, N) + + if self.knn_modes[i] == 'D-KNN': + # (B, N, C) -> (B, N, K) + idx = self.groupers[i](new_points[..., -3:].contiguous(), + new_points[..., -3:].contiguous())[-1] + + grouped_results = grouping_operation( + new_points_trans, idx) # (B, C, N) -> (B, C, N, K) + grouped_results -= new_points_trans.unsqueeze(-1) + else: + grouped_results = self.groupers[i]( + new_points, new_points) # (B, N, C) -> (B, C, N, K) + + new_points = new_points_trans.unsqueeze(-1).repeat( + 1, 1, 1, grouped_results.shape[-1]) + new_points = torch.cat([grouped_results, new_points], dim=1) + + # (B, mlp[-1], N, K) + new_points = self.mlps[i](new_points) + + # (B, mlp[-1], N) + new_points = self._pool_features(new_points) + new_points = new_points.transpose(1, 2).contiguous() + new_points_list.append(new_points) + + return new_points + + +class DGCNNGFModule(BaseDGCNNGFModule): + """Point graph feature module used in DGCNN. + + Args: + mlp_channels (list[int]): Specify of the dgcnn before + the global pooling for each graph feature module. + num_sample (int, optional): Number of samples in each knn or ball + query. Defaults to None. + knn_mode (str, optional): Type of KNN method, valid mode + ['F-KNN', 'D-KNN']. Defaults to 'F-KNN'. + radius (float, optional): Radius to group with. + Defaults to None. + dilated_group (bool, optional): Whether to use dilated ball query. + Defaults to False. + norm_cfg (dict, optional): Type of normalization method. + Defaults to dict(type='BN2d'). + act_cfg (dict, optional): Type of activation method. + Defaults to dict(type='ReLU'). + use_xyz (bool, optional): Whether to use xyz as point features. + Defaults to True. + pool_mode (str, optional): Type of pooling method. + Defaults to 'max'. + normalize_xyz (bool, optional): If ball query, whether to normalize + local XYZ with radius. Defaults to False. + bias (bool | str, optional): If specified as `auto`, it will be decided + by the norm_cfg. Bias will be set as True if `norm_cfg` is None, + otherwise False. Defaults to 'auto'. + """ + + def __init__(self, + mlp_channels, + num_sample=None, + knn_mode='F-KNN', + radius=None, + dilated_group=False, + norm_cfg=dict(type='BN2d'), + act_cfg=dict(type='ReLU'), + use_xyz=True, + pool_mode='max', + normalize_xyz=False, + bias='auto'): + super(DGCNNGFModule, self).__init__( + mlp_channels=[mlp_channels], + sample_nums=[num_sample], + knn_modes=[knn_mode], + radii=[radius], + use_xyz=use_xyz, + pool_mode=pool_mode, + normalize_xyz=normalize_xyz, + dilated_group=dilated_group) + + for i in range(len(self.mlp_channels)): + mlp_channel = self.mlp_channels[i] + + mlp = nn.Sequential() + for i in range(len(mlp_channel) - 1): + mlp.add_module( + f'layer{i}', + ConvModule( + mlp_channel[i], + mlp_channel[i + 1], + kernel_size=(1, 1), + stride=(1, 1), + conv_cfg=dict(type='Conv2d'), + norm_cfg=norm_cfg, + act_cfg=act_cfg, + bias=bias)) + self.mlps.append(mlp) diff --git a/tests/test_models/test_backbones.py b/tests/test_models/test_backbones.py index 6a3d2cb422..5c9f5edfe8 100644 --- a/tests/test_models/test_backbones.py +++ b/tests/test_models/test_backbones.py @@ -297,3 +297,36 @@ def test_pointnet2_sa_msg(): assert sa_indices[2].shape == torch.Size([1, 256]) assert sa_indices[3].shape == torch.Size([1, 64]) assert sa_indices[4].shape == torch.Size([1, 16]) + + +def test_dgcnn_gf(): + if not torch.cuda.is_available(): + pytest.skip() + + # DGCNNGF used in segmentation + cfg = dict( + type='DGCNNBackbone', + in_channels=6, + num_samples=(20, 20, 20), + knn_modes=['D-KNN', 'F-KNN', 'F-KNN'], + radius=(None, None, None), + gf_channels=((64, 64), (64, 64), (64, )), + fa_channels=(1024, ), + act_cfg=dict(type='ReLU')) + + self = build_backbone(cfg) + self.cuda() + + xyz = np.fromfile('tests/data/sunrgbd/points/000001.bin', dtype=np.float32) + xyz = torch.from_numpy(xyz).view(1, -1, 6).cuda() # (B, N, 6) + # test forward + ret_dict = self(xyz) + gf_points = ret_dict['gf_points'] + fa_points = ret_dict['fa_points'] + + assert len(gf_points) == 4 + assert gf_points[0].shape == torch.Size([1, 100, 6]) + assert gf_points[1].shape == torch.Size([1, 100, 64]) + assert gf_points[2].shape == torch.Size([1, 100, 64]) + assert gf_points[3].shape == torch.Size([1, 100, 64]) + assert fa_points.shape == torch.Size([1, 100, 1216]) diff --git a/tests/test_models/test_common_modules/test_dgcnn_modules.py b/tests/test_models/test_common_modules/test_dgcnn_modules.py new file mode 100644 index 0000000000..031971b459 --- /dev/null +++ b/tests/test_models/test_common_modules/test_dgcnn_modules.py @@ -0,0 +1,92 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np +import pytest +import torch + + +def test_dgcnn_gf_module(): + if not torch.cuda.is_available(): + pytest.skip() + from mmdet3d.ops import DGCNNGFModule + + self = DGCNNGFModule( + mlp_channels=[18, 64, 64], + num_sample=20, + knn_mod='D-KNN', + radius=None, + norm_cfg=dict(type='BN2d'), + act_cfg=dict(type='ReLU'), + pool_mod='max').cuda() + + assert self.mlps[0].layer0.conv.in_channels == 18 + assert self.mlps[0].layer0.conv.out_channels == 64 + + xyz = np.fromfile('tests/data/sunrgbd/points/000001.bin', np.float32) + + # (B, N, C) + xyz = torch.from_numpy(xyz).view(1, -1, 3).cuda() + points = xyz.repeat([1, 1, 3]) + + # test forward + new_points = self(points) + + assert new_points.shape == torch.Size([1, 200, 64]) + + # test F-KNN mod + self = DGCNNGFModule( + mlp_channels=[6, 64, 64], + num_sample=20, + knn_mod='F-KNN', + radius=None, + norm_cfg=dict(type='BN2d'), + act_cfg=dict(type='ReLU'), + pool_mod='max').cuda() + + # test forward + new_points = self(xyz) + assert new_points.shape == torch.Size([1, 200, 64]) + + # test ball query + self = DGCNNGFModule( + mlp_channels=[6, 64, 64], + num_sample=20, + knn_mod='F-KNN', + radius=0.2, + norm_cfg=dict(type='BN2d'), + act_cfg=dict(type='ReLU'), + pool_mod='max').cuda() + + +def test_dgcnn_fa_module(): + if not torch.cuda.is_available(): + pytest.skip() + from mmdet3d.ops import DGCNNFAModule + + self = DGCNNFAModule(mlp_channels=[24, 16]).cuda() + assert self.mlps.layer0.conv.in_channels == 24 + assert self.mlps.layer0.conv.out_channels == 16 + + points = [torch.rand(1, 200, 12).float().cuda() for _ in range(3)] + + fa_points = self(points) + assert fa_points.shape == torch.Size([1, 200, 40]) + + +def test_dgcnn_fp_module(): + if not torch.cuda.is_available(): + pytest.skip() + from mmdet3d.ops import DGCNNFPModule + + self = DGCNNFPModule(mlp_channels=[24, 16]).cuda() + assert self.mlps.layer0.conv.in_channels == 24 + assert self.mlps.layer0.conv.out_channels == 16 + + xyz = np.fromfile('tests/data/sunrgbd/points/000001.bin', + np.float32).reshape((-1, 6)) + + # (B, N, 3) + xyz = torch.from_numpy(xyz).view(1, -1, 3).cuda() + points = xyz.repeat([1, 1, 8]).cuda() + + fp_points = self(points) + assert fp_points.shape == torch.Size([1, 200, 16]) diff --git a/tests/test_models/test_heads/test_dgcnn_decode_head.py b/tests/test_models/test_heads/test_dgcnn_decode_head.py new file mode 100644 index 0000000000..6d1f149530 --- /dev/null +++ b/tests/test_models/test_heads/test_dgcnn_decode_head.py @@ -0,0 +1,68 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np +import pytest +import torch +from mmcv.cnn.bricks import ConvModule + +from mmdet3d.models.builder import build_head + + +def test_dgcnn_decode_head_loss(): + if not torch.cuda.is_available(): + pytest.skip('test requires GPU and torch+cuda') + dgcnn_decode_head_cfg = dict( + type='DGCNNHead', + fp_channels=(1024, 512), + channels=256, + num_classes=13, + dropout_ratio=0.5, + conv_cfg=dict(type='Conv1d'), + norm_cfg=dict(type='BN1d'), + act_cfg=dict(type='LeakyReLU', negative_slope=0.2), + loss_decode=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + class_weight=None, + loss_weight=1.0), + ignore_index=13) + + self = build_head(dgcnn_decode_head_cfg) + self.cuda() + assert isinstance(self.conv_seg, torch.nn.Conv1d) + assert self.conv_seg.in_channels == 256 + assert self.conv_seg.out_channels == 13 + assert self.conv_seg.kernel_size == (1, ) + assert isinstance(self.pre_seg_conv, ConvModule) + assert isinstance(self.pre_seg_conv.conv, torch.nn.Conv1d) + assert self.pre_seg_conv.conv.in_channels == 512 + assert self.pre_seg_conv.conv.out_channels == 256 + assert self.pre_seg_conv.conv.kernel_size == (1, ) + assert isinstance(self.pre_seg_conv.bn, torch.nn.BatchNorm1d) + assert self.pre_seg_conv.bn.num_features == 256 + + # test forward + fa_points = torch.rand(2, 4096, 1024).float().cuda() + input_dict = dict(fa_points=fa_points) + seg_logits = self(input_dict) + assert seg_logits.shape == torch.Size([2, 13, 4096]) + + # test loss + pts_semantic_mask = torch.randint(0, 13, (2, 4096)).long().cuda() + losses = self.losses(seg_logits, pts_semantic_mask) + assert losses['loss_sem_seg'].item() > 0 + + # test loss with ignore_index + ignore_index_mask = torch.ones_like(pts_semantic_mask) * 13 + losses = self.losses(seg_logits, ignore_index_mask) + assert losses['loss_sem_seg'].item() == 0 + + # test loss with class_weight + dgcnn_decode_head_cfg['loss_decode'] = dict( + type='CrossEntropyLoss', + use_sigmoid=False, + class_weight=np.random.rand(13), + loss_weight=1.0) + self = build_head(dgcnn_decode_head_cfg) + self.cuda() + losses = self.losses(seg_logits, pts_semantic_mask) + assert losses['loss_sem_seg'].item() > 0 diff --git a/tests/test_models/test_segmentors.py b/tests/test_models/test_segmentors.py index faff3f9515..0974f9f507 100644 --- a/tests/test_models/test_segmentors.py +++ b/tests/test_models/test_segmentors.py @@ -304,3 +304,48 @@ def test_paconv_cuda_ssg(): results = self.forward(return_loss=False, **data_dict) assert results[0]['semantic_mask'].shape == torch.Size([200]) assert results[1]['semantic_mask'].shape == torch.Size([100]) + + +def test_dgcnn(): + if not torch.cuda.is_available(): + pytest.skip('test requires GPU and torch+cuda') + + set_random_seed(0, True) + dgcnn_cfg = _get_segmentor_cfg( + 'dgcnn/dgcnn_32x4_cosine_100e_s3dis_seg-3d-13class.py') + dgcnn_cfg.test_cfg.num_points = 32 + self = build_segmentor(dgcnn_cfg).cuda() + points = [torch.rand(4096, 9).float().cuda() for _ in range(2)] + img_metas = [dict(), dict()] + gt_masks = [torch.randint(0, 13, (4096, )).long().cuda() for _ in range(2)] + + # test forward_train + losses = self.forward_train(points, img_metas, gt_masks) + assert losses['decode.loss_sem_seg'].item() >= 0 + + # test loss with ignore_index + ignore_masks = [torch.ones_like(gt_masks[0]) * 13 for _ in range(2)] + losses = self.forward_train(points, img_metas, ignore_masks) + assert losses['decode.loss_sem_seg'].item() == 0 + + # test simple_test + self.eval() + with torch.no_grad(): + scene_points = [ + torch.randn(500, 6).float().cuda() * 3.0, + torch.randn(200, 6).float().cuda() * 2.5 + ] + results = self.simple_test(scene_points, img_metas) + assert results[0]['semantic_mask'].shape == torch.Size([500]) + assert results[1]['semantic_mask'].shape == torch.Size([200]) + + # test aug_test + with torch.no_grad(): + scene_points = [ + torch.randn(2, 500, 6).float().cuda() * 3.0, + torch.randn(2, 200, 6).float().cuda() * 2.5 + ] + img_metas = [[dict(), dict()], [dict(), dict()]] + results = self.aug_test(scene_points, img_metas) + assert results[0]['semantic_mask'].shape == torch.Size([500]) + assert results[1]['semantic_mask'].shape == torch.Size([200]) From 459c6379a7c85dc52f35632bcec00c47a831b406 Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Mon, 6 Sep 2021 13:10:06 +0800 Subject: [PATCH 25/51] Change cam rot_3d_in_axis (#906) --- mmdet3d/core/bbox/structures/cam_box3d.py | 11 ++-------- mmdet3d/core/bbox/structures/utils.py | 4 ++-- mmdet3d/models/dense_heads/ssd_3d_head.py | 3 ++- tests/test_models/test_detectors.py | 3 ++- tests/test_utils/test_box3d.py | 26 ++++++++++++++++++++++- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/mmdet3d/core/bbox/structures/cam_box3d.py b/mmdet3d/core/bbox/structures/cam_box3d.py index dd2a8eed19..ac5cf8286c 100644 --- a/mmdet3d/core/bbox/structures/cam_box3d.py +++ b/mmdet3d/core/bbox/structures/cam_box3d.py @@ -137,11 +137,8 @@ def corners(self): corners_norm = corners_norm - dims.new_tensor([0.5, 1, 0.5]) corners = dims.view([-1, 1, 3]) * corners_norm.reshape([1, 8, 3]) - # positive direction of the gravity axis - # in cam coord system points to the earth - # so the rotation is clockwise if viewed from above corners = rotation_3d_in_axis( - corners, self.tensor[:, 6], axis=self.YAW_AXIS, clockwise=True) + corners, self.tensor[:, 6], axis=self.YAW_AXIS) corners += self.tensor[:, :3].view(-1, 1, 3) return corners @@ -182,11 +179,7 @@ def rotate(self, angle, points=None): self.tensor[:, 0:3], angle, axis=self.YAW_AXIS, - return_mat=True, - # positive direction of the gravity axis - # in cam coord system points to the earth - # so the rotation is clockwise if viewed from above - clockwise=True) + return_mat=True) else: rot_mat_T = angle rot_sin = rot_mat_T[2, 0] diff --git a/mmdet3d/core/bbox/structures/utils.py b/mmdet3d/core/bbox/structures/utils.py index 9ad4a6f6df..64483506cc 100644 --- a/mmdet3d/core/bbox/structures/utils.py +++ b/mmdet3d/core/bbox/structures/utils.py @@ -71,9 +71,9 @@ def rotation_3d_in_axis(points, if points.shape[-1] == 3: if axis == 1 or axis == -2: rot_mat_T = torch.stack([ - torch.stack([rot_cos, zeros, rot_sin]), + torch.stack([rot_cos, zeros, -rot_sin]), torch.stack([zeros, ones, zeros]), - torch.stack([-rot_sin, zeros, rot_cos]) + torch.stack([rot_sin, zeros, rot_cos]) ]) elif axis == 2 or axis == -1: rot_mat_T = torch.stack([ diff --git a/mmdet3d/models/dense_heads/ssd_3d_head.py b/mmdet3d/models/dense_heads/ssd_3d_head.py index 83fc37cc2d..0ce042fef1 100644 --- a/mmdet3d/models/dense_heads/ssd_3d_head.py +++ b/mmdet3d/models/dense_heads/ssd_3d_head.py @@ -391,7 +391,8 @@ def get_targets_single(self, # LiDARInstance3DBoxes and DepthInstance3DBoxes canonical_xyz = rotation_3d_in_axis( canonical_xyz.unsqueeze(0).transpose(0, 1), - -gt_bboxes_3d.yaw[assignment], 2).squeeze(1) + -gt_bboxes_3d.yaw[assignment], + axis=2).squeeze(1) distance_front = torch.clamp( size_res_targets[:, 0] - canonical_xyz[:, 0], min=0) distance_back = torch.clamp( diff --git a/tests/test_models/test_detectors.py b/tests/test_models/test_detectors.py index dc7a70812b..c9c3527350 100644 --- a/tests/test_models/test_detectors.py +++ b/tests/test_models/test_detectors.py @@ -437,7 +437,8 @@ def test_imvoxelnet(): if not torch.cuda.is_available(): pytest.skip('test requires GPU and torch+cuda') - imvoxelnet_cfg = _get_detector_cfg('imvoxelnet/imvoxelnet_kitti-3d-car.py') + imvoxelnet_cfg = _get_detector_cfg( + 'imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py') self = build_detector(imvoxelnet_cfg).cuda() imgs = torch.rand([1, 3, 384, 1280], dtype=torch.float32).cuda() gt_bboxes_3d = [LiDARInstance3DBoxes(torch.rand([3, 7], device='cuda'))] diff --git a/tests/test_utils/test_box3d.py b/tests/test_utils/test_box3d.py index 974f22caec..00fab68db5 100644 --- a/tests/test_utils/test_box3d.py +++ b/tests/test_utils/test_box3d.py @@ -1553,13 +1553,37 @@ def test_rotation_3d_in_axis(): [[-0.2555, -0.2683, 0.0000], [-0.2555, -0.2683, 0.9072]]]) angles = [np.pi / 2, -np.pi / 2] - rotated = rotation_3d_in_axis(points, angles, axis=0) + rotated = rotation_3d_in_axis(points, angles, axis=0).numpy() expected_rotated = np.array([[[-0.4599, 0.0000, -0.0471], [-0.4599, -1.8433, -0.0471]], [[-0.2555, 0.0000, 0.2683], [-0.2555, 0.9072, 0.2683]]]) assert np.allclose(rotated, expected_rotated, atol=1e-3) + points = torch.tensor([[[-0.4599, -0.0471, 0.0000], + [-0.4599, -0.0471, 1.8433]], + [[-0.2555, -0.2683, 0.0000], + [-0.2555, -0.2683, 0.9072]]]) + angles = [np.pi / 2, -np.pi / 2] + rotated = rotation_3d_in_axis(points, angles, axis=1).numpy() + expected_rotated = np.array([[[0.0000, -0.0471, 0.4599], + [1.8433, -0.0471, 0.4599]], + [[0.0000, -0.2683, -0.2555], + [-0.9072, -0.2683, -0.2555]]]) + assert np.allclose(rotated, expected_rotated, atol=1e-3) + + points = torch.tensor([[[-0.4599, -0.0471, 0.0000], + [-0.4599, 0.0471, 1.8433]], + [[-0.2555, -0.2683, 0.0000], + [0.2555, -0.2683, 0.9072]]]) + angles = [np.pi / 2, -np.pi / 2] + rotated = rotation_3d_in_axis(points, angles, axis=2).numpy() + expected_rotated = np.array([[[0.0471, -0.4599, 0.0000], + [-0.0471, -0.4599, 1.8433]], + [[-0.2683, 0.2555, 0.0000], + [-0.2683, -0.2555, 0.9072]]]) + assert np.allclose(rotated, expected_rotated, atol=1e-3) + points = torch.tensor([[[-0.0471, 0.0000], [-0.0471, 1.8433]], [[-0.2683, 0.0000], [-0.2683, 0.9072]]]) angles = [np.pi / 2, -np.pi / 2] From 2ae6b55274079d190eb029016d10d7b641067eb9 Mon Sep 17 00:00:00 2001 From: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Date: Tue, 7 Sep 2021 21:54:57 +0800 Subject: [PATCH 26/51] [Doc] Add coord sys tutorial pic and change links to dev branch (#912) * Modify link branch and add pic * Fix pic --- docs/tutorials/coord_sys_tutorial.md | 16 ++++++++-------- resources/coord_sys_all.png | Bin 0 -> 60886 bytes 2 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 resources/coord_sys_all.png diff --git a/docs/tutorials/coord_sys_tutorial.md b/docs/tutorials/coord_sys_tutorial.md index a7e7474736..01842f2264 100644 --- a/docs/tutorials/coord_sys_tutorial.md +++ b/docs/tutorials/coord_sys_tutorial.md @@ -47,7 +47,7 @@ The definition of coordinate systems in this tutorial is actually **more than ju The illustration of the three coordinate systems is shown below: -![coord_sys_all](https://github.com/open-mmlab/mmdetection3d/blob/master/resources/coord_sys_all.png) +![coord_sys_all](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/resources/coord_sys_all.png) The three figures above are the 3D coordinate systems while the three figures below are the bird's eye view. @@ -132,7 +132,7 @@ __|____|____|____|_________\ x right ### KITTI -The raw annotation of KITTI is under Camera coordinate system, see [get_label_anno](https://github.com/open-mmlab/mmdetection3d/blob/master/tools/data_converter/kitti_data_utils.py). In MMDetection3D, to train models on KITTI, the data is first converted from camera to LiDAR, see [get_ann_info](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/datasets/kitti_dataset.py). +The raw annotation of KITTI is under Camera coordinate system, see [get_label_anno](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/tools/data_converter/kitti_data_utils.py). In MMDetection3D, to train models on KITTI, the data is first converted from camera to LiDAR, see [get_ann_info](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/mmdet3d/datasets/kitti_dataset.py). In SECOND, the LiDAR coordinate system for a box is defined as follows (a bird's eye view): @@ -153,7 +153,7 @@ We use the KITTI-format data of Waymo dataset. Therefore, KITTI and Waymo also s ### NuScenes -NuScenes provides a toolkit for evaluation, in which each box is wrapped into a `Box` instance. The coordinate system of `Box` is different from our LiDAR coordinate system in that the first two elements of the box dimension correspond to , or , respectively, instead of the reverse. For more details, please refer to the NuScenes [tutorial](https://github.com/open-mmlab/mmdetection3d/blob/master/docs/datasets/nuscenes_det.md#notes). +NuScenes provides a toolkit for evaluation, in which each box is wrapped into a `Box` instance. The coordinate system of `Box` is different from our LiDAR coordinate system in that the first two elements of the box dimension correspond to , or , respectively, instead of the reverse. For more details, please refer to the NuScenes [tutorial](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/docs/datasets/nuscenes_det.md#notes). Readers may refer to the [NuScenes development kit](https://github.com/nutonomy/nuscenes-devkit/tree/master/python-sdk/nuscenes/eval/detection) for the definition of a [NuScenes box](https://github.com/nutonomy/nuscenes-devkit/blob/2c6a752319f23910d5f55cc995abc547a9e54142/python-sdk/nuscenes/utils/data_classes.py#L457) and implementation of [NuScenes evaluation](https://github.com/nutonomy/nuscenes-devkit/blob/master/python-sdk/nuscenes/eval/detection/evaluate.py). @@ -171,7 +171,7 @@ The raw data of ScanNet is not point cloud but mesh. The sampled point cloud dat The raw data of SUN RGB-D is not point cloud but RGB-D image. By back projection, we obtain the corresponding point cloud for each image, which is under our Depth coordinate system. However, the annotation is not under our system and thus needs conversion. -For the conversion from raw annotation to annotation under our Depth coordinate system, please refer to [sunrgbd_data_utils.py](https://github.com/open-mmlab/mmdetection3d/blob/master/tools/data_converter/sunrgbd_data_utils.py). +For the conversion from raw annotation to annotation under our Depth coordinate system, please refer to [sunrgbd_data_utils.py](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/tools/data_converter/sunrgbd_data_utils.py). ### S3DIS @@ -199,25 +199,25 @@ Finally, the yaw angle should also be converted: - -See the code [here](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/bbox/structures/box_3d_mode.py) for more details. +See the code [here](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/mmdet3d/core/bbox/structures/box_3d_mode.py) for more details. ### Bird's Eye View The BEV of a camera coordinate system box is if the 3D box is . The inversion of the sign of the yaw angle is because the positive direction of the gravity axis of the Camera coordinate system points to the ground. -See the code [here](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/bbox/structures/cam_box3d.py) for more details. +See the code [here](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/mmdet3d/core/bbox/structures/cam_box3d.py) for more details. ### Rotation of boxes We set the rotation of all kinds of boxes to be counter-clockwise about the gravity axis. Therefore, to rotate a 3D box we first calculate the new box center, and then we add the rotation angle to the yaw angle. -See the code [here](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/bbox/structures/cam_box3d.py) for more details. +See the code [here](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/mmdet3d/core/bbox/structures/cam_box3d.py) for more details. ## Common FAQ #### Q1: Are the box related ops universal to all coordinate system types? -No. For example, the ops under [this folder](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/ops/roiaware_pool3d) are applicable to boxes under Depth or LiDAR coordinate system only. The evaluation functions for KITTI dataset [here](https://github.com/open-mmlab/mmdetection3d/blob/master/mmdet3d/core/evaluation/kitti_utils) are only applicable to boxes under Camera coordinate system since the rotation is clockwise if viewed from above. +No. For example, the ops under [this folder](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/mmdet3d/ops/roiaware_pool3d) are applicable to boxes under Depth or LiDAR coordinate system only. The evaluation functions for KITTI dataset [here](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/mmdet3d/core/evaluation/kitti_utils) are only applicable to boxes under Camera coordinate system since the rotation is clockwise if viewed from above. For each box related op, we have marked the type of boxes to which we can apply the op. diff --git a/resources/coord_sys_all.png b/resources/coord_sys_all.png new file mode 100644 index 0000000000000000000000000000000000000000..59f650fb2d4bfa91bbe4c519db3a337f9a21598b GIT binary patch literal 60886 zcmd4(c{r8p8#azVNJTW5w<*dz&tuCb88fqt88QzcnX(K?rH~;a^SH1qQ!>xBStK&g zLK(tB5=$ic-Vg21-ur!z_rKpCzxJ_@RLk?+_cff?d7amNi_p_iqat$X6e>mZ!q^$%&6|rZ2-XsNoCcCX}<_$qq zKJfoUGmaPULeRRK#x*6QyH;z{r+zXc>kl`h7BvU>8PD(xKsBMxug4W7F7fP~Z_K^c zyz?!VXNT(67^{li*f@@nz{huoQfjSzMl!6~_Kj8V;rEP;#~E6VeoJd96B*)5TOrN8 z{Cn?vzkvVJoqENW()Z4t0*ioukzJC7cL9E}X>h0!9e>FYvA}cuRVc?ZW_Z2eR~$!t z-tkvZ{IxqY$6qSOtH%%@f2mH@rT_ciHU9tVp^A2enJBD09+UUs-p^CWQ;?05Q@U69 zRPgudN*Q>^?O_zB`YRIoO&q)0HB_QCN()aec5!E=|5<50T57SBBy1h>Mv4RqSE6r; z6fL@;BlGED(EF~cOhl(vpgVu0aO4A2bY-7{{JolsxW0lZ^J+?*t(RHFM!I)zR~9L> zERMTBQf$)b(4Azdm$iE6YUe)FBwO5vtR+$WY>n^yxd~J!lT-FS@R%wgq2n(pDN#HT463%mRU5~ws5AE*ZqNHc4E4q zM?y^}>yR71k_h^t;afE_5qS|OXW<)Ik&#afISf<0PRBfx6=W-BH(kr9$5^4|A~2IW zz^5X&lCsxwouVaga1*`=bZFw*3djsN#n%HADh#8Y5rz0Id~+1NAQ5z6=VdqfsAP^* z6ynKqgg&EPY%9{)EU$Uet}dFZf{+L=?Own%?^NY%;7%0H=~JWD&8Si)lQgnS+VJic zG$h~=IezgY158r4-o6;EB+w7E5chl9%Y@2qY6#Gct>@mCiy-6{{k^K!YnOJ$-E^{v@~kLZ*HyhN5}wO#L_HCnwiV-nRWj#N zG5$r2x1=PlsWiL2Gf^UB$`QR(we~tId0^p@O~;gnv+WbUchda@K@1t2@okj|nzQwK zjSs#UDH=)U+gCm2LB${0wm{p-H85WjLtu4pI7s?gVqsnGHl?I{8Cs6DE!WSIdNOxi zRu*ejh^u)~o$$`E5wo6mffV(@N_o|6U$dg!_EGg%q*swp%yM;!HZtf?EV`tB)wJi; z042`Fd7CxTa;Jq4wf@cNzG3?O}d##?!Kex z6ewHvU0rUjj*qKY(Gq7yah)``sgV1Ry-nYiQWuEtYi1W*glFagqaKzQk;bp8zv=4kl2V`C3 z)20>Mq0)&32@emK)$0b0hDm;G*u~!rf@y)QxE+r3fhybRaIMDv^KY z7ScVxVXWqRY2>+($Zg*@OKEeZ;oN*QK@_X7q~1#A`<{AF`o@mM@NcsvJFjc|u1D zfgG74@grwEui!0XI4jJJ}`0#ppG8n@4y+Esk-81OO1s)M@T!%5WE5+oG(U+C49ld+B?BMqoN|JY{ zUinP@5jZ+mE@-qN$ej7*n8%gk-DAU4@+{u1$LNqYy0_z=&ykP1}|;NPev< z5n_vEd@e;)*Q_)BgN`R^Llkzrl*<%wX@Ojmi)ERrspzJR;Ms9EX_xeg20r9gxOr@{ z94dzKF{)*>RJJJ<>!^^C)1-nXyp_A?WRuVP$?o?? zeiSf>bK?XmW~zE*$#*ZylJ`VNG9tP3vxTMJMw*BSh87Q7vJ1`#&dgK_h4S&5hm>ke zO)OM2d?$vsvm|FyhZkM#X{lep!LU=+VOAlnH`1_k7dcq?H znK448yiz;(=hS&f1XAFod1z%v&M#!B;7NoV2K$_(?AK`E^YZ6()Gv3K&&=V*_$xcP z2>nFRb{a0ZA^e4Y2~X#_D3Zm+j?Y9H!!rCG@8&S!mw;XEROptr&3!Angs4UlR?IWj zV|Pg*ftZnb>_+K(1GQNCLe~oPP1%z)31tSi63i@0@y%GO><(6btCGsPqTDe-uDjKl zHN_16w*psftD22B5yPrg?lrp>BONCU)b7wElaN+xB!u%T@|v1m)*&hva$czxc%2xP zM4w3FA^Hs4x=Qu3hGo(*L(!D2I<&-~MXs2zCQ2*>wHGpozoQh(+s#pZF?TIMsNXi7 zzw`c_TQR%aruw!*B=6|$VK<8pyo25)7fS9G*OaTTu6hqQd+u%Cxh1ue$rqV1IqI@O z2_qU5cW|Z!_bh^>6VD2@l%gq>P@;X>V58Qt-Bz z)YJU=IMeWHB~rAQL9IIYM~`{Ip2G2vHqEt7AD=fe-MlQ0*=rHA$(_#`SSdvP$xFgL?Nvd7ZI3|DC(=$gH_S>2#hi;YMSf zQ-nHVY|;uhR-K!yAJWGaOY72XJ6D3XwYwRpLWU%)$+@@M6g9DRJ3<3 zl&DhC{A~I$^<$GIr(pce14B_9YWIo2hK~M8FBW9J&EFIg4`}5{P3k-J+&iDoemnkN zdGJw+I0AzpJu2>)Q$GFw9r@&yk^OP;aPuPl^Yy7(r|!`*%d~@Lm!X2S4MB?<`+MbP zm0bIQoiQx*SC*Q#db^WK&8zwL16M~&CuW(XJ$imwwT1-me6Mkzde!=UCU9fTbKtPL zXU1d|?};#33nonFejl9W+i98PIw1EE_s{elGZ-s-xf|eG5dq! z$JC6;c5S3Wqv)wpsL?o&R-F<+|A*$;j^muw!n9`f1Q$W zn9U&HKiOIO34C~9_NUZODbb%&5R^SB<4OX>otH$HvY&-s>+N;OUw?&-lUd3Q|Ia5Y zPwT|Ewyk#)L2Pt%MroI3h#=bqrj3%E(yN#5bHx7lv(BuXZsV#%(0S?%j`1!n>UMcZ z@oCUJ3j~Q8%#Bm^wS-v&uzQ|`6a}96L!mU7u^0jcF?cY0hS@SP38Wq$?mMnW4ME}W zq-R78h@ta&anGpU=oAlpw#8~Go+7nljX@*7Ph z5Lo-{M~x;DsQvBn8sbtvF7TMp5kaEgqOFgf|HK-Cv|yGShlWEapLdQ0VpDC|jCn={ zg);2LJm(?zSiEqB2;yY}i#2$SQv_DHI!CX64txh;bnI^cxGWJ`FR$ zxYRt67OZXV|G7gr_%sdwuMJMUa=2~`K@4K>B1oS;2TcqQTKtTq4FqXinjnMJ?|U}c zp-+GXvR~#5%VTYm02}`1{`0~H{Cl+jjVmW0G?$Dlp6M15v`jL4^TE+87`VPl~M z%^!`e-aagZTcf+DAa$Z+d#&_brDuQ?#QyKiKYIH7bPQ^$*0T@fy6EOVuYni# z@7L_Az$SjOOA5c`?dy*`^&pvG!zAyX)cwbvii|o6dkZDcGrT(f@F3tv=x>;swFK2q zH3fM8a{lwy_{$vssfV~T^$c%0ufYyg6A|JTA1v=lLmVlgCE5IMpTZMNt|;FJ&xhZgu_R{ZLCJr%LIq4|Sr}Zhl+pF<3$D zHCxMa{eh(w!#hS(B{Lfkgzo1Ug!$ki zY3Z3S;$GQx#srm46a||ms!M1>#?gLN51mAdOw}S(6E(KY7c=uT<;*vLLtKvK<*xdCIIb5%L5<~U(0XaV02^ZO}A_qR-Z7Wpzj^tHU%@b;f_vIJnrziS7 z6Zl$dFyGSs%k}MOchS)4Y>OeCs&_C#F#uEqkTD`r)q*|c!j#IP){Rh3bROO>wk+OR zyhw*o>%kDa!RLoQz*QVaGDHgFWG> zKP@+^Bg7>eJ=x|PYu4B}{-wIccVyp4a7+xCjk(nDR^ep?`(=^`M}Rz>V!Vg6WZ}1x zP3vg*WHbRS0QU~LVPaJVWq%(MJFrfCMU~4(4w+j?{#2@pVIA`$S^o2H%<;<) zP#N(v?9(TdjOw8{Rj1^Phm%S)g+?oO)~a-rRLc=X?yglao_B`jPK{K%5t1XTn;W%a zegr*c$mH7B%m4`6|Hwt&NA@&N&WiV957i6bk~@Od0k5+4Zg?a%QWfI9j~-Uv$ahU! z!%tU1q}fcV@|wR<)bAI$&9w+UCY@GL#DiAUEe!QA850r}PV4GfymKWi^6QABH&6EZ zMwlZCY~7vX^OiN=sz-dp6!P=9f5@QR)Mq<=uwEMPZjR@ZFQ({jTPUVUNb+4H1jXke zryNZNu&ik`G?0&OXVy^)ZsmGEZ1B#kS^n#lYQ=N4Ih(tn#9ihL&ETELwZn9;@ER>i z7kwW|s@73bAkM zBRgrYh7&0@t8^@oHBS9HQ5aO7Zst&unF)+B(x5^|6WP5^!1CEn)nFrMGYg4rehrf- z=JUpwKwH^onztB{xhINxv#_dlHZovAp`rwuk-e-K_UX7He%I?#8fz2JJr@1&5($&} zD!!|x@^!9E?!5yIE9>uK&E)Iw!Pq3Xly@Acn96D;Y|rE) zgv?|-?E)&0NG?ZIe6DM`Z?G0muU3?#Utc7aAThgyMtzukpD_74VRRsHw7GWjla#w< z{lP8wnYkcz!sJ_Icemjja{AfK7fJ&H!(dKuc3t@{8~=k8%ot|L#O&X zj)@g~6LQ$NVym&Rt6ufZR>t-AT3mEO?;m5rU~-nP7{{j5o)Nl#YL7#fND6He|K6^l5J_)Z5ZQa?l1IU z_ZWQU{nBF|DWo5oZFFfkFuHXoQD+$dB=(I>oa)LQtLDH)&Y9P<+*)6=9q=0wq;D>x z%KC8@JNLZ0c-Su9FJ+9((46d=(h#pDwg{q%Y)*i|88< zE%HkikzE5=_kmXp{HEkIC|3XCQ*_F3z_+ZDEXVezsq4t^l!jpVhkCECH94AJ9yBGX zZTirA-{(ut7IjABxSO-t2+ll@;1lxj6YmB$#03)*zaGq~$&xMlooBWW@q0O(7e-t( zvx>*h(0-+5<|M0eDZ)#-=D(lGLz+B%ip29FmHW0oVEN5d{OI=@2m)S3-}M}d3CI%B z++rI#BLP{gZFg1l&`g_^n}IT^qQ6mA5r9P5(A|eK*X~SKSWEWwQ)QxVF*fp8jHDIb z&D2NxMs@q@puNvzG&-mB3UAMtJ1%k5G(|?g<+Wny5ZQ&noy|esZ>`QIQ-AuYi}M=s zWZd%B`9`*u%bbybt75Mh#f4+7@{B)A{wx?rww7sKk1(-I`z{MjIuPhWZ*J8`pgHN%k{V<@G`hL+ zl;0r~i-!>kyt3xLIi}5Sc-G3TKbpD1ands`fAVaYD3T8w8tu2BGAWzCS5ZY)#M%t+ zBrvEv+fa|l1prB{LyP9#q2_jbYDD?|yXf^h!{H4?rp&W)Pw$>;FC2FpekahZ7K|rf zy+5%{T#})t#rWBI%Ib3(bM*xw|}XqURZ<3=mCo@gvxu32|z>Cz8;y5MXMKrfDwK!v=echFfu+|F-rZ}W8`_RB&+ z*fz!D3ArlymiOA`zT)9xh_Y~s2z@pM@L(Zivuf$?4*8&Wf&SC6`q0r*O|8)i^a48I zt$Sm)qjn7bL*df}9?&PmF>+o*e&K7Fh`bzYpnytRT*%{bN0zrC9=%rqPy!-qtO5k55NbE=YQ zxLZ%7cIrNhjjZyLxos7=)uSFxM)QJ%T(1}#@^g9Hnlut^Q?Z1ktXKO0dx6;UTIGe4 z4K^p)LXPW*+gRCq^1(dCT;ijqXcPY}9T$$(UJ&FKBo1 z67s%&2XCE5GJ<+K%BiYSo=<>4rjo)f5WbU^-&mxNN94R*0n$;)@r?MHiA})#KXZnA z({XYqI#W7y4Y-g^T;kLwFeB)ne7Z|?bvZnkQKvljkc1)pxSvd*6Zi5;mGL4zFMb@# z{NS4#MGZ&#gDzHbsEtq~o3%u>TK50E%qe1z=fuwB%5e<`(m))$0t3?HwlcfxMD0Db zXY+bV=4iV&-L{^Df7On`j$y*_^p8K=nLe?8olJEo$0{+))3!pC{EaRK-?nxW0&6DuYDPlbE6Cl@4d1ykjM5R0^LO9$JMT8d5Ur23&}TyJZT{$M`nQLN>^u0> z&vENOwx2B?U)Tt}>(v|fyyEb(ujUgxLw^AC)>C2(6%5%cb8xi0m(L%h@ntVd`D;DQ3(R}* z2Cw;!Al}d%a@pB=feYe!IYexZ+i0S}H?n68fE{G4L}=ic*>N~HW3zj8(jSNbTmB+N z|8ZKBHyygi|Ii72B3zM-YW+K2_T7~{sDMo2`hvZ<<1jqr$lz)d$sA7hOh5DHSy4mb0c5eUgRjEAXj~F7e=n7& zIQWch4d_EkAO?kAvpBKYyPcb1$-L~?0~0cX@F5ZxvLgXuSUaN(?=sg;rq~m-Dgugt z@X?14vNH3Yj3fzKyb88d#-~WrhsJ)ayrv^Hg!4Sd=GfxP@Y|424=(cRZ8)bn`Hae* zB5D@|@OAt7jS5>8<2Y8GDhT}$KrYyFwk${mRYuKXRAyQ%(aJvt1nR+ScoUsxZ1Xa6 zC!KsRqy?zn_j%MVoXhIi0prI2HI4eI<2bmiaES}F>8&<02t+1P58tl9vxN6Pn^PoD zQao8;9Eh5y=e0ZR)GK;{`J!2Vke2p@==6nuB4Ac_KrpqD*7@&RT2$mVnl5Jiq2+AhA>j}eb|@MGhR*E z8$K12pZ`F{gKRtEQWlMV?yv1gkYmR@G2Psm>I1#1J-&L{(*)62yb`Rd@(8D7+1`wRL|EVBW>6L5_N9fj1 z4`b?%C8vC{6OH1Fus$l;39Tcj0<~s1K(R?C@3<{|qT+Csg*bpN6za)ilD~ z8(_7Nv~PxXBn%P&PSQ8 zq`mR_HVV@ISA4FtTado~3PTU5hcWa5@n6)-4|EY~D{m-XKR#w8wXP%B62RVNh>imF zIB^X+e#M&d6y0oFkzC?|damDo&M{sPHOs2lGc2x@&x^mpX2skJqaO*KQrnX61#@3Q zI3?4!e5n5htUXm;vmE$@)%m}W>Jkq-7G=KA3cxQ}<)4j+9!qZjLe!>}xyl_`%A(mC zcMIUZpT}lz_)EF=yM))=iK8SxMnLtHjbDB0WIYKU4fb14#STb9RsIE6HIDC7K`-!+ zF<)PC2~iyBtL&vd4ROJA#%vA*dS%#le?JsQb&N2fpFzG8ZG1dN3QPT1AwYzr?TACA{uLiC zJTJ&YKh4AG4+oke{cxa}w5GmgPkIg1@KN3%mj|S^Nyu^5{I`?7CdUGXDGoDy+?5gU?lUbbMO|o;U$|Q=K zPy9jV0z3e1_f?nOpQ(~=uTU@AuY?AW{j1zIcw_&uCbRJA{HU(U$Hpb5;F-{8t|RO{ ziuC^n_MX>lcav!*r_Uj3zw{64rexGf|2u$708BQkAZO!kPKkIPbya;-k}sZ+4=Vp3 z>vYVocSVljgCag{L!Si~f?_ML^*jv+r>LF~!Nh@CIQbEb2CB`^kI8Z9-f_|OQF&5) z^>@*I!2O>%3*!9)8>~fG14nn@+VKCZx|V~F>^O(2W8KGJvRjiwUSu)xC#H^TAxd_a zV*g*kp{n`%F;y=~?s1WXaU0G55CqQ0Hq)EBI!y6HpIiH=)7Cw#;kBL{D{OafaQ<$GoC0|oE zc9&`X%uE{Ea{BKGxu8S+qEl|^Q&C|i9zN*wvpmCP@0COMs{_05)BgTyEWXd>VX0e< zeq*h&&kq!c)e)(iztsJRS3nlS8Okh-$+ODUOm{o%x5N`=>mIAF;V$SX>cWYY$YLEH zpApFp{w?mTpgT!lOL7;BP0D+Jm2;qfjfUhdZAa8-PX%=R;k0l)j`5hK|F*%>96KI! zkQ=PAP7f+8L;T-tYD1U%LK02&hH%vqCO&9O>k7VEV9%^3VC(amc#(-q*0MYnzld$e zpz7k;v=cU7D*dZ4V0mtqp^VlUqC-eRTgd9`=!23N#6IzYbjqv8qA`j6na-Nm%5Kz) z9TrQr7Gb1URkZm}$96W`fy6MqZcr&{H@laWX@e$i7sTu zOk&oO4_kNT7BAtwu=Q%b-dqFkF>((*9``Z`v#nN%F0{aKC3GZJgoCCLAG_>&@4BWvUXts5pNqzzZJ&OrXO|wWdn~84AbZjG8ml5H z``MbtI`_pfgIwb*%MG`P{yTlfQ(OlYomQ3j{WrHKLWZAj{D6KunD_T<;ITI(5%qLP zA3qf3XE_`VA!AuT30ZC(P0l!xW>|ov&U+R1S&Vc$)71lSlarVeHT<8B*=c($qpy9g znr-BhbzKu*4Lr}OBzVEKYw%9ks8#vTy|wDdDR64!TP|0Z{=Rz^i~3{FuN;71Vk6Hb zYZ3CmykL6B!v8lL~USW{X_>HTrYs7BI$~&H`Yd{R>wkJGAKbfsnn146lIiQi$h;a?9?ycew zhuCy{y+|6dnvv$8xDYNcXPI0odr4d?cR9w0Z1j&thvM_?MJXEo{zkbHMhTgD1|xw+ zWXhDhSsn3@7Z+S71iJLyW0S_*A9G%bGAUWL;+K#$C!{iz!R#1tD*aArL*}a%NCsmj z?%BJA!HT3SuL4x7FZM?}yf%{!@$!ep#ly;0Ufpz52-FcrQ=}P*J z0%(fYWqtkM)PmVdz1eqj4z*8h-=(>Mv2tTPbt2-*JT67xUe2C#wl(NH3o&4Sx4Z{F zxZ*NPN;Q{MX4>nl*xCZkp(V3#w)1d+i`pj(JosBMJZV?N zLYs5DggkG$>oGKE=D@T$PKX7xx+U`&vf)JS$3!S5oU~R{f?@O1wSZT1{Z&GbbM`pY z1j#V0c_me(3ZYwEGZLST`Vik`UN8oRp?f-AJ#%if8#dNK(Q|?0OCzhxhe$HL zRZIL9;!;S>MMO9Nl*VBs7dgBz)@`kSMjiNe{AG;9@T0<_wK+{gm7lunW zrC?R4k4kxl*()!nqL|^!7Yv6J^3E$VMERGpZxsgLkgYGMN?yatAD~-W(y?jkm(D(Y zu6})Ue8a8tY_{&L=NNL0!H|ks2RgVxW0#*fU&NmCPJdP-tGu_^|8q%Or@MNiXZeuA zkbe&QclB6CNP;+2;&!jtL=zmUUtwTfd6Hvys$UwnfT4{?ADkC6xFmyjIc3AGo`5{T zP*GeK+|-yy$vM`_SjKv_tKBRxb0-=Vjo-&Gy2gG24EUI26?NWt(6$6Kw@WOeAvtooD`O>%|nRT>YWeW#q|2PvWy)9P{aN_r#y`N^B@$@(k7{-piSBjKa zPDZ*!%AI>c-|2~1B@zeQ1&*3$8NBIx`O#IE!Z>EjlDKF0furP}87QGjh}A7;18cUC-brEjn1I_=&IjtJ^CN8DO|#rjb_uab zEI8inL<(L^a(O#rdygyL`7zAqugPJUi0)~|#M7{3eoT!cm$WQ8+}W;!=75ltZ?!}u zI=?$@mgRXVVyEIT@9h4`kFj;KT5lxTX4F zB(k;n%QCjFLTnLU|+terwnC>+oK!Q$1F_O_04lu zdc{fDvIaWPvTd0V9=|1?cskVvJl>#;iHJg1JwsLdNlHGARw6|;EB%KFT85J>-OPQ8 zj0SQ}ya2s`jd(8#7k#Qd>FAFyDr}(Ekzqe{Yj~d~?bHc6o7HcezqT0xt@7hHDeHN$ zyEXkK8SeVZ$VSp)KFPXhawgLS-Vos+V>tU;2Tk-&67#r3wh7BC&ti;LRoRBY0r<5T zIXvcPx#QJ@yd;JZw}%{RDr5%Dqq0ls3BpRZwbdV`mKlWva$Y0Iu6PkqM12`88KePz zF^TLf?=`E7;AoegVP{F7vN_$Wu)ii?CM%2*=A@%n($Hq9Q~-smJkz9LvHOMaMWZkY zSoPAzFb!jX7fu+mnErSQf=LlTSmIx@*Ui?XpBd$qVk?SO(^0EyE&E?4mcYpfvyJdm z9SZi(o>8r+S9r94#q?6HYfKFP;gtT2MO^c(P?+Z(BT+-ISA4R3sEoLqzk;a^3iYXB z*s!7)jsvO`ST%S@cu227L3@%yd9R&qQv^Cb%Qr%waHqm0rbMaaS`lMOaISll?^|2f z&g<`XOGQs?X~poJ6a=d8E+~$+xufORV^zu*q0|pIM}l?gVby>~Rup(Hn_A)da>nzTQp6{?{+w>ufn^wtZy#v$Me@%a=!5 z6XGA$WYL51&%J;NjwJPxhP#ECXcS^KzsT3;lg>T@oux& zJE_L=Qc<`O(hhrO(oym=9wRzlJF8DQ309=#aQOJo9ZHHfRSOtSR`+|tXCq7wN;|81 zB{bRadyCJ=Q68ciMTe`Bg0e*f+S%s})tyn>L5PA}cYcp@tI9fe0I{}Fmg$wdrZERK zO+u;>?j{KvHg5sW1V`|x_vzi+uL-;XEf=!L``I6XK)F$jwPfVB>{L_4LQ{O6>v`-2 z8Qvnd=%yAPDVtTs%wUFB%dlGh?Cg44uv|f?ndzO|jMqJ|GjyDE>xbH=-zsBpJgbk2 z81%K!=1+w>jV!FbJag?DvHjV9SgjRHn~)+!@46T09M4?6k^l3T=-`rG3aKQ!Bi*Zc zLLwtoYTnf>x4|x1(0z8E?=ySX&=9cz=ylSRzZ-6})LbfpGDUc2S#-3HpI*>p{-unO zKIJt@#*rt2QYQ%wt|u(!jaQA_U&N=l@|cqtu9}iaCbCEEn27}a9DSl~c?H9dlgYk& zlTd?q#YNkOc~MC^#l7NbwQ-!`Tm<<_3=W9@MN!AH`GF`i$_s~IBNj72> znbOV}pd#mA$$nqc%G}oQp2Y)$X!GV8g~yD4@r}+`CRbIpW0F=St70NEohb3&a9doI zeJUUrqx~n1X(=wYN`@;E%JFktRb}hnaaI{a<6AUm@LEx)b{H3vUA+o_UMV%P@+j9Q z1QaJpN9gY;Xt%a095M78)UM+(VD>|z?!Z%XYa%{4^T^lS$AIJJyd>aYIRh45a7VeU z2Q|z?RCLHshEX*h3NJ+5GjHo*3~(5IX_z-c6^9GYPBLk(X|~$jGHV-Bh^qLsxj0ra zmOT8mDtse1TmavU3>5=mdLKWFm1~I^nZiZc4Vm6b-c}fzyWgxTTfisf91sxV>Yin* zzxmfBdL3h~;GWiQyNvEj3j3Iii6BaNGnF~X6o&o*{Mh$#~}C2|hT8&>*5 ziiq2_2pd(qH)*IPF1A=YqXtRZ*6A6|u6H0`8Zy$qJvYp6 z+bo_aA}CazcMXr8cQC*5a)5l%G@t3`N_E<;2cAz~xmeyx!+M~c=`R$=S5%O>%#w~h zlp9$7_K@NrpG2M0ackL&C_C$T&Qi{vUirL8VbBTxgDc*(9m&bkJ&ZBT>!G^8sJhIt zlkS4hDg9*T*jGNO)X7>k zA`pnO6V3rXVfbMAliTA}$qpLr*dJDX)mB-H-$m(SY0gA$IEY^Lj=>)qZ!0W*e>GnI zosUaSa1(2A!=t>^eNgU4W6zu^a!!KtjbzRNq^`TxdeDRdAjhqp_enhQ8&edh?3k%q$dUZ`u4L1sIs=!fgnSq4o5Jk=4dM3&E~Ase$WZpm zgQgqj=wDc*w9hHa`rCAO=Q|J1M`n!jS7UiDUJWI6E9iQ z{Jg~Iz*+RGC~t~kwAPeyWlEzMs;vL; zE~4{Q1A8>eQxF)>Jql5t{<`X9DW@+k3F&@)6#$qX&Y(EA76rXO>Im7 zV@k28cE!n|fbVrszwU8qHYovE4v}y+rM5l zHEET*X&BK;s&;FPV!t1qxZ%`W_mUJVj;}9GLP&UG>L~hoaDBHeIm7MS^w{BIiCTCM=@il$w%NqKOow-%H zH3H4(grAVDLf^O;E^zkdp_=r#bm#<+k~wRD$eE}Md<{r1?E##OaOS9y%bx5fCR`kd*9Dwb`h_4+uk;w4EMMSK;R-l zn1xkE-F&z;T*w{;jt%c7$0V<+Le7TAG%`6Pu(7iEGn0c10@^HXv&C*fYXYP5(u-;8 zuDz0p3f=}P@Cb6q{U~8MiL&KXr;4^1KrgFu6Lbrbd;w z^1~~E)2E>=svFb>IR(X@t-%$Yr2c-SBGDeB1;^}uHr5)vUkLP2=m*Dk783!y7wJKZ zF4TQvL`n8#jXe809CpDv&~0jsZAygN(7;l|1t{n@3S@;9QRPn3ajL@v9E$wm(`$hC zreh-f1=dR+jIinli46Pih)*}g^sFZrw469H^dm5(2L2m)an!Fw1bjViSXxG-Ara5x z_VUNqDSrK%2qA`FB(EQPvWz~KJNNq5kGntdO)699Ee4?`ur}f0o3l{pv)_K%T`%o? z^ota_*btEhpY#L^mk+6r;P?!@E(#JQeG2+JvMqc>1b6Ceco-oFQ@QM&bO~6ZWp!vX z%;R6-Z()Dr2|c`*PPsQ5Wz?0Uo-J8=_U>5WJn8gY?MV@iqnU`?)r|2XHbfM80cdvm zPyVJx`T&kV6Cc(8;^5*zTMH3%`FLY=gelPZDl$n(KJJVD0Jo1nZvO-kFAxdT7CD(( zA`uQ)Uvax?N#tbeol18>s}}IC=0ZmLO1~E;L(R^E^Bn|sqU;1;_qG!5_18@zU~j>o zjn5Wj_hkeuSZc#Y6Yvf9K^z?|)F!7u%(a8#o<4Nq_`s6Y*73$oC1ww@K#yi>4+1OrII!F@Tbl`WrWLlN zAcw22(%&#j1y@Wq&4-Wj;S@}vK7Kd}T6u7a9dxZxv6(?GsU{*L(>FlK*bKm%r%QzN zt{QEC{JB64O1PcA?`6n`9j%kc)!y^@f)@B(Hh*fTDNsS*&!^{WA?UX=VXJ~uCUjm+ zHln+wL=+nER4!Go`G-+lX24N^Ek_;)m}Km+Zxw!b!~tUTl?(WWV3m+>SJ}lmsU&K<3n<@ z5d2&X?>~w`DJ|;-ncNGfk=oJKjFn_HIGKN|D8nOOAVs5WTmhOh^pa#>|DFHfhz=ZZ z!IH5D7rku%NeuVH9ar^bzG^E&ActgMXZ(g2UrPisQ}OCN3>JO=lQ%YDN&W^k{zP2D zYl&(Wtub)9syXvB6Ktav_#_BkjOevCho}V5^3+}cKlNA~@1gn+*iSYy!^#;}ctyrZ z5HN$d8*om6Ig%xd@bblffWXM}Pq6w@8>~HsP-pY@Txb9&6mTF+0!_s5FO6~-Nu7?4 zHw+FS!L}>N%E*fJhDJI(t=GKXWLOgQB%lKJwzuIx!_Vz50ulJpe>3N=IBB&Hlcx*( zoJ?TKCJ&1_HPn!t6DW2Jd?3j3)&Yvq@g4`+)dxl4{i}N*bzh#w`eOn^B_0BKiPsA~ z2J6e<{*BxIU8Q@+xB#v*Nq&OQ0)=fqJOMY{7s~0_U zw0hcaQx>sj!q*^Enl%2GuuxWB69eNfd`ICw_jE!t_+#vJTbPXeld3x?}uqHJ%v z5h4@|_y%@Zw3~#}UHCZ8ZY8nA!h)d9%vGHvN)TdvlQLH4PyJ)@;VyRWCobP83nq2p zdrFAM;L~*PrSDP!AbG?E0fEQ%=nw}ObQ zIGXo^vrAG?c>B?bHEs2;S|&p_4d7JP8PpAU#&-TPY!fqT2z;J5m2g)n=wCS^IlT$@ z3$ZXT!I0M7B_{o>U2nIe(JY3(JpvXBcj0?mWn_}~(By+d9LeUfXW;VX8N_q~&eE37 z#$Gd@rd3&ZHE9lqOHBw$K=B|7%DJ{eQV}EdAkC z<$>UVKEhXJWl+Hu6QGY=@# z0lhdZB=TOkK`o|r+0yq-)m^7CGwwFh(6T+L({LiBvTpqTeBIq-SV)4x`Hq#!5l>wJ z>=j>Gq}1W+xc~=017CPo-dWhR91C++x3!Pvp2lYZ=O_VpAuT!-h-{qUrxtL+o)+l< zlCoAY6A>HNaj`I-+4P~xZw2Ygb`o2NDGrs;k~1?#WLx>7(t!b<1kF#Q=SkcC@}vZ; z2Hd+Dd)P48+Z6A@chc~fPu5lPhcfIQuN;g<;FT{OYdim1`HKTsR8AdgYFa)uySvz^ z5c&&qRGSU{2HD@(Ry7l*9!qL?L?Qv~h@=RyEt}rdX~b<&X2Om{l0hH`jE>HJqw}yH zCqhk+EuOM|m(TUaZ7NbNr#5M%dOY##AaGv#InNwdykhmY&9F~L<@$Kfj`Er0KZd>;Jq3m_!njnJ+(mH7ZsEENln1-G0 zb%!V-jt)en>Y03~$Df7Xi
wKrO>?DzlbPLE!eFZKkV7Jdr`XtL z)>CP!{|bP9l-0EY!h!DCIVgZdxPKf)3XUUnOk8BNDjqa?w^=5{tC`bo49=FpqiprZ zfCP)2IN~M%VNZdJpS+#&1RtvkTbzyt7ddA^ZkmWoGA%tObOnBD@ns^L(?<{mt{O@# zv2C6GQFJ|i@X5*0U*KuRwRXGzs@;^FpL@ZL!j4)c#U>_xZVinXV#Rv^8dXmgz)d|k zX$r$AxNtehQj7!~WBpw`8eG84Zv8(J)b%Vkd?E1ZU6a@qGRQ^-UI_TnDB&D1)}Mgy z7}~2R=O2$3j+AuYG2Jge$)G3E;SY-8dmO(nj$UE`Pe1NaNa4Q^J+B#8r(_Z4am12I!}#mp(LKOtowHK=-e?&pw-v( zVV4O$&vIN_=|)I3GhHpg-BmCa1*o$7g$WWdg1Hr6s40&bRcfeisB|W+R+~-KV$0#P9sJtLQT|(Utk*iPPUDC< z1=GG1`*JmhQ)Qtx8m_(Eanrs?1Tnz<7L7c2#vzfS)^T)pB|oPayi=@II^KZn=F-nq zSv4k98J*K6WE*}nx@AeZk@#)iVNsU08%L$M6R=$#uZ+?@C>;|#>o&UTVzw!1^e%!U zu$g^2+XbAWG1Ev48RFXa#<rf8 zqLS5Y!st}m=c6)tFBs`?jaDq_GKP5&3OHpGCHS~v1U{J+EH37BC5Z^OW+@aVy`WY= z=?V(pJ7;c8$m~!sneJa^fN$aRYisYv*ov>*<=Lq-8klv!zxlwCgE=tH*Ri}ajFPl4 z82x3;%WclhX)5^4t=tfYvNHjV=aUSW+tKVBK>$p^F}bmaN0$+x?TCyuNkNq)#i|8e zFNPlyJ0HLq4Y{Nq-d78{d%jU!A1-l5R5hgwV6R_t0r%pxNB?-0gOgt;L3XuYS`9E8 zSv8CuA)J)d7NRp+PtY`Q%@Vp8IC`dfywUe&gWBMx2Ldv+euYGuSilEo$kA{&@ z`~b%v_5wL`^25gE)3^5G>75HovD~_OYWCt<#@XuhP;|Ba}gaCuW$O( zEg-=zuL~E34IZ4S?HLucroc><<6;Um_T8s9!F>vYU`*Jj`Tuoef5CUsNueJ)5wQhb z=dRaactj_R5;(f6s0XkM7*pYvxqj>GsY$ct9`g8jb@KwTh*ELhE}8_cta@kXO`Uzk zfIl(t>5_d+c)hOO{wH)4DRLi;@HF7Xg4c3`)kOdXfyGL5-?X=p3;Iq8ANK%*Us^?h z=TMPm=5yXX3`Yc~)^lc8Irb@V)nVu?^?`2)9+;{3CYl>ZG)UNM5|I?GHYanO5!cNUNm2q;(6Lp1W zvkcv&(6}SH?ya54ZLZsw=2MaFnw9SNcwY>lyx#vu*_#JK-L`$>gQ!S$ifok%A(cIm zHCwW09ZO{k*_!NBWGQ67Ft)L;$<|nAROBkzvSypCW1p;J_dCC%?)$o*=Y8Ji{r!!Z z(|I1}v3!ot@i{v{@VnYu^gS6MU{(t zU*oRm+JB6Zo$4@6RC+g=$bT_st%l2G(1w1`Xte)g`JS5Eokc;d*o2C9DTj`tAzbUw zh@o5M+!idFz4xU6gi?aq5CWXYZ->O3i)B2Y5Y2zlznCp0Mj`^Kw&SYC;JOE~DC)Y3C?8J|OdvR3xd&yPW)Xc38lv}nq8(15a<#{HX!N>v! zKo0P)rTy{q&l{aJ9M?D{ZnYyH%OT35ReO&op}uF|TysEB#~%A; zLqpa=tfh3{rA+6}c7#(6W5(b{;oeYWNv^ClNHYx+$sa6$$Up)N$Q#>>`%}Cts?@%m zZOCOb*~Z4)Ff7KK8hzp9LxzV9^A1$n>lr4i%Z?h8k0kP4EMlDBK(cCxS`{5Q;woAy z)7fIvz3Vs41e^T~=1o^Bc70JTzTQpb@|sYFwk+c*9Bz1u2OFd|EzQq`qZr2gzA@lW2Q+b_+`7;$nIPTX$sP?Ipf3GpQM|- zSO@S3+BD1t6=9A6^KRLb69R? zT;8r}+jcz$u9{G1?A-~jylb5;o_|6NuNL)2@~ zVhwY79tgxGLf^R5qwseBbSa>y!~Ic;2$lv$05DgGAMhR;ngtV4_kXJw{I?V>B<4zY z8}X2~>QFnD(Ps>09{xCcpMId@*A8+gU$vx6km>Rqo)nK7K&WBaAj_fUy_p8`t)+Za z|| z=((LRKJa(11PVBkd8US`4l7-KND>9FPtq}|oRLzB_b@}dJ=iy@iMy;{Y^YFOa4Hs+ zwP=Cy_1f=$^c&3Me&+Zwy6zSbR-qw{+&f+THvQKbgLj>I{rOGFXL>7PgU(I|#e@4L zWR`FKWX$4^2BLp!TT~&hq{N-+wb}4D4F7Ud!>mwK<1Iu)GIjY*@4tSj*DSk^fZ3re zE4W{{55E(cfIr{szBWAG799?Hk{7pY-kUYY$;Db|4;43+a&_aXUkk+Ej7lr>d}uH# z4huCVp=XaF;{hr$7oe2yP!Oeb@eO0Qccg%;<52z)+rO6DOMy@uzj1R+s^{us0ntLG ztiJ98q2u-hbIr&((fMEp#Wg3Hh#_6y*DLhI@s*Y8`D?RPRc2*Ho|UBu^L5{`0)rMp zWjmTCs!>LME?P5a0a$b`DfR%F4P|0B8O#!N8_kDu`Ba-3!weeEuMcF8#(riD^Q@#b zP&3rfWQ|36Yd2VUna{5q@7@496@|?9=)PW`gxRfTFP%lt)-Zhy|G8y>ZS(c_)d*Ja zQfjDjss{*+gwtT$VO;=JP;UKw`#9g;<*~l(dkVT?3VY^v&YQS&*e|GP3K*>2^ez_o zVi)#hrCOiQLujl(ccINVakz?rN_yHmp9mkS&aC%~tu|d)E_8BI=rAA)HuzE^je-$gpAt z!0$hs4Q4P&L=BSeO{tUmPrzjj=i}wb7ObhZ4X4Yj7%IHPr< z3}r2remJOg*AgpR{4lt)vPP6JT{cqs3`s3vNkSyer?7EJ#q9n!!}xeEjuhOBvVU6F z*Nsh<&_Axi#~p8Z`2r8%W~%9O2*VRkiA`Cca-Gq8H=C`~a~!=*3YHFq{)RAX8s#4E`@S?pfJ<~)$X8v&Lx-fCb7(1Picc~3+# zqSW_8E~w-XKmiSAya4Nr8W!4>tCjK&YDYnS_^E`;lNL`t)mDi!)R~iG*(ukWZ+BB( zNAv6J9ky-&hf-_|jDu8=!y2QZs+%98+5zF(-)$YB&jg8p{n2Oo_Wi@ZQ7iVsE0tr^ zu)wiZCwltd;z^)j|7W!WZrR}k*KZU6;=Dl}BhuHB=05)YwM~zah*2-L^Q{J#rA*AA z=~7=OSg6L|l-$BhWU!d**Z|O43d{t+C+};GwRGXNK^HUOiqWaluyW@3$v6$9!2l4n42CiOL&YoKbLP@X);#3{eV2se6fKzYHSaNDo9xlMaU#cGU}-13{;WUtdL}=GPDLW zYk>OF!4#oKPumk>Y=S?1+$3XJ+qr3;9g$i)aNH6DhW#*h=W-ugaokqDZOiIFy%xhr zUu0PF^&!qa#V4tJw>WOk~Lrs#UPN_@Kgq zKb4ddY#WAu5?Tu|z4MIalrTyA?1JMte>RZf3_`ghn6kAK%$?baz0^0bqaP3Si}8-+ zr`|!P#V&8^arNO&Z?O?|-a3RVeWE0kgK-<**#W>k9R|Zf!z79**_G>tN3R3WHab~+ z9MP3MX=h+#IG-RA7-L2m3J=vY6@4&>DVxMYgv|r7EPnGRK(ed&N&`)F2sB8boDyzy ztmK!cJmXw9RlXXuw_f!_Xi|fevVQ~|$`b&xQ>6&%A_j-Nq>z0Qkuxo9b$|^-H zsPn>@!6>|UMH9iElu=|m{OwTr()(b>>(wsop>4~Rx0foK;>LUIwA4q;7ZyD|GHK<_ z7B(1p}qG)eCS0MUMy!bbNw$1+R)g;xz{{F$xK-aSOGY?W(2om@|lu3*}{Ws z%yg%27QrD$-aGgTr>x~28P5KeeL0g@#Ljg*yFIc6z_UTh4l=njURswD@;UR@(SK8A zUzC(`U}2$GxM=yrz_fXzX#5=@UQRk{g9JgqM4ON^iW1ODKg#3$M{>^6$f<>pmHS1f z+$wr)gY3x^2O&^-_AD?8t%1;wwPPDVqjoOL_(;tPMDPPTW30z)r()0a)_RMG<*%K& zFiLed_~}tiHg;@8@=G#$qQha}m+A7F~ouS-SKeq|RTEYFKM$&t_GMN6ZQ6-F5k zcx9>pu~vD0ZtpzG6zH+pckTC;>NziEVxih_yW~x4 zs%v8?!SzeC;+ist=mQhCZs{-iM7zJnX};+2(0;HjbJD*j|Kse2pjb9(F(j2d%Uba$ zCiBVSh}O9g8$_N6ajkq5)rVrf&G~7Pp{TT@!GlT8C#qJ#jX6{m64Fb)aQ~xTMkFQ5 ztIOb)x!93FX=oi@%`Wq^DBV}M<|)J6^_!G$uYpM`kz%f+ffp3!PB^fyeXpm;^iIDb zcRHGNgu#GP-e;N-U59K^p^% zyus=9V4KGZuI1ZzY8+93!NwJLsm5DeUtuiaM}D5mZqLZQCBkrw%)<$=o{Y&lxyB_r z5cxB8IFgFcs&}8 z9%nAf9{}7ULnM?R+6_Pyum|#-31BzC8vH<$Ef?&-GMI(kewIYusCpZDOY6DCS$guB zK~f0GkZp3w2ms}Oi|F<)PJ!&PY7=0#;iM#J!!T7hYi{4yY}Z~ z9-EytT&M&3EbntOKPs>dl)*ee=HdjRq4)8m$cZEW_=rHd!y$||Qie_1dzVmPU`;MO z{%2ZdXf~Apu}=B^!)ERxuRjH}<$tcjVISB2cSr>mV7^*nd-y5fl%q7D>jezuKh^^z zS^r}_N1+pj3<2beZBVzK{681--%Z_|G)E4rd<^@4>0I|}9gPje<^QvZ+U{J)$Y5Xb$G-2z$B@BNZ~o=nvM zj06U>mHvk_BMEN)4`&93&)*#0+JBzeZ$|`}Xx(?MGHDi!z;yvqe?Taq;fQ97D><~^ z6o*%vM4N`xKY!b*1n{uZ;}Z08zYpE_MbGJ;>oa^{|4O@JsqDD_%YDoCV>9aps-nDt z@+r`@oHp(FeQ6Iixe|Y$I#hco1_ukjg_F$;4pupW>~)>-3TMl4$d{cw^kjf$`;-0{ z*&6Mk-)M?@`trYBc~W=p|L2we?#}#AA5OaRz&ri<{xgZ8EIYz5Zc>Xi8M>&Hc1-_w z*8^(;CfgJ>amw!l=GZ@-BfIRapbe6pzD6{u{x_HRf47k~#2v9eCY+?mTz;114FL@^ z_c)+o-hBzY>i_L#A3|6E)k+Qn9jU$~x%Ph=$^ZFczXoje9mzX^P_;unej)m;qL^Uwu4<_VD2_p#bHe+a=b!^p=Uk8gdKd^B4BqXZzSC7|I7zv{N} z!?tb&fs@_QJcj>s+w4-T)5kZV@&SksB#SI!)$qLGtI~HsshB7-%i6+c-HeG4@2M); zcGlh1d!$o$cRqvjFso_w555aMD)KC31C_M2$qN*sLs?a{aXra?ZsX@&{#{zX{p&+QlQM}Bc z=hoB2+NcK*3y@x*Xn-gHHk@BdU~u$QhvH#`&!APm?jRRl?v%61V6$r)7fo`H%8@m1!Nldn2mT+d#rWKtv9(5#IH8 zdfn`d)Z?{-vy{xTx$DfE`alcsuFiPZ zhMw5NSAG3HV#qT?&ng;1s()0M4e(SPnW>!eKB#q4IR&mPa*hi`nLb!AHr#W&-#T4g z^kl@o@Mpv3`^^taOzR)tGb^Qf+qJS%+dbo{Urc6tL%VZ>U8sgwY+CA0;dj%|5VvUe zeAH!>a9qFK=Gd{Ez1yCJbB4X;0tuV?Bc__(a|b55o+Vm&(mIe1vM1uI>ED>!^45yikHfNYgV>Z4jby(FXG6qY4=G zYfAY9BVLQx$QrGAQ(Ite-R&sJnwn}9G-zFPH!O+r-fo}690)++8#h61_mMr@AWXq= zV9Pz~*B2cZJK#>Wi5$`RBPh4j;W05bKDKE@0s~Wy(+q<2gN6^gS$E^o3 zyQXH)c)nKTC$?AKw?`9S->WaK9k3_ptPh-ja#LriSHKg`>)8r$=2pd7F@wN?w}dRA z!KlJP_2w6O_uf?&-T~6jTxAZ#A2w4okGt=kDROveVs@!T&Zq3EP?1SmIFyaz2E$nd zLFnRNf6l&`@8}yU`|y^VZU)oP#nvFT%XS8|!TEar2k&ZI3KO(_g!F-;{VE9^O}gXx zhPpGi_H*Wd4ePuUcQN}Ht7YAsM|}~-Cc!v71d?6-Cd) zZ(vGqS?}DJ+A3(w`*~khJH>*(l2|sMnblHA{ko=(-MOP;gF5}2 z^fbP#c8`uVXIm{zwJ`sx{I+x{|G=@~g8Mf+zmg)|URd|azfYE=H3qYb>UES)nPY#M zm7UF)+;^-hB0e?*oygrS6(l5>Mt9wQVDRmBVFZ$Ny1;mmH81{i7YiK(3pfpn5l-zo z@r9=>)xNCQgexapsgleNNO=od8f-r<3&%~QJE=k@s2xngdt96d4a9bEVm)*Ym8jBX z-r8iN6VcRPBJ#@` zvH${fb5~_c122Hp_k{|5_KFB=eo@fnZDLXHUe+hlqKykFLCVE-NL+R`@#i$Pu)y84 z_42yZkV{R^v0+}RyrW}as@p#q%#;DIm-7jG*O^)ebOf2%!|% zGkxT1&^6JQp-XRQqt;CRE}T4z&!nWLC{53IaeU>1n(?bgBei0JT0dOKGq_rg*IK&X zVWVAC$Qd`6y2+t&mDQ$vymtJ4d603H=4G#fEMi*jdrY-zSy6w_h6CuZ4Bf|{XpTTN zuO2hC@bqW+2o=IsEiClv>u)#ss!A>k=UbFI$QA5YtqW1v85`_$jO(FCY}K?HBC7;1 z@|Tt^es_PgQ)xzU9I|#YV~GgUH5)z8FQarAibxC_yI9U1`Ad|nlvimkJLEHSTzWt) z&?l**!y|As(RxMqKmr1EC>1HckMG!<3h~SzumV|@ItTPk?m+-d=T>+h18L53<)&FS z6{@d%9Y|_u_8lyWviKAOg2k!%`5CVb|K`snNW{HQ=-oj9;5kyJUqdOH3~S)s!sqdU zmn|xFO^Z@{t|y#8_Js7{s-swm-79V^LMbX=w1;l)6-nmejqAI8`Wv3~u3RX0v;#6t z&%kYk5)gSN2-~9$+yJFMlpO!7x`qh4|EMxcL7IP9LxJL^h3_|5@E@g^|K9fhSc+Yk zYzvb9-#{4XDHZbn0fhN??ft)lFuyVOf1?WjQN=?VX#Z=<|6c$r68z#m>VHMS$D9v z9xO%7I66hcfaH`(9m4qOuLZF1joohY{92QiNy2NEe}za$A~wfRr}nox@H7nZx5p(& zx$aOW_rc7td)mY>*-=TvRx9`!^0xKYyOh5Xs9xqgat zc6mR+;K8R5-v^ZscD@r!e<%B2m;*LH<&)JN?922o6B}GG5rF|5I8hOr>qMV#kF@Z*bjZ09TU~@f@=?bSUW|@tv2a;;>k^Ix}QdTmB1@p zdHgwqFPUs7eEL_Fqk&nb#dGaH#tVCQx#m3mq9IKjXpts<;+__q+FuvJ3%mc-C0+JZ z8AGo#Q0;X6h5}iKKi>LBtr(oa=?^qTNM(O3(J=C z%9qz$Bx`f=4}xo)_5Sf!{&r4}&UVjloJ&84Rd2?W>01&}oihqe&&9_+3i-8Kff_J4 zn^+X(--DJJu{UEVG~1hNZ}=75zPDbsrOuMJ+0<40k}Fw6zKkDMYA$m!a2wGajtoFd z$IbRYs3_EtOA?-)m?Q*ni&+|#4x@#QC^i7UINrf*j2h($-yQcQ^A`LFZCoJcwsdZ) zPpNxUK)rh+X6?N*x-&S{=WoT-ox%Q9dhE#RJv5MU4yo5$tb?7~*>i zBOQv3S#6imKr!rgZ@y}@#RMneZ#L^ODn_i|gt=~usJBpo-?AUUeaWzfy;b4wz=tF@btGfsprTeESP3vKt+ zti(V36)f@!PN+z4W3#qm{wD4;C036$drPWtwaod_1pXWwk_A2S<7)xD)H~?{)RjEAdF5zKYbHJ;G`{`N*_5s^7qc z^>WCwzhdl~x2%~@T%?r$h6I{>p#B=kh>9(wDzNc* z4g1@R>Gh5jhTQ{|y@5g|HO_0>#P_NgnES2WQ6QDx$@V;gzxG(J4td?8c&TH8ko(ec z?NcLiSx!c^-)Ce!Y~bJML?2iw%)R=NM(Od0KIj+j;ED6je%o;AkY%1fI5LlgHy9eohDyy${$Jw^j-79haD5s}g5i68-TD|5}y`elX70|;9 zduKsDeR*(Va=i>!Mdq{Cwd&Z&S|D0N#5s!K+vViop@!$#v0?B~gSed~?){&nK)JsY zHP6nXmALXnds?uq>l0Xd0N;G4@ZgIErmI&U&$6AUAbvlY0V=6N`OP3YNyN}lq@Xg3 zL5xU%7v|%$8+)@TCu0Cm1@aW$FCx!~!40cjQOA6Uh1!?~yM`1SKRQOiVs@9?{pR$x z)M?xr$xu#SOYWcN|5vK*Xk(Ra(w@W-m(rYV;+mXTaLB zHoLw$uT(DLzdO-BvgkbFt)lEnTus9dr|-vs-$h_n?dq1i`=MXf%cB473>#=)V7Feo zcRc<4^zT|0&-8-@+tqmQ$xJ#ijp`-=bvto$XPKYD)7z?vn?Abie`kwYwh;aYxXFtT7@fJ%O^2$#&k zkO9=`qo|%)LU8+q$PDws18L~7GM_->run1IS6jLH;Rb?i z z?i#3&X=EaUVuBYVYCkLqxk#1T>Km3lS?VooHwx{cvAD8*Ia!YB9QbB1=vcls|I!#+ zIgt-@N_OPwLE39e+k51h6fuhp%Ol?Payq?~XL>&GKHFYj`$e(vy4)*r%PryO4p;1s zQO1lPI6pwp!P#x>hFDGdjm3B{#AEsBL#H5vQBT2+e zCb%Y3;x(>I6x@Fsg$@JzALbJl-8L)pF~?lF)x@lSZ;3*NrUwsWYsBwqyT4TNk-q`L! zbs(N|fYmE#o~{SrkcN+95}2x+^I$)QTNlPh-t;!(ExaUIzrPOt+4Xw->zniYJ9rw# z44%AgH8q!=lJx*`o`(>AJ%8Q5U(Aw#z*>o0Mx2zkTQe5xFsBa1(KG`CQO*;cYK1>p zx<$v{FDj6Te&w$1o8>OIrH1(q+xvNRqGKnMZi?n;WKztl-%iX?mVSR8-tIKIFf3Lk zRZ4FVDbtRo3y_4{|7@2FxZ&BzF&Zs$<`N~ku3yjAL6#u3 zJ*-I3xudCe2k?<|8qR$q5&bF)s^s%kVoUc67#^@>g_kn=PYG_RANT4(siYtL`m{<( zfs(Q=Bc}$e@+Py)IkCi4?WayuJ>YwJXYrnGi z^cAxc88WfAk_c9@JxctHYnjc%(;Sr#Diwfz7o=xeXsE=JxOlmaQ<4P^ja=@U$-qEb z z2YbFbO$|FE69g4^@0M3d&xXHwevW~%qi;TWee@N*MJ+5@Uxi#|hj&I3%K~Y9S7Ij! zm--4QnPIJn(>=dLXft83%dZf-r4Ef2xS?R- zo&%|SER)O&;fg}*0ZUgy zU2l^g(IvorDOki~iM{_6xUh)YSoZ^Bz?OQ_Z3&6`g$LlnTj6OU3;fp9($$r!!Mp=m zH1RvVr)?z>Ei#dUit+J+NSUd)E_1UnRm5&DVzVc2n#TJxc}D4xK|%?Y7#Y3L52!B+ zan8C8xGo0lIr>p-pt#$N6l{{O{*C#>eIN-RIM+lN{Y?qd{~YJu?z&VSHG+;r&nP2( z20o?kZ1=CmcMim+UEe;rGiaKcL{0~Be-bu|uaF+A=*&fukEM)WXJ@sF&-|+8GB-Sg zyXk$V!t0BM3T}#}G+Jclc<)6J?R(T@$>?d6fDaqu_dodca-DX-#n-0t@7*4&Qo zIMXVnto$z5D`cdNWzdsBHn|;Rd|w%NzlcC>7ugqtP2au0>xUsWY<=Y7 z$+e{7U3P@Bu^~tMm;TvL<|ydS7=;KR>Qdaf51wLJpY5)uw+oqyZ(G2(zWR=l^LK)R z5FAltF4`yKD*dIxevCjvyKIzK=-r=(hC)T(4-EGk?2NlL( z|FmF+0fdo6iZR}22*q_Bg481QyHn0}8$;IUMJMD=REEk;F8z zXo#y?@*fgwqXQ|JCu7sF=#11j=Uq$MfmXP`S-MEPk#M}?dVmKZ5$oYxGopfPXwR!v z+Py;!o_9jPw`1Jte1E?8*Fi5l7|~1IA2B*mCH9=_$M}q_6>2mwdz{a{*7~S}T*1!y zGA)_R=19U+mu&|94^RTK+K{B@)z>G9$&PnK{8 z58kn?yqLewK(dr#VdM9g)#)r(D8TloIUCb5%Q*@@Sm zE}TSd^H}mMb6?u-hptG=nTg>#9J1_+g63b>Mkdv9HH--XJEoYolugV^JP@7hqyF4q zPjNeuf(k4nNwCVNs-1!x{DzeUZZf_;|i`YLl2`hBg|EJCRfm7 zvQl?sm&@8_3!dg-2HFM$dnN}PSSaGQiaEuG1+mbnG1+*WsK~W_1+nf}>;Vv*r)wXR zQ@ExK7XsuTc&(=3VoKZx?eowa!03Lw^V)7lVCI*e=E>L&6@zz4JEZ5kw@@^{#^|B~ zPy{~__)OU5YjRK<8{oG4^8L{R&7ap>`Gm)Y)hU8!iHM{BcJyIC1m6^+iH;@cUAqaiJPw4cdxD;0p6ml zMZpW|Y{@+4gJ}yy?;g|&B`pJ1CusF?<@mR2$huZ@70-zHB8y%U2qt$R1-HMt(M-A7 z=8{@BqpXVU9~w;%c@&!Tj1r(zfEM@&KG{FxZ`1H_oEUlF8C$1#UE6P)wD_ zjDBbDjuapKN-%kb0QXJ<3#|8T?j89fBvgv3sF2L3s^~TQ+kh<%jPA9H&o1j+23dNt zDiLYR-wwVc2%a|1Rw_sk30VBV-o6BE9yJH{K^P?O6pSfOx>xe{$`nxL%?Bat`5cIJ!{ zO|H|qE`4O4 zAAi=bmjFQ7#dqQ__O}oFCYoZ90?O)!4#rZ;_YyVNb0&)z%e;}+9X$ebnI^HGPBeDk)59HE1otz;<_~YRxBIQ3w$%3r5Nwn4aPET; z?wxy>w=a^O1(A3?jO%ZrBoQxVUV#h8*V8l6iMaiQ60CJ=y{si{2~mFzo}>rsnfx&Y z``es~p@384qO$;?rCQPVFjaywROL>FaTJrrvqEOiH)jmWv}FT&GhceOu@y?`tfzT9 z|K${A;bN7xx6tL{6<;(-+tdYv{T0?}^$^6Pd^*QWPp^`#sNtaR^WQ=X+4j`k)WA_)B3 zGXdy;T>eJfsRGAF|J9T&^)X<{eS&&9NyWwBsZDJyT6Y)qvK%}=e<4~+?9AxP>RSZk zRJPt8C%oC-KCykVyk(}wuRA3amvkCGmNa)Q-F4{&(`V0!ukodmq)vfLKuQ^Nq@Qch zKH=aqcCiMlrFn~)gxLjG?k-2Q%M#J;oKJs(RibxBkUId0j2~3yG0Q0PbK2!7y7&g9 zJ-Fn%DX`F>dli*(MX=`p*zK3Fl*w;Z9IuH?3m-Zwr=A$1|7{9me6_b0KxDsRy^{N^7|D5+AkG2Gi9>3!D9*X z=_FFSQVuxd%*UM6VYoS22Si3q8WIc(_UUBlwI-jis^PdKrj~UF)vq(_8QATyU;TFL)3Mu%J zBH_*km?CVrjSt*&AY-2KF#0fVa4VkJAOa^lm7tX(PGYYrWtNide4v{a^blN=)KSo1 zMGVq!xaknI_-+(2#(Bn_>YblO^Q!4;4Bt_Yto9`OicD>m%rdSKKx7o;@dVZH-@Ci` zcG)TZcJuv}iKkg^2Rkx|jUQX;Bjdx4WM4(66xi7Jb`JL881hc|VO!Qo<7gB$mHcYR z=)yzP*N(4aFQcYUBd?KqBR12vth2Nj- zR%hF|-rFngnb&X7if_~>ubzFK^os9MQOLC5>|oy$aYHsNd^LIb>WJ&9Uro2en)sb> z{3M(3R+@54bYJ?S&DESDm8B)A>M5#>QduX0LR*l0j|C!lXWv021T04&p~OKMBn~4p zk!v&xJPS$FFF}V6_uy{qqoFgL>fw&)T-_wSC@agf8MWs#y<(Q}myqrheU+TpA1?G$&-^pEJs9f`RvWBwg#qALvp&;A)Xxd*;N#`6!FDHNyMy1l%Rs=U;W6h zplRWo)67f%M2YO~YFj)-hCp=T@v6lOX34r_r%plivzMd!j*V2qkMf1UWQcGKO1U{j zOpC+K1<_MhtB7fHG7+1^00?8p$%UiD{Y``tH1R-&>#KV(+k*R4|JK0kIxnb7xn#I%E(~dU1?q9Ck>xYGO6RP@Wyl9eqSZA2VIa#y7MXie==94*ssU7fsebl!*J`cP*Z{(=I zaHy;F+cs6x5n4rl^2qAk6cX|7n(^9MOca47^N!LENMNOE%OgkLYfWMIThXd=-uMD3 z?`Psx8iS$ypcOOY)xrsWb^9WeP@I|4Q4^gPtu>dE_Dwmc<* zrwQ^W>6LYpC>=8lAYYi%Q~fkmLax{K*qgQ}3KsNOh1$b_?8>BbGtS)M*AGXbUl+{f zhe(;iw8HwP`O?)=gh4UcPCs^9aGh=0N5#C!#cOy%rOb0=ewrrwEU>IJ6vawkjWl9{ zBWj){K4tiRmR52?oc&^=_U9aPd8O^F6c}(O$ zWh?*51*0@(7|X8(gb_~+cmO}BNSJ|ac%^ksL_e)nl?6AEKnZPEtI>C$j`Mbubl#ILanZ> z0+X{!n)Y^SqEpHJXBt|*T*T7^?vsXetTzq`2NUXdryfeUFM%edQ7Bu`V{)zJ5~e;k zAZK^ihjp-K&4SMBHU`Ko#B%RuV90h#T&4t@!!~69RoXqtP4QWqw4k#y!w(z1!V1O&9QZNvgwjDUjlG zd`5VPY`{wFmiqEJdc(WX-7NC!W)(Q|1A;V+oo{44jSch5&cc+(ZS6?&Fw)Zy01gOj zt5JALGhsDh{5v2rBj)@b#!kAl`ZPNX#P)I2E4R3yr-W1qKG>ZpnrKGcQ&rzrFXsAF zaK2_``DO>(?M^>p%IM_=U$C%p>iucb?_kDRwiI~$A^vP%sxMxq9r-lN-DgRv^ayd# z(c|JWl-%1(4wO+}hRvyG>}@1#JWP8nfSP4r+t>Cn!N5Wj0fN}@;x7P!J$Gqjm$?I8 zLO3ub_W+N%{lFyvF=pnWwVHj-?3>L5A_h@U3xTNg1#fK_D3-q`N+U`fUq{);58+19 zbpBW1R+0k2#qr)5QE! zy|X?07Fd73>QX+{^-Z3o=#iF9A@McvIKX-EMtJ5+|BY+-wB7r7VFcQ>A!{Y@XeFWo z$pvDt8n;cj@Vg(?GvgecOuXlSL`5=?sIYoggB?MmBH4|k1ys+B4X2F8m0fwHc2t3~ zwh%V(k|{l4_Y;JqR%(@b8(`>fQ!Jl+y}OnNa2pXnez^U6--()#0n?7}VjAUlI(j>I zF0D)XM}){ZqzL=`=z4oQL*+Yu%+Gz}NBhXZ{)Dvh)Jh^^iC_CPpXO2FS-*0E=oI$A zr&24qnsNrHA)o6?&duB3N8s;BHKU)OWH$;FK)iqM4yH`Y;zG_-WRho)r|m6Kzwo0D zoCd93C!A+#tO#s8o6F((OiqY@62SYmI#%ODM9%xWfS0M}?F)IgbTv~qOfD~*0^WM% z>)K4J7%(FnbAyhM(tXE$=}8Sozo0-rrrKS^#~7o)9y82l6UJZrA>C;L!UvA93X&*A zHnH>P8qs)Z;$};f&t$<&t^}Q5zB_!!1fy#bL;Zp}P`pP=ne49OiJgJ`816m0`jUek zEU?U*q(QWJ5Za(FrGm*|I8Hr$sezeJ?E2T3oCYS--Go?eE~j~3(9?(}Ck0!YwZVRA zM8o`R;1W%T6@>PEgDWdpK%37-6W!*dGUQG00q)=_$OhC5?m2a{BiQsZpo7y zLBkKE;rv?9ab-PBxx%o|of^{^A}1iNxFEqcDQdGH0ckr1FS|u;?=UZ^V-17?*Uv8+ z5V{5?h`=X|E+HBO;i1qBdr}2%zYlGDvl~6t+R~Eu_4q|e;;c4aFvtEE-lDuEj?UMz zeq`h;j+J-%#F~qSv1;|2bU2n-}may=UOCTrNqCWMAM?YHctss@W)~Wi?x)ULr zyZDx&vIC0aJfmaJs21W&ua{C7^pzo&uBCr5Qa4Rm29b2WAvZwmo4-{EBX%l??u^rc zXv-u`AW~G<`p!R@I}L6Sb0?`qlEmD}c`a^r1a$75uqq$PX;^h_PrS{YEjPk^M=R2C z0y~-u-`^+V1shKY)0@D0;^lW1#SogW$U5&mgE!m`I>z7iuJb<4xK4kd--3;OV}b_* zR;<|K(|m8N$CBd0%Trjua9Fkl0dgf6QooaUzxoTaHKU5GTSd0CWPa@}TNq9>z#EC9Nd)zq;817%#Oiso%=4oms4a?|(<<-RwmJngm!-ZQT$XCc5m!yAMYc zD5W@4r(pSB&#h?#RaRdQ+J3*0Eq@;hrsmJ=8olSw`WGE72q^Mbd7 z8}Vh5fYc^cGvqu${p)I?>r$Au$5M-2yO9ENhA1dDT1al0ez7O5+3Mx%n<-0{bV@2= ztkCfNaX^syEWy=}6S!{4F*_&8f3lq5aq_$VIPs<2jeTF-fj@>`5sVpkX8nVaGcr9IHwe_6tAw?C9Ajg z^KgKGUl^3nr_gclfFKmb6M@qPW$b?K4_Z@qTI5!O8?P`Bs3NHz3D!p9*bU1pvP*~# z8bbc9w9Z$yi!UPML|`g2QiIR9cP%|0K`fl7?{=0@NqHE1SQC(BI&^_E#0*U{-reA!BFaX=l5% z<8|}kUVeeOs-0q6DPkRNN-(ovW}`XI=BpyN^!7JZ&Ug{nv29a{3Xl?(i@Z%MsXXy)ds+nOOGL!T*pZt9&<+a|8{7ky%-i;bARHOq` zfa!xtjf9iadvy6iD)aMR-upL@PX_Im5^8EatA&iPpVf)cFK>SH!^_}2m+ZAU{5Aj= ztN+OOPt%U`oP?0u5?R1q2w-e3tcK#|`^)HLSXVb2G$^XAcDa}rSGRXN0{npmPC|`X zYbt_&4T;aAr{5!E%Eo)8!mBCHGC1koh`&!%QkHo9pdrBN{8so-Ppr{Gc2jLG{uuHDhr!T_ley2|yh}Ht>R1KK?QSW5Xq!2xHalSD*85pLWo+$s zr;?U>5K-TatRTwH7i-&oxYxWlXVh*!qkx+D3G%3PvbA&<*={gR-PSN~e%(^$ddGy4 ztZKuO1w0&QtV3bgx^@p}?aSky=f*Z#GGL`w2mvxRUKQ>dPJuLY%q%0n78JR^8YzTY z5b9V&A2jQx*b!tm9uIj;n9%i+O_fLa_+pfh(Q1N&MP9`K)XHXgiKS>ot@wYA!282iVV z_X5fZ`EAvEALzLJxFzb0_b+t@s26UYH;f5KD59jW_Koe$^^r`pWjHXFoI;B%48y5i zRDK<@<-n!nchSY)9b9tI131Mm+6+-^W78&wABf15>gjWB%H#h(qTV~6?eBdb4_Zo# zzNo6*rHYnPqqb5MwMU9ZbQra3#U3>}6s=j*jJ;y-2u0Q2H6yhnMy%N5d-VNz{C@u@ zd7kskIrq5k>$>lT?M!d(^0c@`Q&A!FwgK5RI|JL&x9|i`6>F_rm;onC&VPZ%$*xa% zwWYK=V=`<5$6c-cjDMDeb1Q`2;!>I@wA=4t!-6rQc!+c^As?gmviBm4# zsri$5W46hFzKKtQS z{`lw@gOAFAeYiBB6Ah6F()C(BvJ(Jpa(I!F131}tb#}H=K+!`3Jq?zo<#>oS zBjMGQNbHI}Z)wk6SG(3tC%Bpu-$vfZ+kui_p82sx4B|F|d|K(mg%-vwm4JOfk@eIa z-0s7jEKG))&P{g2%ywug!CFs=r<*Z-F{bAfxW)6y{|04bvs%@L7FGHLCQB*7%*y8H zgWe3z<@7sVf}PiA&^skKz{TP;hGU*gB(DJ7hbxkZ=iL3xb^bsDjSiJ*5V{V?2VjrP znBPzZ<^>kaFr@I=m!8X*s4TXKvE3Glwd?!8>P&xp*lnkh{X1#rASJ7*KXiCBFW3u} zhkBL>_FU5&x;j*uZ}%vS`?102+Na@pGC7NfmGt)mOi0OU{Xom?Dzera4xoMR; zwCj`wP>ZWiX$X->W35h-G#5c4ZxO#w&@qm|7a(`_aua$jvR99OtNd!~+HHtKk!^#} z{b9Ac_!Cebfz|{1G|c45QNwjr)@zt3w~@N=hB7HEFWmozQOi4t3n-5;=MnT8eze$W zrKAUa769SL6?U;#Uws}A(g^vJpJZ-V^s2N}K*CzDSR>K8@F~hX$-;ug$ZE9#ba`6s zwIV$BvF!Ee!8=?h2Xa)i_p6q2`9eL&z|juhLy3QNNdTDer(FcxJPdZEVMi3v!JAzz z1;}-4seyD(b*q&ohJbDyS_{VJ)tO9f#-4SWaZ5Rxv)$r(vcvUe6=Yfh@9Gl4zbx2L*bKlH3KofFg_PIq$DhpB~DHnHp0AbgaBCvx5lb$ z#cAv~7c)B}U!I=qR=hWNSEoV`TGEO!k7uI%b#Wf!GXtlgHLA4v`5L+WJ-Q^+YwEFA zhviQra^EC5x@cYXK~U*e-3Tc9U+ahHz4+*(B^l8l_8y(@=CeyZNq`#FlT_m(?Ar!- zc{cJef^J*~s}2Hi)+fO@!l>?BZUd@3_gm9RYs%ua-zE?^AFPsq(|?GT|5WZ7)H=BN zoq(!8l-f*2qk~LaMvj6f9SYHZF!JJI)F}J?gyvRqLkWqzd9gP2sOzIa=&biP92bA{ z8Mvv2gH0ldj!U~Piz#x(l$V}tZ`fPi5jAsdVoU=v;I`^*NUWZHE zscZBI=6}&7Gfl1an;{EP-OMrBM8&lN$kGb}?>?!&5i(__3yjL}IGQTi_FWuv!08@o z#(F+k^74M}+RVK4)F~rK`GPfWKx4*4|vIm^F^$>5`RIeM%`W9Y)E?a%XrhN$Kmt&%B z9sDZovIchNx?9uhQ)-6z8$eZ?EwlPoeepIN0oWMuvbIjC3RW=aBW0k5b5H(6KLVZ@aiSZz2-Z_n z2ka6NLqVpL3G=g4z7ZMlWTjk z5xoorQwzZ8xe{`bM7_2@pvEBBav2O0`1;G-MWnYVE2{Y5$vog+V7(M}Q{}UVe(8D5 z;*k+O;|OTo`0(UAfYF6r1Jq~uQjvQrzeUNyCoS(+Vv*||&<3YJxysAl1ih3FICtX@ zU{C_B>&Cki2syBoM~_s(i6q-3$))uwE!<-ZG5D`?T^$p*ktXYM=O>CM7YOxOHby9 zG8y+2=_FcU2Lo|E3fQN_xcRDb@)F(t?XHRP*j}X z0qWRZwZUvq&vuiw_+oZwbQOBlu52zVq=SFjNNlzGU|!H~>15IGqz$&D2JH|C_gxR< z>f(yO8M((H!=ZOzpR(WOO%?)raB>Uj&U8C);tVGK$NiX!Mfn%ymVL?6A5;?MmaXUo zr5{W$7Y8q~uJ-b7hGs6i>ZxEs@96^O&L=cZt<)6;BHp%iWPJRgqhDShhYjd+~}jao8@@E_*yoO5$X%A5m`) zYk=SlL4pOdVbvnsEE-VfIw`hz!vXc?`@lXkxzJsEQ5!r`iS`CRezAmlJwC9{4@|wOUH+qJ~Md3}k6Yl~B)BJcBEi6K%sAAYBO;FJ^<;**uC zSl}I|OoyJ8F_3i$m4epxx)o$G*U?kDNtwAOZUe~g>EXg|a-XyAkp9?4jk^+_ zBmHls?fAoRVig%Ks+8*(uQH~=87{f4y_-F=u6|w~mK2(!lbJ>&Y~(f~{U)lZ z_3mEmioIa)cc^kz*d_gp$$U(SsDBL(CNpSTB@V(AtHzz!1=30H>5*2I5z?M@U8ZPG z9e$uICgOxO7hFDhi}4HE@Ot38*Szs=)UlfTpnJFai*H7dWXp4fU!Qx7zzz34fA4+B zPuv0^{(j(a`+hi#@{PGKwUiN6dhRXkOYQ{1wd$ZvcBjPXaFYq;9t>zz4Y9D6!H++` z^o&GbFTu(E{GR>Ma_3#l>G2l#pvNjSfooxw3xTTJMVEc5 z!DTJ-LaZZKlecLG*&hzBJr}%sl-H5Yd!7H9waJys**n%iQ1h#8Zh-n?pZGu8phzmC z1#2;yRF-mlk1fx=OQ`J)QY2wrQu7M)u)oaAh28fcx@CK{e}dlmcWfA5A8b%Kb z#XdPY7{UDTYFGbJw;#mCEN6gVR?q15PrgY&uJrnW1dA(Y1n+ye zI^vza!X2hLbLF{+zRz&p%I;~WYtifIRBbI=5E5}=PBInpVlKJtzB$o zx6}5?Q{@tpQxk$)>(csv;6%)jsUdl+1=(Jc=5Dcjg`0;aILSoM#nD9UJ)6B(O)i67 zM^*1EQ<4Oq6oBr+`nR}G)~OcH6m;Egb}3l9s{U?wS3T41_X5RdeNC_>7Ei`N$Pj&R zO4{U>G7TtI18egE&>lHsUF$tS9P!Zjds{T-UXlua;aS^TA8sWtw^><^z#Wgs#6PNw zJ_oQO$6#-=zA&exX&?$AHR;YLaG3G2Di_v`r6aDKqm#s6M*yTphV7(ps3pnxu_hD} zs&^NEsvfh}3G%b5#bmGA-ZO4%N2q1HPPb6hk27}p;fuMOJ;w(l`wA}HpzHU?eGy%% z>_-xq&#cAI25?e-hZDG1O;mW-=LHTs>#V&($`$y@%)gj1)O0)DUP^K4aC-E$3g%ne zZ>~$tEwj}Q$@^XJd`gK82|>D1EQ|E%f<4k(T`LP-QT8Ap$;9cRAEy}OVb3R-W6aRF z<3}5yFhB`G8dJ zsByKdS9Xo1*IjuB0l)iSq%C)a(>%V)^I&l6JpS|A`&`ZrS@Bb~c!B3#hv~tdGk9^e zlcHT9aLQx0wSTZ3z6L3%QO?GyR@?sByYJZxtjGW;R(WVPoV!Q@N>D#k%83a4BN)9^ zm!qtfLuZtiUwXWKCUwHIb>H_K1`Mhszmxs_Nb56mU&fgM*z8fInEP2E;`D3pNpguh ztjdSwR+=h3+wTVmB6dCZ(PV>GfyVbaB`4DQ9|zabm;2-{-1A~-;e|#Ks7&<3FGXuP z_gZ`p!|007Xm9=vW93&eTwJxb!VjwZf(4>QzY44I8H(odg&gJ}LPPZMbRY4b^fXzH zEhJ6*dCx0IeY&kGndAf22yT&5|2*z>CxzXonN6AWej)B?mJ2LAO8nlj@>!ID`*H??1gFkO0)3Mc(^x zd>8=ej_S}vA8ah3i zr<3bkN}Z>Y)j^252OJx;`VFmY;#1F@A@ z=LLGkYmPXsSf`=XKB{tYV3IM;aVN)5RP^9Nb$+~740>(mAn>GTl2aX|3lkuD5IAQ z6na!JxB-XHYZ;SwF0rswms^JP*K1!)WFv@USh~$~D4sd9r6bmW%eI8l_$=@Pffz0k zc7Nz`{e*p8O`F_y+S=Agb%=?JuEnq$o-6%_3d{6rg&o~$!&-`<11r8-dcJ8_*c|; z&}XhTPA}W}*U_zU%lBe3Pv4lkHhdAr32%#~`Scid?G(uN!-`4M4xWSFWCu@ZGm+tv z9m_pstO`2P{!D*GqSLEwb!!0EU%{!|(tKan=s?TRK1mSSRrQq}JN!?ueBNh-z@bet z(N>;kP@mUQH(#PhcsG0*K&8L6rjyfd972A$$s>AA2%us`p$u9?s&KsGVZ>rvq_WfhbQUQKz>*6t?mTT%+KWQ*ZC0<23~65WpyP zC(#)HyYVsz0j%U;l3WUyy-WG0XGvJN#KQM`A!kXZ(^>3Sc6-SucAViK%uzuAI;IVy zLq-|W$!^5!jtJjq!6%7Uj6GQ%(FHHzI||s1aAe@}8lsmuRnQ{I85ljSZt}m*DTgnJ zbzyIN`YGt8ADmu9Z`AaLQA~r16!`Y%fUl58Net1lsvTe5p_-AA?A=QiJ_nixXKK5T zvVA|0opFK)xua?NE$(h_aGt@Y-Cv0Tv^=x>jvVW>2=emFYJI}4e#fz62 z!7$sxs(SPCl)R`GqHmnw6Rx{Pxtb3Lk5(+@oW*O}HA)|?gWOgUp0_MabUR!r(-KQq z*YZ2iYbYNl0rs8!?;*rUO%MJ#Y2{q7g|0ng$QjN2R&!9jhYer~DpSRMUWE1c7Xs!% z@I!2&uc2F&CvCGlJ$5q|IjbM_4(Uo0bX%!zP`SSBHg9`BIc!uP?f5|oXmF|Ast(RF z?ET2KT$R2XAJ~6l35R%fIBSYciYqu!iWTeM0&Wz!RrLEF6j;uKFC+*nQEP=g$N@~O zx`=lufU)HVFxyp^9a7jJ;bj z{r;YRdHXZiw(*ZLRSaD~MD%plY1(xCeZOz*E2RdA&w5FOPuFV2i0Rsw1f83#nW>y9 zpqKRz_wel|df&d)Dx5Fiytu~PVifJI7*EE%ni@M9a437(y!UGyRKQLkC@{ed;m{z0&vd}dMZbl;o$ zswR}u4j&X_g6*hr@RNd*J->we1$UUiUgmHe%CYa$xY~d5EY_I}1KeHPyBQ~P^ni-} zur%?lw<`KdiTfY_VfNyy7{Pbcu4HSTV^e*&Un z3*=$+(7i#Idm~2H{Gie}(k0NPvDF{pE3(Yt2{JIxY>yJr12Ym~c&wYOU(qsoX%a@1 z1S(4?I?ox%R2s;OpLZkQ(BjM{c)K6w+IjRiZ=T0fbep)f@|*Vi0C=c9&P9<-hgO2= zyxF}zl;7&)V9Hvo@7}w0y?%{&k=?s}6#wh@N&uis0toW-RbK4^F}*V93eQ1>as zli;GktmjIRbA}O*P0l{QPI^$^33CG_&iyn_k2C5u!z(b?B|ZCG!_EZT_6Sc!)7Z1iWpP(sV!xy2{Nj)>j5G`1lC-@nFWD4#0tU(Xi4}1 z7y(zLEOu2<@fFwRSDp9G?^6~cJ&tB;*lwhcN?tlPDrxq9yxNP8zW?1jT|e6@!_fHU_*PIrb%?i#T<_zC;fW$@wb zx>Y-eWjYyQ9QJUX$6smEHFlvTi;~&n&^ADS{0Ul zO#e?&H7b!cxui)rNJO107yKs+A@E~s>``cCs;M(o;}b6FB~>jaAn463Z5%$Y(csgk z+iXnxh;+U6$J>+Zv@c5^`IPTb;N9XxVk zz3Bn4>TjmVt*qqKL@@~jDWAyR>={Ye3lTPaR!k9vKamqkt?P5S(~=S8w~BK(BESHasnL{1bSDwc{v_l>m$EO%yEwVJ|CCL9!SB8S@c`W~ zqDQaSW+r|pTW)odn&->wDnLFLv2JwQuq=)YgM+3N<854fK#PB%pg0rLr;q9diuMBeu7%%I|;4DWT_(fcdt_D&@Gh`FU;|d&HSqjWlwhC ziq;t7s-L(;(Ye?z8!p0gYNdSlqfIWGU!Zf7Jo5mc!jMM{7_bq&oXW-N<=j>}WC2<)&g@XvcE~rA>)9@Wd<_%4#pDWX~^Pm!u zlMJX&z|w}Ep-$>{DHGhqae z#36@6#ZV*;wZPK)LPIUwy5>9?f2IAf@R!1~DW9PkjOf-{;lJwQuy? z=4^31n;46@78KXuQhrJ-(*n*vLuzdK{RF_7R#w)wYW=g^W|RPdxgZCf6Y4Aou1uMS z^64?rZ?X5uqRS0^J-z4mbc{Zqsz1iooMqJ1TMzC_7~38)EpIq^z59LE|KYsfLjqaO zfK)==6S*S*pPlT2qt+d7nH*0WHo`$Ue7yfvk*@v3x8)Y+H|+FDMwF)0ZtV!VGzbrh zHOm43KDTs^ZCKK9m?_$Z^(D>6X-a~cT0~&V=d7H)-zu9i4;*N$q9PM%qHE1k%@EP{ zA7ciUsP6OZ8+PxWOAB>ZaVq~+XfK-Qb#jOS*Z~{2*x9MEM7!^Rb^--A@(&h4EyC|l z^CGpD`}=<&4+$cPPYo>n#u*0!6D(}aDb99EcTEjPCQ1n+*rZv|cSbGRGkl<_7??cM z@Jl~pBCX=kD|q)lZ8JzW{?3;w%yEymKUV2(T$%VGt*R94fiB^kzQTUWx)+3o;iQpwa z!mHTla_1X6vmd#Pk>Pj{h1CP&~Y|+W4KiF)tua3GS?UQ*y-G>r@3}h_kl*s4J-MzEi+=^Ch*U@>ekIB|_?AU%jyxF8ZX^3@+@3ayw z^}Ye*b*?}U&!_BtFu1SQ*z+&oF0DF~3zq=;!9F=;%>?JPc6uzgDBxD|PI@18>a_NM z!a(kArNLxmx@)0;*4x`J&&IUnj2ymXCA>=yeDR6vx~rP6>7wN9-|X}=Z3+H<%A8eR zfaBF28Brz%;%AI{l&&=sAMVBuoqgVrTD3%Un5KYKZ1BHON$Huxna6M6lMLvfN+ zw}Ay!{lWtZUF7rz>mAherrW(7t4HXNVdU=rW`q>xdm5o>^jH*xE`o6zuokIfQ_24L z3@}W^7uqc-wTFW`hb&8B7DTts%fA8)g+#G_f2G%fY9M_{#wuIzu>gESV`0^fY@x?F|rI1+!Gy&Honv6tWSw_y7}N62vm~KHsb(+3%x}oWX9H zU)dfS{Oj^s`cxwdX{>_DBykR7G?K+H(0lgfa7rEEy~!SE|M55I)egn+5hTAln;r zg7H4P^KGH2YGXvH9*nPHD$HLsi0GfWeAsl@0sZAV&Laoj9pw|+=h!0!=a1tJMKwyI zZFgy(mxFq+l8~^YAOdNEx znPrO3S$Vj`Go)Xo+8*s=%qX+w>Q4b?x%); zRP^JY^zXqN!hLI!V~K!(AT<8Zp@`@X8q6*7Wdw(}&~oByNBv`FAXC=Cx*cuolbb<$ zB+s}5HtclfyG(XX@c|H#f^tBzB{-&l*_*gUY!Mfnk3` z$p;jjhT51=kMbU!NroX${;oQVtmd6P`Qt6nA&mTc`&ol221YQYE^zB$B!bWo3jY!M z_+#5Y=3vUols20ul zlIC^~P+Im6^k2DC`**)QJB;sJAYBLZ(do__#j~Qkwb?yJPn`>}%Z^J1)R#RkUe^ug zc6of^<2J+RD`D3Z^gOe0 zTkJ`4>fGIuUR2##iMMrI>)P*@*iCj0h#na3PG}jisStr6Mw@Im-;7+4N6sW0t+-6R z-Sm}d(=cajiY7C(emv|U6wzlznq{JzsVyhAU0w4ETc zXI5sJ{4NY~TlDQfM(R*LqenKi4hDl?pKhkAS2Vxz!_9bk-lQsLrLY|JwU2M29orKo3w z=bvjG*anvRRl0Y^vZas8Xtih8p+Z%7`6*MZt3n_{C#wCy~HOjuW-8R24eI%oUkpDsaGDV*-7`Z{KX1s5{O|$py zmdaHk9jSu--IGzRB1}U*MDmHatHgXXfnxLnms^;BPiP`Sl`<~Atch~^&~S|0WdL_- z@3uE>QP#yU-#+P_$l+Bm7R_7qpMSt@Bi?V1NXlWXOwnEa4PgXFmqmwEMuGZ1+lt}z zb-gi9n=%voceIdl-N`1n#g>4Bi)#e&;g}ywQYxB;UaH^#;zrOPx$rci+9R$#CLpYq zkhd-)Jz;CX<^7j0Ky1LvZZQcdtz%>_QAW+!C3`$1O5SHp%6>!pu31b}By}^XS6#41 zEwy#Zv(t9+QJK=!Po{o+hif(_H;siB1v5gz3e_AhZYv*AVLR)KZIB3b{W zN8iZq>E5@WV(Jm@3?f>t+R&DF4nq^|s$Y6!*_MABBl;0pAs_kMI~q?AX2Pnz?rnut zrlZsx9vS%%W392|7sE6Q&-=g6zl^(SSc%Fu=-|9 zF=|uvj@I9scT(*W{Fu)hj&QA^NmY{FZ&@iQ>xzX{O6$3eao4}UB7aj=z~b$Sj>o%J zrk9o?ZR{3YK688lcWP(p%D&w~{?#+`FfViN5jxyd2e*c2n?yQ)o~2^r*UJ>o-#5Uv z(v2yeV}}otyJKCI>cE3$Y75Xvmy+o7U|DNC2%-@8GGTZ_dx`_z&-DPZ@TkI@(>;fedX_}8qEG*N!;>(9SE*f(ksiA~O-e3DG zxaAQi?zg8nTGb;GHC9KgRl9ye&Y#4{%{LzExNSvQ{JWsq1Nu9L0VL#mz9Wl>R3lFB zKHFXy@IUVVsrp&hk2!6yId`3wpw-Sg6JkgB@lJVE^?nU_t*qPAAEs{~GN?b9&Ms}^ zw4j&%3zip}rucy}(=@)fPTKfoaB+&v{u{8mrxc@ri6f1vXxSOkV`;p(1-5(xy{Wc6 zg*kQuo5de2cLTSv`OL`XK2~i)6w*hdYDl>QLOT{Q+vnQC%d-*9<|IO~O9j{Ekx;<| zj(0(0D`hIvyA`N-4xjR|PQf~5NT~qWtO{sUFuEvEo0AWDbugIHB{p%d!y%?(M67OG zcLZ`b`Dan|I?vNe+>`-C$eR>1=CK$LgaGsLrC2wMD(nI)nN{X(<}#6;oO%gDPVuZU z0tO`;nr=8um%4LUx;nIK zqyTzmomy&(-*MbngdaOvQy;KK9U+Hur?aeN`%1p8sfA+OFKAg5PW=h?=waIlyUvH^ zF$P^dayU`QSEWyS!s0KzPLFjKceV$M7=p7OR~4ykc8VFeSk5WZ33=@pfopP=w6g=w z&LsgoTOzvo1mJZDCV4rH*k#e_EfXbZvyi`1Qwy-{U~eY&Kit#vAqOz0Ph$x-v6dYR ziHTUpKpRNv;7n2U-Qm-4O^c3b-pB318sfB4Kvd_%bhki-^AqJn13rXnwk;8TSY&f9 z^{G<~!UHKgu{&I=cDXg{*s`Kq;cbXR$$5(BshYuZEIBftNQZ*;#rg6t(YwiGyxHIX z8S^i!<~!@nz-E;{jf4YZjw(Wnt*lslZ(3Qcf^pN#|6=Q|oD`;!jhSw)>8JXth&+%R z&AZOyjVqiVyvvzXExrGyto15?uOqrm)PwHI^T~StqdENPamv`y)(4;8=b5Y3u5>u} z#sg~4Sl2@pkMzWPcEpddq1>}aeyY<=*Pj^zLpGjhJH2jYYVL8Ed;|?p=%Y$8fQJ3l zGEcM{#nAmOsSd>nz8Wz{+Zg?LGp%VIsd9;|_CEC94BuCi@i#Cp>bLD* z=;c7?TXUm3+7P5l1TfrAaR`r$Hvk?RW?!1$EP#Si~!;vS<!xily_PF6APGI{FvnOlQwvZ62`S|5`bu zGkJ-dzvwLzg(dPEN%RDi*8>`*`o%E0htTi4y)*@ ze0dAdh1){z=w&ny9}H%Zoov0+LYC+ox6?b3huRHTu%NA|_ux>+W9w zkzSJKP=R0hNTUo176&uRQiu38fDh70VjuP$Xy{$M{akFAwUKXxrIEY@7^(sr!d#Ah z58?2J{bsDj!M)UruaDm88+qB4sSCQ&LgthrV*-w46os)^>|@oyEY0wfRR^8LT+A&t z7w-1|RDz(fd$Za6nlAI7isZ@sDbMzql?x`*`(zHE+B^ta$r1A(?w~J(koAcf!Fl_K z+I3iDrB@<=oXp@lsoXrFa-^Rx5=4fJ)f4v4VUW47gA?V$(_fr1OY0yaJZ(bs8+-d0^bci2R%Vl)OC9v=UB=fa5TG-SNU z7?Wt&0ZF;R*TF2Od~Y5D&%acyf^B8~a6BNr_1)*BCjSZF(n4)*dy#!t&g+FvGOC~y z@Q#du3z^UD_it!uNG%`y{q0uZm7PXvf-cE zF(Qrwz4`2y_Z{==X@=-)kDgfb`k;(`Sg<9gn3OfM2$#K;hh?7_yJO+9G;%RXZo+lT z0wWh7lvl+uGnH-I(fz}kY6nfWO9rW|b9tJp{7YWLgid^(B8MPTrGaWh8|Op7G{=wT z+^ctVzjdFoSYCZj2ortl$VtPpvhv*oVRnP7&BScjpS;eS|P?8~zcDKQ5pt<-NN_-q8{KT&y>WbHQj z&O-4+NZS+hh4^kdnW2?2OBix*`$mVupmr^Z+}}G<*j!DUFuYdZ$>!%&W-PZ{Rm+>i z9%Ib&$UyF|0W#UXI`wsyPjYw2CBb81;^pkFPr3`j{oG|^NTRDs-B0$iS z_J?;V00U~2!v-M}e)4%@R#cGN*5okXY*CTrR;D-f`?a1}5eNf|n z)%>77uwP*prKVyGotUgb+3YX$LI^dxN|qN~vx?h~Y(m?0QXeUQx%S_c@()}EhKY@N zq7r;DJda4(1j3(TriBmoeIIt*7n2;R^Ki)CIP|W42c&X99>oRO*wQ>(E*t*zQQut7 zoyKjMu=C53?S_PS%k-0#hQk-!#+v-+Ds;8^DUY{MF+puysIoUjI$M{-YUr+`&mxnU zJs56S4zK-|N-n@)Ss3Ai`qPq2oD<2>%^fbvtHOyTz6>e8)v#j>{S+&4dgO?@CWC9R z5@cEB9lZH0oV*peTd#&5k4JdSIaiN6?FHPa@;+VBawPOVk21?ny1MXNxU-t3{L39W z7xNgnXI^@pRFxHESxAEB(la7`jPwRZui{w2M#KKCaQQgrNXbg|>(Q#!%EWy6PP{2Y zZFW$y1Kg#T8hb^3Hx=YVV9WcldoW^PF2@zClSPAz3d17_+xIwLH3Dz!~p`m+s{=&J=R{YOV4RtQkv(Cm1Lrs1?8l8*f%E5ou zkyACUqL+OzsA|AbmtW)*W5wrlXN!@t0l)Q~vN*+WjkSU<3fb5QQiZE~sy)^`lhzvc z`|N7l9`NI&6T28@xC{7-If;@j+VXRyu+^X`3OsHs=8d`848rFuwI5dszn+dbvpENU%)Hr!5oiefrV z$UQ2aqcYBH^+OqR_#fFIrsl)o9XYxZpXLw$8uJJoQ{RiJ1t5rO#FC%wgHeU87ao~o z2g`Am?x9r0;|%S%dx|J3pUSNi+T-O8jGD{L^fkPdTY6N#5 zExFl`pA|f`(@3Su-Mkb*FLR6@Z^?o3Us-lqOw!3d9#Ai0D0$(TJ2q~k(u?e{NQ{U1 zp>6t%Y2?rreDGVq`+AK)a1zDCq5|3+pLp+ zSk3V7I`%?VYi+(ujAJfsppTqZ$!rs2WBL3@3Ax^3drh|Ub0Mb%c(}Ny54R`nzm1oc z>MHmI$TlxK*ZRtxvcUWfKI!msi2va$xuepWa($rT&qrbaRoBkwtq#T*?$`?{T6P}> zU-_=dqr!Cac2S?y-q3t?7{n~U&Kvf~_@S;OG&wced3!rkLv6})BLx?uc`q+U| zX%0Ci?EY0!uBgAv@ph%Dkf^riA!AAmihjN!<>M~ndLG5+J0NbSt9?JU`ymJCIuh@* zTL-}L5^cNC3SH1(}#b;R5nhWESK3+*fL0jAM;m!P*5~TH8CUAa*dwu@5h3xu@?g4!T zG$0|5*39+&^OQizM%4N`^nDDg)z{@MSg-+?ypL`Qf^|V2J-FhS}_)#yCc_$nk+zwZZbiDSNsD|%|+h~N-#FJ+v`uT~am#CP3 zB$jVm>QNe&&PBfb;QNcEh}?XwhR3)8-&xw300xJQ@uXBMuP#z)4d0QcN@VB05O5TK z=C8kMCR`>3#%94PQ0))R2f7|6f($Xg#cznUk&rRNfv8qanx@J>`x2-kXga3jXBLu9 zyAq9k74+0kO*XPq?s9yoWbNO|;O7!TkHE#*9{#dD*+)vV2)C>>QgkPK4~P=J1enfJ z{BOBX5aU(gNUnsXC2BR++-BXjOwfEQ8}RcUE0#w;<5Oi2*=Z&`GT0U>4qyz!)<%{CP%B1fcqv-88g%Vv8K zxdB!2!!9$ozQ-0?GV`piv0Psa>k{n1FJO3nVAMY5V|1Q0$Q42fDE1iTq6=YLt}^A! zWy#POx)gb}nE#8cYIKa;U%LLba6q5^f}fUt9o`)L0=Tp&0`cIQcUz=#W)v8ucgO0t z3Rir$kjiL3{9SPVy&Nq$(Zomnwl(uV7n|df5Mp(Q_9J&Wj!Hc>qM@u-PWn-~zOP^} zbKGnjXy2qvb@GuuPjH59b>d0hHWIFST2Fb)T-XyOrV|W?DT_nzy%Mv3VNF?_3^#SB;h0h^J);BObWw9TbvG=RK+m&f}WEIrrje+Ewa!IIf&H{X1 z`2150$V{{SM%spkhnb>(N^JfvZiYL^Ky@DIWha~=tCP+u)8d`?EIx2p08T}b&cWTr zD{lW>PPx;7-m2P!wf|akG}}Y6R^8G1&P0G3u}O+_6{-uoi#d-ZTlr(6vR_y_Zf(JA zif08ObxMb6OoysA@C^x_9m^dtoCgi#ylqBg^R3n7%ueVh!C35fa_i-`O=Qj{}FLuO> z?{J%fX+D~n^w2ItI9j^7YP@45EAd633x7AykwP;pp(3>!vtr(D1g>Mvs&QKwj?GT6tuxpsU7^5w*%o3p|fS>4x033_&BuzHvh*2X0C}ot{hY4iTBw`oM&n1m5paNzRyu$XyNh}EhztAOSjafP}XwM#X~Uq^{%-|qzLw5?{^NTk7Lgs0uZ)W z+MUPt4$dTe$EQ!MNu7q7Ey z=F=LQN~~B`SUPjQg?DoxzHB}*4`%A}-C`^2l|zI1cqdNy4UTP)MyTNb4Qa>>~E)F z_d<%?ikF_Zb6u_XQOQ`mWx{>5i$|7~xD{-kF27h93AqU^!%l`jvwVvX~pn*p3~_djvscXqqJ z??NxR>Ir#8H(#6jj;-L4-3vb+c0`R?ic4)iEdx;_czx)|g6O9|JkNY0ugHA^J$JkX zm67n<$T4FZrRnTVG%;8lPVWCBRBG=Q(aCzW_`zwp+D(2_&-RrZWCvBTbJu@#W+WNN z*%P1h_NR`(BM#;g?VVy+)*C4-yjK1@#u>!E7+BdJ?3O+11OJ5gGI2lh6I*3XhD^On zoU9BzGkk?l&w5bmQpeFT(c%M_z1VVr&R(}n)mTXzVY}Go93&^WzEjAL;%B)%kMnB9 zB{Q;eT7GT@_mz^I6yI$fX#CA_HquI2x1n*aw4QRfNZl+QSZ68kze*O(AZSaJ?pQN_ z6&g>DMqAXaFdXCbV@))X-esr9aPYy;*;N=neKOmX(Fa@=I5%VO4W(7Gg0-jg5ZBL% z2=GT(CEEQw@l5e~p+o~()?3IO#*cY#feY7s$%2aG4X79t{Rer&1`W>&Cf9ykM*35f z9_V}ORTd_8_S)e2oRB`;{axrcVT#R+uwv@%9lwGpsy4+FYg|G~zIr(JNT^(Caeyt6Pd^)^R?{ zq5vtkmowZtT-p(`SsNc+!K|D~^Iyun1L-%a*j@49Z2!!JkWolQ_=+n=`GhuY^RwB# zdChRA=~66n?JMMu!SN8C8;AtkOx3C`tm<$c7#r|tFB{f{dWEx#Hrxh*7X)T|2}I&L z;a@u*=3QN`ntfWBEvXnC5c+M(&ql*76aI;N2aqwdeARBKVtJ=cGj{NbUk!hDY~xL~ z29u&QUyQ*E0*~_0Uxa(yWTcOrAB{auoeKJ;-jk==9E62ip7b*|E<^$P?Quy?bCp0i zH>%XlU!rtr`Z3;DLbEm?bmEsWdOxdNq!dm|_l1%(AP;9AWg?!oe*f*<vok> z(>Od94FVE4Bf8sOh5~!z&GAEm8yKxp>>B^j)*XHnO@-%9aC=HAqL>5lL<`~0`zY1w zzcy^xus6Ix0*@8moz&-SW>V$+Y!*t*q?Cj^!~j&CYrb8sbIxg3!U5aB+)vs={?fqV z7NS)|4*NtzM8w6(Lgrt*9RA|f=C#boj@jadHow~#bX^5Y*Kl6H^q~_}(ah4RMB&e? zpK%A@N0mRG%`=po5!uR#+zI2E5LL&xn{B&S-L+FNM!;5YvR8icV%{>OqxW3GpGE^R zhCjtF_GSb{bp^7!Gl(~9UEHF>5%Lr2>&5_dNhYhEgE*%t~SR$=Q-fMyZ{k&R@!%hVr)XD_s$P0t|En7^Hcup-&z zn_^Cp<;vm-OkW(BuRNvyRsazC$eVtSLk0+4cSx;`joq?O&U9opgZNGlAi}r2yTzNS zwPBX4*YIpM1)c=$RD{66jpXBU;^Gb@%FE4riGXHh*ui|w;OsbMcnhG$6r=>)Td$#2 zwe0s*7-EO!+4d5n1xV)U$Tu?pnoAKk#ekU%Kg5X0$jbu2EUi%u|6uBlPM{-_YZ~LSDK97<-~cJx+Q>jwFBk=^F7x*S zOvuBCpqo4=d9_6_f#eBE>UQr3LP*5?62gp7?g^_+%cB1 z$OZDM0Pyxk%7A`YRHTVKrC}$K7(coLhLz1w#~i{aeB6`?4JyGa*cUBk2s~Oe9XG%T zb0DvcWLgGUK?);fNR@gZl{fOep#K9niSmZJe;I@6$cEFy4@nCuw&~Ka8QUzxBgKfm%9L4$n8p0Fn;5Dwv7@ zWYe1o6u?p^CXS8cj?pCS|GQ@CA!3KUyu5x_+`6BT|F~q7{b*)xPNH^z Date: Mon, 13 Sep 2021 15:19:53 +0800 Subject: [PATCH 27/51] [Feature] add kitti AP40 evaluation metric (v1.0.0.dev0) (#927) * Add citation (#901) * [Feature] Add python3.9 in CI (#900) * Add python3.0 in CI * Add python3.0 in CI * Bump to v0.17.0 (#898) * Update README.md * Update README_zh-CN.md * Update version.py * Update getting_started.md * Update getting_started.md * Update changelog.md * Remove "recent" in the news * Remove "recent" in the news * Fix comments * [Docs] Fix the version of sphinx (#902) * Fix sphinx version * Fix sphinx version * Fix sphinx version * Fix sphinx version * Fix sphinx version * Fix sphinx version * Fix sphinx version * Fix sphinx version * Fix sphinx version * Fix sphinx version * Fix sphinx version * Fix sphinx version * add AP40 * add unitest * add unitest * seperate AP11 and AP40 * fix some typos Co-authored-by: dingchang Co-authored-by: Tai-Wang --- CITATION.cff | 8 + README.md | 6 +- README_zh-CN.md | 8 +- docs/changelog.md | 47 +++++ docs/getting_started.md | 1 + docs_zh-CN/getting_started.md | 1 + mmdet3d/core/evaluation/kitti_utils/eval.py | 210 +++++++++++++++----- mmdet3d/version.py | 2 +- requirements/docs.txt | 8 +- setup.py | 2 +- tests/test_metrics/test_kitti_eval.py | 74 ++++--- 11 files changed, 276 insertions(+), 91 deletions(-) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000..a63e1eef6b --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,8 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +authors: + - name: "MMDetection3D Contributors" +title: "OpenMMLab's Next-generation Platform for General 3D Object Detection" +date-released: 2020-07-23 +url: "https://github.com/open-mmlab/mmdetection3d" +license: Apache-2.0 \ No newline at end of file diff --git a/README.md b/README.md index 8a2075919a..ef287dd68d 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ [![license](https://img.shields.io/github/license/open-mmlab/mmdetection3d.svg)](https://github.com/open-mmlab/mmdetection3d/blob/master/LICENSE) -**News**: We released the codebase v0.16.0. +**News**: We released the codebase v0.17.0. -In the recent [nuScenes 3D detection challenge](https://www.nuscenes.org/object-detection?externalData=all&mapData=all&modalities=Any) of the 5th AI Driving Olympics in NeurIPS 2020, we obtained the best PKL award and the second runner-up by multi-modality entry, and the best vision-only results. +In the [nuScenes 3D detection challenge](https://www.nuscenes.org/object-detection?externalData=all&mapData=all&modalities=Any) of the 5th AI Driving Olympics in NeurIPS 2020, we obtained the best PKL award and the second runner-up by multi-modality entry, and the best vision-only results. Code and models for the best vision-only method, [FCOS3D](https://arxiv.org/abs/2104.10956), have been released. Please stay tuned for [MoCa](https://arxiv.org/abs/2012.12741). @@ -62,7 +62,7 @@ This project is released under the [Apache 2.0 license](LICENSE). ## Changelog -v0.16.0 was released in 1/8/2021. +v0.17.0 was released in 1/9/2021. Please refer to [changelog.md](docs/changelog.md) for details and release history. ## Benchmark and model zoo diff --git a/README_zh-CN.md b/README_zh-CN.md index 4fbf874a93..cfa77a480c 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -8,11 +8,11 @@ [![license](https://img.shields.io/github/license/open-mmlab/mmdetection3d.svg)](https://github.com/open-mmlab/mmdetection3d/blob/master/LICENSE) -**新闻**: 我们发布了版本 v0.16.0. +**新闻**: 我们发布了版本 v0.17.0. -在第三届[ nuScenes 3D 检测挑战赛](https://www.nuscenes.org/object-detection?externalData=all&mapData=all&modalities=Any)（第五届 AI Driving Olympics, NeurIPS 2020）中，我们获得了最佳 PKL 奖、第三名和最好的纯视觉的结果，相关的代码和模型将会在不久后发布。 +在第三届 [nuScenes 3D 检测挑战赛](https://www.nuscenes.org/object-detection?externalData=all&mapData=all&modalities=Any)（第五届 AI Driving Olympics, NeurIPS 2020）中，我们获得了最佳 PKL 奖、第三名和最好的纯视觉的结果，相关的代码和模型将会在不久后发布。 -最好的纯视觉方法[FCOS3D](https://arxiv.org/abs/2104.10956)的代码和模型已经发布。请继续关注我们的多模态检测器[MoCa](https://arxiv.org/abs/2012.12741)。 +最好的纯视觉方法 [FCOS3D](https://arxiv.org/abs/2104.10956) 的代码和模型已经发布。请继续关注我们的多模态检测器 [MoCa](https://arxiv.org/abs/2012.12741)。 文档: https://mmdetection3d.readthedocs.io/ @@ -62,7 +62,7 @@ MMDetection3D 是一个基于 PyTorch 的目标检测开源工具箱, 下一代 ## 更新日志 -最新的版本 v0.16.0 在 2021.08.01发布。 +最新的版本 v0.17.0 在 2021.09.01发布。 如果想了解更多版本更新细节和历史信息，请阅读[更新日志](docs/changelog.md)。 ## 基准测试和模型库 diff --git a/docs/changelog.md b/docs/changelog.md index c74dea2fa3..47a02d8729 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,52 @@ ## Changelog + +### v0.17.0 (1/9/2021) + +#### Compatibility + +- Unify the camera keys for consistent transformation between coodinate systems on different datasets. The modification change the key names to `lidar2img`, `depth2img`, `cam2img`, etc. for easier understanding. Customized codes using legacy keys may be influenced. +- The next release will begin to move files of CUDA ops to [MMCV](https://github.com/open-mmlab/mmcv). It will influence the way to import related functions. We will not break the compatibility but will raise a warning first and please prepare to migrate it. + +#### Highlights + +- Support 3D object detection on the S3DIS dataset +- Support compilation on Windows +- Full benchmark for PAConv on S3DIS +- Further enhancement for documentation, especially on the Chinese documentation + +#### New Features + +- Support 3D object detection on the S3DIS dataset (#835) + +#### Improvements + +- Support point sampling based on distance metric (#667, #840) +- Update PointFusion to support unified camera keys (#791) +- Add Chinese documentation for customized dataset (#792), data pipeline (#827), customized runtime (#829), 3D Detection on ScanNet (#836), nuScenes (#854) and Waymo (#859) +- Unify camera keys used in transformation between different systems (#805) +- Add a script to support benchmark regression (#808) +- Benchmark PAConvCUDA on S3DIS (#847) +- Add a tutorial for 3D detection on the Lyft dataset (#849) +- Support to download pdf and epub documentation (#850) +- Change the `repeat` setting in Group-Free-3D configs to reduce training epochs (#855) + +#### Bug Fixes + +- Fix compiling errors on Windows (#766) +- Fix the deprecated nms setting in the ImVoteNet config (#828) +- Use the latest `wrap_fp16_model` import from mmcv (#861) +- Remove 2D annotations generation on Lyft (#867) +- Update index files for the Chinese documentation to be consistent with the English version (#873) +- Fix the nested list transpose in the CenterPoint head (#879) +- Fix deprecated pretrained model loading for RegNet (#889) + +#### Contributors + +A total of 11 developers contributed to this release. + +@THU17cyz, @wHao-Wu, @wangruohui, @Wuziyi616, @filaPro, @ZwwWayne, @Tai-Wang, @DCNSW, @xieenze, @robin-karlsson0, @ZCMax + ### v0.16.0 (1/8/2021) #### Compatibility diff --git a/docs/getting_started.md b/docs/getting_started.md index 8a825be617..9433622676 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -13,6 +13,7 @@ The required versions of MMCV, MMDetection and MMSegmentation for different vers | MMDetection3D version | MMDetection version | MMSegmentation version | MMCV version | |:-------------------:|:-------------------:|:-------------------:|:-------------------:| | master | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4| +| 0.17.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4| | 0.16.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4| | 0.15.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4| | 0.14.0 | mmdet>=2.10.0, <=2.11.0| mmseg==0.14.0 | mmcv-full>=1.3.1, <=1.4| diff --git a/docs_zh-CN/getting_started.md b/docs_zh-CN/getting_started.md index 317dbaf6a0..0e5239b325 100644 --- a/docs_zh-CN/getting_started.md +++ b/docs_zh-CN/getting_started.md @@ -10,6 +10,7 @@ | MMDetection3D version | MMDetection version | MMSegmentation version | MMCV version | |:-------------------:|:-------------------:|:-------------------:|:-------------------:| | master | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4| +| 0.17.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4| | 0.16.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4| | 0.15.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4| | 0.14.0 | mmdet>=2.10.0, <=2.11.0| mmseg>=0.14.0 | mmcv-full>=1.3.1, <=1.4| diff --git a/mmdet3d/core/evaluation/kitti_utils/eval.py b/mmdet3d/core/evaluation/kitti_utils/eval.py index 93492c466c..38df7a5504 100644 --- a/mmdet3d/core/evaluation/kitti_utils/eval.py +++ b/mmdet3d/core/evaluation/kitti_utils/eval.py @@ -569,13 +569,20 @@ def eval_class(gt_annos, return ret_dict -def get_mAP(prec): +def get_mAP11(prec): sums = 0 for i in range(0, prec.shape[-1], 4): sums = sums + prec[..., i] return sums / 11 * 100 +def get_mAP40(prec): + sums = 0 + for i in range(1, prec.shape[-1]): + sums = sums + prec[..., i] + return sums / 40 * 100 + + def print_str(value, *arg, sstream=None): if sstream is None: sstream = sysio.StringIO() @@ -592,8 +599,10 @@ def do_eval(gt_annos, eval_types=['bbox', 'bev', '3d']): # min_overlaps: [num_minoverlap, metric, num_class] difficultys = [0, 1, 2] - mAP_bbox = None - mAP_aos = None + mAP11_bbox = None + mAP11_aos = None + mAP40_bbox = None + mAP40_aos = None if 'bbox' in eval_types: ret = eval_class( gt_annos, @@ -604,22 +613,29 @@ def do_eval(gt_annos, min_overlaps, compute_aos=('aos' in eval_types)) # ret: [num_class, num_diff, num_minoverlap, num_sample_points] - mAP_bbox = get_mAP(ret['precision']) + mAP11_bbox = get_mAP11(ret['precision']) + mAP40_bbox = get_mAP40(ret['precision']) if 'aos' in eval_types: - mAP_aos = get_mAP(ret['orientation']) + mAP11_aos = get_mAP11(ret['orientation']) + mAP40_aos = get_mAP40(ret['orientation']) - mAP_bev = None + mAP11_bev = None + mAP40_bev = None if 'bev' in eval_types: ret = eval_class(gt_annos, dt_annos, current_classes, difficultys, 1, min_overlaps) - mAP_bev = get_mAP(ret['precision']) + mAP11_bev = get_mAP11(ret['precision']) + mAP40_bev = get_mAP40(ret['precision']) - mAP_3d = None + mAP11_3d = None + mAP40_3d = None if '3d' in eval_types: ret = eval_class(gt_annos, dt_annos, current_classes, difficultys, 2, min_overlaps) - mAP_3d = get_mAP(ret['precision']) - return mAP_bbox, mAP_bev, mAP_3d, mAP_aos + mAP11_3d = get_mAP11(ret['precision']) + mAP40_3d = get_mAP40(ret['precision']) + return (mAP11_bbox, mAP11_bev, mAP11_3d, mAP11_aos, mAP40_bbox, mAP40_bev, + mAP40_3d, mAP40_aos) def do_coco_style_eval(gt_annos, dt_annos, current_classes, overlap_ranges, @@ -629,9 +645,10 @@ def do_coco_style_eval(gt_annos, dt_annos, current_classes, overlap_ranges, for i in range(overlap_ranges.shape[1]): for j in range(overlap_ranges.shape[2]): min_overlaps[:, i, j] = np.linspace(*overlap_ranges[:, i, j]) - mAP_bbox, mAP_bev, mAP_3d, mAP_aos = do_eval(gt_annos, dt_annos, - current_classes, min_overlaps, - compute_aos) + mAP_bbox, mAP_bev, mAP_3d, mAP_aos, _, _, \ + _, _ = do_eval(gt_annos, dt_annos, + current_classes, min_overlaps, + compute_aos) # ret: [num_class, num_diff, num_minoverlap] mAP_bbox = mAP_bbox.mean(-1) mAP_bev = mAP_bev.mean(-1) @@ -703,33 +720,109 @@ def kitti_eval(gt_annos, if compute_aos: eval_types.append('aos') - mAPbbox, mAPbev, mAP3d, mAPaos = do_eval(gt_annos, dt_annos, - current_classes, min_overlaps, - eval_types) + mAP11_bbox, mAP11_bev, mAP11_3d, mAP11_aos, mAP40_bbox, mAP40_bev, \ + mAP40_3d, mAP40_aos = do_eval(gt_annos, dt_annos, + current_classes, min_overlaps, + eval_types) ret_dict = {} difficulty = ['easy', 'moderate', 'hard'] + + # calculate AP11 + result += '\n----------- AP11 Results ------------\n\n' for j, curcls in enumerate(current_classes): # mAP threshold array: [num_minoverlap, metric, class] # mAP result: [num_class, num_diff, num_minoverlap] curcls_name = class_to_name[curcls] for i in range(min_overlaps.shape[0]): # prepare results for print - result += ('{} AP@{:.2f}, {:.2f}, {:.2f}:\n'.format( + result += ('{} AP11@{:.2f}, {:.2f}, {:.2f}:\n'.format( curcls_name, *min_overlaps[i, :, j])) - if mAPbbox is not None: - result += 'bbox AP:{:.4f}, {:.4f}, {:.4f}\n'.format( - *mAPbbox[j, :, i]) - if mAPbev is not None: - result += 'bev AP:{:.4f}, {:.4f}, {:.4f}\n'.format( - *mAPbev[j, :, i]) - if mAP3d is not None: - result += '3d AP:{:.4f}, {:.4f}, {:.4f}\n'.format( - *mAP3d[j, :, i]) + if mAP11_bbox is not None: + result += 'bbox AP11:{:.4f}, {:.4f}, {:.4f}\n'.format( + *mAP11_bbox[j, :, i]) + if mAP11_bev is not None: + result += 'bev AP11:{:.4f}, {:.4f}, {:.4f}\n'.format( + *mAP11_bev[j, :, i]) + if mAP11_3d is not None: + result += '3d AP11:{:.4f}, {:.4f}, {:.4f}\n'.format( + *mAP11_3d[j, :, i]) + if compute_aos: + result += 'aos AP11:{:.2f}, {:.2f}, {:.2f}\n'.format( + *mAP11_aos[j, :, i]) + + # prepare results for logger + for idx in range(3): + if i == 0: + postfix = f'{difficulty[idx]}_strict' + else: + postfix = f'{difficulty[idx]}_loose' + prefix = f'KITTI/{curcls_name}' + if mAP11_3d is not None: + ret_dict[f'{prefix}_3D_AP11_{postfix}'] =\ + mAP11_3d[j, idx, i] + if mAP11_bev is not None: + ret_dict[f'{prefix}_BEV_AP11_{postfix}'] =\ + mAP11_bev[j, idx, i] + if mAP11_bbox is not None: + ret_dict[f'{prefix}_2D_AP11_{postfix}'] =\ + mAP11_bbox[j, idx, i] + + # calculate mAP11 over all classes if there are multiple classes + if len(current_classes) > 1: + # prepare results for print + result += ('\nOverall AP11@{}, {}, {}:\n'.format(*difficulty)) + if mAP11_bbox is not None: + mAP11_bbox = mAP11_bbox.mean(axis=0) + result += 'bbox AP11:{:.4f}, {:.4f}, {:.4f}\n'.format( + *mAP11_bbox[:, 0]) + if mAP11_bev is not None: + mAP11_bev = mAP11_bev.mean(axis=0) + result += 'bev AP11:{:.4f}, {:.4f}, {:.4f}\n'.format( + *mAP11_bev[:, 0]) + if mAP11_3d is not None: + mAP11_3d = mAP11_3d.mean(axis=0) + result += '3d AP11:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAP11_3d[:, + 0]) + if compute_aos: + mAP11_aos = mAP11_aos.mean(axis=0) + result += 'aos AP11:{:.2f}, {:.2f}, {:.2f}\n'.format( + *mAP11_aos[:, 0]) + # prepare results for logger + for idx in range(3): + postfix = f'{difficulty[idx]}' + if mAP11_3d is not None: + ret_dict[f'KITTI/Overall_3D_AP11_{postfix}'] = mAP11_3d[idx, 0] + if mAP11_bev is not None: + ret_dict[f'KITTI/Overall_BEV_AP11_{postfix}'] =\ + mAP11_bev[idx, 0] + if mAP11_bbox is not None: + ret_dict[f'KITTI/Overall_2D_AP11_{postfix}'] =\ + mAP11_bbox[idx, 0] + + # Calculate AP40 + result += '\n----------- AP40 Results ------------\n\n' + for j, curcls in enumerate(current_classes): + # mAP threshold array: [num_minoverlap, metric, class] + # mAP result: [num_class, num_diff, num_minoverlap] + curcls_name = class_to_name[curcls] + for i in range(min_overlaps.shape[0]): + # prepare results for print + result += ('{} AP40@{:.2f}, {:.2f}, {:.2f}:\n'.format( + curcls_name, *min_overlaps[i, :, j])) + if mAP40_bbox is not None: + result += 'bbox AP40:{:.4f}, {:.4f}, {:.4f}\n'.format( + *mAP40_bbox[j, :, i]) + if mAP40_bev is not None: + result += 'bev AP40:{:.4f}, {:.4f}, {:.4f}\n'.format( + *mAP40_bev[j, :, i]) + if mAP40_3d is not None: + result += '3d AP40:{:.4f}, {:.4f}, {:.4f}\n'.format( + *mAP40_3d[j, :, i]) if compute_aos: - result += 'aos AP:{:.2f}, {:.2f}, {:.2f}\n'.format( - *mAPaos[j, :, i]) + result += 'aos AP40:{:.2f}, {:.2f}, {:.2f}\n'.format( + *mAP40_aos[j, :, i]) # prepare results for logger for idx in range(3): @@ -738,39 +831,48 @@ def kitti_eval(gt_annos, else: postfix = f'{difficulty[idx]}_loose' prefix = f'KITTI/{curcls_name}' - if mAP3d is not None: - ret_dict[f'{prefix}_3D_{postfix}'] = mAP3d[j, idx, i] - if mAPbev is not None: - ret_dict[f'{prefix}_BEV_{postfix}'] = mAPbev[j, idx, i] - if mAPbbox is not None: - ret_dict[f'{prefix}_2D_{postfix}'] = mAPbbox[j, idx, i] - - # calculate mAP over all classes if there are multiple classes + if mAP40_3d is not None: + ret_dict[f'{prefix}_3D_AP40_{postfix}'] =\ + mAP40_3d[j, idx, i] + if mAP40_bev is not None: + ret_dict[f'{prefix}_BEV_AP40_{postfix}'] =\ + mAP40_bev[j, idx, i] + if mAP40_bbox is not None: + ret_dict[f'{prefix}_2D_AP40_{postfix}'] =\ + mAP40_bbox[j, idx, i] + + # calculate mAP40 over all classes if there are multiple classes if len(current_classes) > 1: # prepare results for print - result += ('\nOverall AP@{}, {}, {}:\n'.format(*difficulty)) - if mAPbbox is not None: - mAPbbox = mAPbbox.mean(axis=0) - result += 'bbox AP:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAPbbox[:, 0]) - if mAPbev is not None: - mAPbev = mAPbev.mean(axis=0) - result += 'bev AP:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAPbev[:, 0]) - if mAP3d is not None: - mAP3d = mAP3d.mean(axis=0) - result += '3d AP:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAP3d[:, 0]) + result += ('\nOverall AP40@{}, {}, {}:\n'.format(*difficulty)) + if mAP40_bbox is not None: + mAP40_bbox = mAP40_bbox.mean(axis=0) + result += 'bbox AP40:{:.4f}, {:.4f}, {:.4f}\n'.format( + *mAP40_bbox[:, 0]) + if mAP40_bev is not None: + mAP40_bev = mAP40_bev.mean(axis=0) + result += 'bev AP40:{:.4f}, {:.4f}, {:.4f}\n'.format( + *mAP40_bev[:, 0]) + if mAP40_3d is not None: + mAP40_3d = mAP40_3d.mean(axis=0) + result += '3d AP40:{:.4f}, {:.4f}, {:.4f}\n'.format(*mAP40_3d[:, + 0]) if compute_aos: - mAPaos = mAPaos.mean(axis=0) - result += 'aos AP:{:.2f}, {:.2f}, {:.2f}\n'.format(*mAPaos[:, 0]) + mAP40_aos = mAP40_aos.mean(axis=0) + result += 'aos AP40:{:.2f}, {:.2f}, {:.2f}\n'.format( + *mAP40_aos[:, 0]) # prepare results for logger for idx in range(3): postfix = f'{difficulty[idx]}' - if mAP3d is not None: - ret_dict[f'KITTI/Overall_3D_{postfix}'] = mAP3d[idx, 0] - if mAPbev is not None: - ret_dict[f'KITTI/Overall_BEV_{postfix}'] = mAPbev[idx, 0] - if mAPbbox is not None: - ret_dict[f'KITTI/Overall_2D_{postfix}'] = mAPbbox[idx, 0] + if mAP40_3d is not None: + ret_dict[f'KITTI/Overall_3D_AP40_{postfix}'] = mAP40_3d[idx, 0] + if mAP40_bev is not None: + ret_dict[f'KITTI/Overall_BEV_AP40_{postfix}'] =\ + mAP40_bev[idx, 0] + if mAP40_bbox is not None: + ret_dict[f'KITTI/Overall_2D_AP40_{postfix}'] =\ + mAP40_bbox[idx, 0] return result, ret_dict diff --git a/mmdet3d/version.py b/mmdet3d/version.py index a0c9dcec27..9efec6a926 100644 --- a/mmdet3d/version.py +++ b/mmdet3d/version.py @@ -1,6 +1,6 @@ # Copyright (c) Open-MMLab. All rights reserved. -__version__ = '0.16.0' +__version__ = '0.17.0' short_version = __version__ diff --git a/requirements/docs.txt b/requirements/docs.txt index fece607fe6..40f8c9ff25 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,6 +1,8 @@ +docutils==0.16.0 m2r +myst-parser opencv-python -recommonmark -sphinx +sphinx==4.0.2 sphinx_markdown_tables -sphinx_rtd_theme +sphinx_rtd_theme==0.5.2 +torch diff --git a/setup.py b/setup.py index e46739a631..07fc668229 100644 --- a/setup.py +++ b/setup.py @@ -196,7 +196,7 @@ def add_mim_extention(): 'for general 3D object detection.'), long_description=readme(), long_description_content_type='text/markdown', - author='OpenMMLab', + author='MMDetection3D Contributors', author_email='zwwdev@gmail.com', keywords='computer vision, 3D object detection', url='https://github.com/open-mmlab/mmdetection3d', diff --git a/tests/test_metrics/test_kitti_eval.py b/tests/test_metrics/test_kitti_eval.py index 7405e6b0a5..7447cebaa6 100644 --- a/tests/test_metrics/test_kitti_eval.py +++ b/tests/test_metrics/test_kitti_eval.py @@ -83,31 +83,49 @@ def test_do_eval(): [[0.5, 0.5, 0.7], [0.25, 0.25, 0.5], [0.25, 0.25, 0.5]]]) eval_types = ['bbox', 'bev', '3d', 'aos'] - mAP_bbox, mAP_bev, mAP_3d, mAP_aos = do_eval([gt_anno], [dt_anno], + mAP11_bbox, mAP11_bev, mAP11_3d, mAP11_aos, mAP40_bbox,\ + mAP40_bev, mAP40_3d, mAP40_aos = do_eval([gt_anno], [dt_anno], current_classes, min_overlaps, eval_types) - expected_mAP_bbox = np.array([[[0., 0.], [9.09090909, 9.09090909], - [9.09090909, 9.09090909]], - [[0., 0.], [9.09090909, 9.09090909], - [9.09090909, 9.09090909]], - [[0., 0.], [9.09090909, 9.09090909], - [9.09090909, 9.09090909]]]) - expected_mAP_bev = np.array([[[0., 0.], [0., 0.], [0., 0.]], - [[0., 0.], [0., 0.], [0., 0.]], - [[0., 0.], [0., 0.], [0., 0.]]]) - expected_mAP_3d = np.array([[[0., 0.], [0., 0.], [0., 0.]], - [[0., 0.], [0., 0.], [0., 0.]], - [[0., 0.], [0., 0.], [0., 0.]]]) - expected_mAP_aos = np.array([[[0., 0.], [0.55020816, 0.55020816], - [0.55020816, 0.55020816]], - [[0., 0.], [8.36633862, 8.36633862], - [8.36633862, 8.36633862]], - [[0., 0.], [8.63476893, 8.63476893], - [8.63476893, 8.63476893]]]) - assert np.allclose(mAP_bbox, expected_mAP_bbox) - assert np.allclose(mAP_bev, expected_mAP_bev) - assert np.allclose(mAP_3d, expected_mAP_3d) - assert np.allclose(mAP_aos, expected_mAP_aos) + expected_mAP11_bbox = np.array([[[0., 0.], [9.09090909, 9.09090909], + [9.09090909, 9.09090909]], + [[0., 0.], [9.09090909, 9.09090909], + [9.09090909, 9.09090909]], + [[0., 0.], [9.09090909, 9.09090909], + [9.09090909, 9.09090909]]]) + expected_mAP40_bbox = np.array([[[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [2.5, 2.5], [2.5, 2.5]]]) + expected_mAP11_bev = np.array([[[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [0., 0.], [0., 0.]]]) + expected_mAP40_bev = np.array([[[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [0., 0.], [0., 0.]]]) + expected_mAP11_3d = np.array([[[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [0., 0.], [0., 0.]]]) + expected_mAP40_3d = np.array([[[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [0., 0.], [0., 0.]]]) + expected_mAP11_aos = np.array([[[0., 0.], [0.55020816, 0.55020816], + [0.55020816, 0.55020816]], + [[0., 0.], [8.36633862, 8.36633862], + [8.36633862, 8.36633862]], + [[0., 0.], [8.63476893, 8.63476893], + [8.63476893, 8.63476893]]]) + expected_mAP40_aos = np.array([[[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [0., 0.], [0., 0.]], + [[0., 0.], [1.58140643, 1.58140643], + [1.58140643, 1.58140643]]]) + assert np.allclose(mAP11_bbox, expected_mAP11_bbox) + assert np.allclose(mAP11_bev, expected_mAP11_bev) + assert np.allclose(mAP11_3d, expected_mAP11_3d) + assert np.allclose(mAP11_aos, expected_mAP11_aos) + assert np.allclose(mAP40_bbox, expected_mAP40_bbox) + assert np.allclose(mAP40_bev, expected_mAP40_bev) + assert np.allclose(mAP40_3d, expected_mAP40_3d) + assert np.allclose(mAP40_aos, expected_mAP40_aos) def test_kitti_eval(): @@ -183,8 +201,14 @@ def test_kitti_eval(): current_classes = [1, 2, 0] result, ret_dict = kitti_eval([gt_anno], [dt_anno], current_classes) - assert np.isclose(ret_dict['KITTI/Overall_2D_moderate'], 9.090909090909092) - assert np.isclose(ret_dict['KITTI/Overall_2D_hard'], 9.090909090909092) + assert np.isclose(ret_dict['KITTI/Overall_2D_AP11_moderate'], + 9.090909090909092) + assert np.isclose(ret_dict['KITTI/Overall_2D_AP11_hard'], + 9.090909090909092) + assert np.isclose(ret_dict['KITTI/Overall_2D_AP40_moderate'], + 0.8333333333333334) + assert np.isclose(ret_dict['KITTI/Overall_2D_AP40_hard'], + 0.8333333333333334) def test_eval_class(): From 66f0c0726a6d3c0d217ec219ab787e0838168ba7 Mon Sep 17 00:00:00 2001 From: ChaimZhu Date: Wed, 15 Sep 2021 16:35:35 +0800 Subject: [PATCH 28/51] [Feature] add smoke backbone neck (#939) * add smoke detecotor and it's backbone and neck * typo fix * fix typo * add docstring * fix typo * fix comments * fix comments * fix comments * fix typo * fix typo * fix * fix typo * fix docstring * refine feature * fix typo * use Basemodule in Neck --- mmdet3d/models/backbones/__init__.py | 3 +- mmdet3d/models/backbones/dla.py | 444 +++++++++++++++++++++ mmdet3d/models/necks/__init__.py | 3 +- mmdet3d/models/necks/dla_neck.py | 231 +++++++++++ tests/test_models/test_backbones.py | 23 ++ tests/test_models/test_necks/test_necks.py | 42 ++ 6 files changed, 744 insertions(+), 2 deletions(-) create mode 100644 mmdet3d/models/backbones/dla.py create mode 100644 mmdet3d/models/necks/dla_neck.py diff --git a/mmdet3d/models/backbones/__init__.py b/mmdet3d/models/backbones/__init__.py index 26c432ccfc..9403bd72c5 100644 --- a/mmdet3d/models/backbones/__init__.py +++ b/mmdet3d/models/backbones/__init__.py @@ -1,6 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.backbones import SSDVGG, HRNet, ResNet, ResNetV1d, ResNeXt from .dgcnn import DGCNNBackbone +from .dla import DLANet from .multi_backbone import MultiBackbone from .nostem_regnet import NoStemRegNet from .pointnet2_sa_msg import PointNet2SAMSG @@ -10,5 +11,5 @@ __all__ = [ 'ResNet', 'ResNetV1d', 'ResNeXt', 'SSDVGG', 'HRNet', 'NoStemRegNet', 'SECOND', 'DGCNNBackbone', 'PointNet2SASSG', 'PointNet2SAMSG', - 'MultiBackbone' + 'MultiBackbone', 'DLANet' ] diff --git a/mmdet3d/models/backbones/dla.py b/mmdet3d/models/backbones/dla.py new file mode 100644 index 0000000000..7005020b67 --- /dev/null +++ b/mmdet3d/models/backbones/dla.py @@ -0,0 +1,444 @@ +import torch +import warnings +from mmcv.cnn import build_conv_layer, build_norm_layer +from mmcv.runner import BaseModule +from torch import nn + +from mmdet.models.builder import BACKBONES + + +def dla_build_norm_layer(cfg, num_features): + """Build normalization layer specially designed for DLANet. + + Args: + cfg (dict): The norm layer config, which should contain: + + - type (str): Layer type. + - layer args: Args needed to instantiate a norm layer. + - requires_grad (bool, optional): Whether stop gradient updates. + num_features (int): Number of input channels. + + + Returns: + Function: Build normalization layer in mmcv. + """ + cfg_ = cfg.copy() + if cfg_['type'] == 'GN': + if num_features % 32 == 0: + return build_norm_layer(cfg_, num_features) + else: + assert 'num_groups' in cfg_ + cfg_['num_groups'] = cfg_['num_groups'] // 2 + return build_norm_layer(cfg_, num_features) + else: + return build_norm_layer(cfg_, num_features) + + +class BasicBlock(BaseModule): + """BasicBlock in DLANet. + + Args: + in_channels (int): Input feature channel. + out_channels (int): Output feature channel. + norm_cfg (dict): Dictionary to construct and config + norm layer. + conv_cfg (dict): Dictionary to construct and config + conv layer. + stride (int, optional): Conv stride. Default: 1. + dilation (int, optional): Conv dilation. Default: 1. + init_cfg (dict, optional): Initialization config. + Default: None. + """ + + def __init__(self, + in_channels, + out_channels, + norm_cfg, + conv_cfg, + stride=1, + dilation=1, + init_cfg=None): + super(BasicBlock, self).__init__(init_cfg) + self.conv1 = build_conv_layer( + conv_cfg, + in_channels, + out_channels, + 3, + stride=stride, + padding=dilation, + dilation=dilation, + bias=False) + self.norm1 = dla_build_norm_layer(norm_cfg, out_channels)[1] + self.relu = nn.ReLU(inplace=True) + self.conv2 = build_conv_layer( + conv_cfg, + out_channels, + out_channels, + 3, + stride=1, + padding=dilation, + dilation=dilation, + bias=False) + self.norm2 = dla_build_norm_layer(norm_cfg, out_channels)[1] + self.stride = stride + + def forward(self, x, identity=None): + """Forward function.""" + + if identity is None: + identity = x + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.norm2(out) + out += identity + out = self.relu(out) + + return out + + +class Root(BaseModule): + """Root in DLANet. + + Args: + in_channels (int): Input feature channel. + out_channels (int): Output feature channel. + norm_cfg (dict): Dictionary to construct and config + norm layer. + conv_cfg (dict): Dictionary to construct and config + conv layer. + kernel_size (int): Size of convolution kernel. + add_identity (bool): Whether to add identity in root. + init_cfg (dict, optional): Initialization config. + Default: None. + """ + + def __init__(self, + in_channels, + out_channels, + norm_cfg, + conv_cfg, + kernel_size, + add_identity, + init_cfg=None): + super(Root, self).__init__(init_cfg) + self.conv = build_conv_layer( + conv_cfg, + in_channels, + out_channels, + 1, + stride=1, + padding=(kernel_size - 1) // 2, + bias=False) + self.norm = dla_build_norm_layer(norm_cfg, out_channels)[1] + self.relu = nn.ReLU(inplace=True) + self.add_identity = add_identity + + def forward(self, feat_list): + """Forward function. + + Args: + feat_list (list[torch.Tensor]): Output features from + multiple layers. + """ + children = feat_list + x = self.conv(torch.cat(feat_list, 1)) + x = self.norm(x) + if self.add_identity: + x += children[0] + x = self.relu(x) + + return x + + +class Tree(BaseModule): + """Tree in DLANet. + + Args: + levels (int): The level of the tree. + block (nn.Module): The block module in tree. + in_channels: Input feature channel. + out_channels: Output feature channel. + norm_cfg (dict): Dictionary to construct and config + norm layer. + conv_cfg (dict): Dictionary to construct and config + conv layer. + stride (int, optional): Convolution stride. + Default: 1. + level_root (bool, optional): whether belongs to the + root layer. + root_dim (int, optional): Root input feature channel. + root_kernel_size (int, optional): Size of root + convolution kernel. Default: 1. + dilation (int, optional): Conv dilation. Default: 1. + add_identity (bool, optional): Whether to add + identity in root. Default: False. + init_cfg (dict, optional): Initialization config. + Default: None. + """ + + def __init__(self, + levels, + block, + in_channels, + out_channels, + norm_cfg, + conv_cfg, + stride=1, + level_root=False, + root_dim=None, + root_kernel_size=1, + dilation=1, + add_identity=False, + init_cfg=None): + super(Tree, self).__init__(init_cfg) + if root_dim is None: + root_dim = 2 * out_channels + if level_root: + root_dim += in_channels + if levels == 1: + self.root = Root(root_dim, out_channels, norm_cfg, conv_cfg, + root_kernel_size, add_identity) + self.tree1 = block( + in_channels, + out_channels, + norm_cfg, + conv_cfg, + stride, + dilation=dilation) + self.tree2 = block( + out_channels, + out_channels, + norm_cfg, + conv_cfg, + 1, + dilation=dilation) + else: + self.tree1 = Tree( + levels - 1, + block, + in_channels, + out_channels, + norm_cfg, + conv_cfg, + stride, + root_dim=None, + root_kernel_size=root_kernel_size, + dilation=dilation, + add_identity=add_identity) + self.tree2 = Tree( + levels - 1, + block, + out_channels, + out_channels, + norm_cfg, + conv_cfg, + root_dim=root_dim + out_channels, + root_kernel_size=root_kernel_size, + dilation=dilation, + add_identity=add_identity) + self.level_root = level_root + self.root_dim = root_dim + self.downsample = None + self.project = None + self.levels = levels + if stride > 1: + self.downsample = nn.MaxPool2d(stride, stride=stride) + if in_channels != out_channels: + self.project = nn.Sequential( + build_conv_layer( + conv_cfg, + in_channels, + out_channels, + 1, + stride=1, + bias=False), + dla_build_norm_layer(norm_cfg, out_channels)[1]) + + def forward(self, x, identity=None, children=None): + children = [] if children is None else children + bottom = self.downsample(x) if self.downsample else x + identity = self.project(bottom) if self.project else bottom + if self.level_root: + children.append(bottom) + x1 = self.tree1(x, identity) + if self.levels == 1: + x2 = self.tree2(x1) + feat_list = [x2, x1] + children + x = self.root(feat_list) + else: + children.append(x1) + x = self.tree2(x1, children=children) + return x + + +@BACKBONES.register_module() +class DLANet(BaseModule): + r"""`DLA backbone `_. + + Args: + depth (int): Depth of DLA. Default: 34. + in_channels (int, optional): Number of input image channels. + Default: 3. + norm_cfg (dict, optional): Dictionary to construct and config + norm layer. Default: None. + conv_cfg (dict, optional): Dictionary to construct and config + conv layer. Default: None. + layer_with_level_root (list[bool], optional): Whether to apply + level_root in each DLA layer, this is only used for + tree levels. Default: (False, True, True, True). + with_identity_root (bool, optional): Whether to add identity + in root layer. Default: False. + pretrained (str, optional): model pretrained path. + Default: None. + init_cfg (dict or list[dict], optional): Initialization + config dict. Default: None + """ + arch_settings = { + 34: (BasicBlock, (1, 1, 1, 2, 2, 1), (16, 32, 64, 128, 256, 512)), + } + + def __init__(self, + depth, + in_channels=3, + out_indices=(0, 1, 2, 3, 4, 5), + frozen_stages=-1, + norm_cfg=None, + conv_cfg=None, + layer_with_level_root=(False, True, True, True), + with_identity_root=False, + pretrained=None, + init_cfg=None): + super(DLANet, self).__init__(init_cfg) + if depth not in self.arch_settings: + raise KeyError(f'invalida depth {depth} for DLA') + + assert not (init_cfg and pretrained), \ + 'init_cfg and pretrained cannot be setting at the same time' + if isinstance(pretrained, str): + warnings.warn('DeprecationWarning: pretrained is a deprecated, ' + 'please use "init_cfg" instead') + self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) + elif pretrained is None: + if init_cfg is None: + self.init_cfg = [ + dict(type='Kaiming', layer='Conv2d'), + dict( + type='Constant', + val=1, + layer=['_BatchNorm', 'GroupNorm']) + ] + + block, levels, channels = self.arch_settings[depth] + self.channels = channels + self.num_levels = len(levels) + self.frozen_stages = frozen_stages + self.out_indices = out_indices + assert max(out_indices) < self.num_levels + self.base_layer = nn.Sequential( + build_conv_layer( + conv_cfg, + in_channels, + channels[0], + 7, + stride=1, + padding=3, + bias=False), + dla_build_norm_layer(norm_cfg, channels[0])[1], + nn.ReLU(inplace=True)) + + # DLANet first uses two conv layers then uses several + # Tree layers + for i in range(2): + level_layer = self._make_conv_level( + channels[0], + channels[i], + levels[i], + norm_cfg, + conv_cfg, + stride=i + 1) + layer_name = f'level{i}' + self.add_module(layer_name, level_layer) + + for i in range(2, self.num_levels): + dla_layer = Tree( + levels[i], + block, + channels[i - 1], + channels[i], + norm_cfg, + conv_cfg, + 2, + level_root=layer_with_level_root[i - 2], + add_identity=with_identity_root) + layer_name = f'level{i}' + self.add_module(layer_name, dla_layer) + + self._freeze_stages() + + def _make_conv_level(self, + in_channels, + out_channels, + num_convs, + norm_cfg, + conv_cfg, + stride=1, + dilation=1): + """Conv modules. + + Args: + in_channels (int): Input feature channel. + out_channels (int): Output feature channel. + num_convs (int): Number of Conv module. + norm_cfg (dict): Dictionary to construct and config + norm layer. + conv_cfg (dict): Dictionary to construct and config + conv layer. + stride (int, optional): Conv stride. Default: 1. + dilation (int, optional): Conv dilation. Default: 1. + """ + modules = [] + for i in range(num_convs): + modules.extend([ + build_conv_layer( + conv_cfg, + in_channels, + out_channels, + 3, + stride=stride if i == 0 else 1, + padding=dilation, + bias=False, + dilation=dilation), + dla_build_norm_layer(norm_cfg, out_channels)[1], + nn.ReLU(inplace=True) + ]) + in_channels = out_channels + return nn.Sequential(*modules) + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.base_layer.eval() + for param in self.base_layer.parameters(): + param.requires_grad = False + + for i in range(2): + m = getattr(self, f'level{i}') + m.eval() + for param in m.parameters(): + param.requires_grad = False + + for i in range(1, self.frozen_stages + 1): + m = getattr(self, f'level{i+1}') + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def forward(self, x): + outs = [] + x = self.base_layer(x) + for i in range(self.num_levels): + x = getattr(self, 'level{}'.format(i))(x) + if i in self.out_indices: + outs.append(x) + return tuple(outs) diff --git a/mmdet3d/models/necks/__init__.py b/mmdet3d/models/necks/__init__.py index 9752a8b490..de763c956d 100644 --- a/mmdet3d/models/necks/__init__.py +++ b/mmdet3d/models/necks/__init__.py @@ -1,6 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.necks.fpn import FPN +from .dla_neck import DLANeck from .imvoxel_neck import OutdoorImVoxelNeck from .second_fpn import SECONDFPN -__all__ = ['FPN', 'SECONDFPN', 'OutdoorImVoxelNeck'] +__all__ = ['FPN', 'SECONDFPN', 'OutdoorImVoxelNeck', 'DLANeck'] diff --git a/mmdet3d/models/necks/dla_neck.py b/mmdet3d/models/necks/dla_neck.py new file mode 100644 index 0000000000..30f1889b67 --- /dev/null +++ b/mmdet3d/models/necks/dla_neck.py @@ -0,0 +1,231 @@ +import math +import numpy as np +from mmcv.cnn import ConvModule, build_conv_layer +from mmcv.runner import BaseModule +from torch import nn as nn + +from mmdet.models.builder import NECKS + + +def fill_up_weights(up): + """Simulated bilinear upsampling kernel. + + Args: + up (nn.Module): ConvTranspose2d module. + """ + w = up.weight.data + f = math.ceil(w.size(2) / 2) + c = (2 * f - 1 - f % 2) / (2. * f) + for i in range(w.size(2)): + for j in range(w.size(3)): + w[0, 0, i, j] = \ + (1 - math.fabs(i / f - c)) * (1 - math.fabs(j / f - c)) + for c in range(1, w.size(0)): + w[c, 0, :, :] = w[0, 0, :, :] + + +class IDAUpsample(BaseModule): + """Iterative Deep Aggregation (IDA) Upsampling module to upsample features + of different scales to a similar scale. + + Args: + out_channels (int): Number of output channels for DeformConv. + in_channels (List[int]): List of input channels of multi-scale + feature maps. + kernel_sizes (List[int]): List of size of the convolving + kernel of different scales. + norm_cfg (dict, optional): Config dict for normalization layer. + Default: None. + use_dcn (bool, optional): If True, use DCNv2. Default: True. + """ + + def __init__( + self, + out_channels, + in_channels, + kernel_sizes, + norm_cfg=None, + use_dcn=True, + init_cfg=None, + ): + super(IDAUpsample, self).__init__(init_cfg) + self.use_dcn = use_dcn + self.projs = nn.ModuleList() + self.ups = nn.ModuleList() + self.nodes = nn.ModuleList() + + for i in range(1, len(in_channels)): + in_channel = in_channels[i] + up_kernel_size = int(kernel_sizes[i]) + proj = ConvModule( + in_channel, + out_channels, + 3, + padding=1, + bias=True, + conv_cfg=dict(type='DCNv2') if self.use_dcn else None, + norm_cfg=norm_cfg) + node = ConvModule( + out_channels, + out_channels, + 3, + padding=1, + bias=True, + conv_cfg=dict(type='DCNv2') if self.use_dcn else None, + norm_cfg=norm_cfg) + up = build_conv_layer( + dict(type='deconv'), + out_channels, + out_channels, + up_kernel_size * 2, + stride=up_kernel_size, + padding=up_kernel_size // 2, + output_padding=0, + groups=out_channels, + bias=False) + + self.projs.append(proj) + self.ups.append(up) + self.nodes.append(node) + + def forward(self, mlvl_features, start_level, end_level): + """Forward function. + + Args: + mlvl_features (list[torch.Tensor]): Features from multiple layers. + start_level (int): Start layer for feature upsampling. + end_level (int): End layer for feature upsampling. + """ + for i in range(start_level, end_level - 1): + upsample = self.ups[i - start_level] + project = self.projs[i - start_level] + mlvl_features[i + 1] = upsample(project(mlvl_features[i + 1])) + node = self.nodes[i - start_level] + mlvl_features[i + 1] = node(mlvl_features[i + 1] + + mlvl_features[i]) + + +class DLAUpsample(BaseModule): + """Deep Layer Aggregation (DLA) Upsampling module for different scales + feature extraction, upsampling and fusion, It consists of groups of + IDAupsample modules. + + Args: + start_level (int): The start layer. + channels (List[int]): List of input channels of multi-scale + feature maps. + scales(List[int]): List of scale of different layers' feature. + in_channels (NoneType, optional): List of input channels of + different scales. Default: None. + norm_cfg (dict, optional): Config dict for normalization layer. + Default: None. + use_dcn (bool, optional): Whether to use dcn in IDAup module. + Default: True. + """ + + def __init__(self, + start_level, + channels, + scales, + in_channels=None, + norm_cfg=None, + use_dcn=True, + init_cfg=None): + super(DLAUpsample, self).__init__(init_cfg) + self.start_level = start_level + if in_channels is None: + in_channels = channels + self.channels = channels + channels = list(channels) + scales = np.array(scales, dtype=int) + for i in range(len(channels) - 1): + j = -i - 2 + setattr( + self, 'ida_{}'.format(i), + IDAUpsample(channels[j], in_channels[j:], + scales[j:] // scales[j], norm_cfg, use_dcn)) + scales[j + 1:] = scales[j] + in_channels[j + 1:] = [channels[j] for _ in channels[j + 1:]] + + def forward(self, mlvl_features): + """Forward function. + + Args: + mlvl_features(list[torch.Tensor]): Features from multi-scale + layers. + + Returns: + tuple[torch.Tensor]: Up-sampled features of different layers. + """ + outs = [mlvl_features[-1]] + for i in range(len(mlvl_features) - self.start_level - 1): + ida = getattr(self, 'ida_{}'.format(i)) + ida(mlvl_features, len(mlvl_features) - i - 2, len(mlvl_features)) + outs.insert(0, mlvl_features[-1]) + return outs + + +@NECKS.register_module() +class DLANeck(BaseModule): + """DLA Neck. + + Args: + in_channels (list[int], optional): List of input channels + of multi-scale feature map. + start_level (int, optioanl): The scale level where upsampling + starts. Default: 2. + end_level (int, optional): The scale level where upsampling + ends. Default: 5. + norm_cfg (dict, optional): Config dict for normalization + layer. Default: None. + use_dcn (bool, optional): Whether to use dcn in IDAup module. + Default: True. + """ + + def __init__(self, + in_channels=[16, 32, 64, 128, 256, 512], + start_level=2, + end_level=5, + norm_cfg=None, + use_dcn=True, + init_cfg=None): + super(DLANeck, self).__init__(init_cfg) + self.start_level = start_level + self.end_level = end_level + scales = [2**i for i in range(len(in_channels[self.start_level:]))] + self.dla_up = DLAUpsample( + start_level=self.start_level, + channels=in_channels[self.start_level:], + scales=scales, + norm_cfg=norm_cfg, + use_dcn=use_dcn) + self.ida_up = IDAUpsample( + in_channels[self.start_level], + in_channels[self.start_level:self.end_level], + [2**i for i in range(self.end_level - self.start_level)], norm_cfg, + use_dcn) + + def forward(self, x): + mlvl_features = [x[i] for i in range(len(x))] + mlvl_features = self.dla_up(mlvl_features) + outs = [] + for i in range(self.end_level - self.start_level): + outs.append(mlvl_features[i].clone()) + self.ida_up(outs, 0, len(outs)) + return outs[-1] + + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.ConvTranspose2d): + # In order to be consistent with the source code, + # reset the ConvTranspose2d initialization parameters + m.reset_parameters() + # Simulated bilinear upsampling kernel + fill_up_weights(m) + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Conv2d): + # In order to be consistent with the source code, + # reset the Conv2d initialization parameters + m.reset_parameters() diff --git a/tests/test_models/test_backbones.py b/tests/test_models/test_backbones.py index 5c9f5edfe8..392e0ec4c0 100644 --- a/tests/test_models/test_backbones.py +++ b/tests/test_models/test_backbones.py @@ -330,3 +330,26 @@ def test_dgcnn_gf(): assert gf_points[2].shape == torch.Size([1, 100, 64]) assert gf_points[3].shape == torch.Size([1, 100, 64]) assert fa_points.shape == torch.Size([1, 100, 1216]) + + +def test_dla_net(): + # test DLANet used in SMOKE + # test list config + cfg = dict( + type='DLANet', + depth=34, + in_channels=3, + norm_cfg=dict(type='GN', num_groups=32)) + + img = torch.randn((4, 3, 32, 32)) + self = build_backbone(cfg) + self.init_weights() + + results = self(img) + assert len(results) == 6 + assert results[0].shape == torch.Size([4, 16, 32, 32]) + assert results[1].shape == torch.Size([4, 32, 16, 16]) + assert results[2].shape == torch.Size([4, 64, 8, 8]) + assert results[3].shape == torch.Size([4, 128, 4, 4]) + assert results[4].shape == torch.Size([4, 256, 2, 2]) + assert results[5].shape == torch.Size([4, 512, 1, 1]) diff --git a/tests/test_models/test_necks/test_necks.py b/tests/test_models/test_necks/test_necks.py index 1e48493cb8..3ea3db1ce4 100644 --- a/tests/test_models/test_necks/test_necks.py +++ b/tests/test_models/test_necks/test_necks.py @@ -57,3 +57,45 @@ def test_imvoxel_neck(): inputs = torch.rand([1, 64, 216, 248, 12], device='cuda') outputs = neck(inputs) assert outputs[0].shape == (1, 256, 248, 216) + + +def test_dla_neck(): + + s = 32 + in_channels = [16, 32, 64, 128, 256, 512] + feat_sizes = [s // 2**i for i in range(6)] # [32, 16, 8, 4, 2, 1] + + if torch.cuda.is_available(): + # Test DLA Neck with DCNv2 on GPU + neck_cfg = dict( + type='DLANeck', + in_channels=[16, 32, 64, 128, 256, 512], + start_level=2, + end_level=5, + norm_cfg=dict(type='GN', num_groups=32)) + neck = build_neck(neck_cfg) + neck.init_weights() + neck.cuda() + feats = [ + torch.rand(4, in_channels[i], feat_sizes[i], feat_sizes[i]).cuda() + for i in range(len(in_channels)) + ] + outputs = neck(feats) + assert outputs.shape == (4, 64, 8, 8) + else: + # Test DLA Neck without DCNv2 on CPU + neck_cfg = dict( + type='DLANeck', + in_channels=[16, 32, 64, 128, 256, 512], + start_level=2, + end_level=5, + norm_cfg=dict(type='GN', num_groups=32), + use_dcn=False) + neck = build_neck(neck_cfg) + neck.init_weights() + feats = [ + torch.rand(4, in_channels[i], feat_sizes[i], feat_sizes[i]) + for i in range(len(in_channels)) + ] + outputs = neck(feats) + assert outputs.shape == (4, 64, 8, 8) From 0b26a9a93bf8ce9e78c6ed15f573f1b4717ab42d Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 15 Sep 2021 21:26:00 +0800 Subject: [PATCH 29/51] [Refactor] Refactor the transformation from image to camera coordinates (#938) * Refactor points_img2cam * Refine docstring * Support array converter and add unit tests --- mmdet3d/core/bbox/__init__.py | 5 +-- mmdet3d/core/bbox/structures/__init__.py | 7 ++-- mmdet3d/core/bbox/structures/utils.py | 34 +++++++++++++++++++ .../models/dense_heads/fcos_mono3d_head.py | 10 ++++-- tests/test_utils/test_utils.py | 12 ++++++- 5 files changed, 60 insertions(+), 8 deletions(-) diff --git a/mmdet3d/core/bbox/__init__.py b/mmdet3d/core/bbox/__init__.py index dd06b31c2f..8c66630682 100644 --- a/mmdet3d/core/bbox/__init__.py +++ b/mmdet3d/core/bbox/__init__.py @@ -12,7 +12,8 @@ from .structures import (BaseInstance3DBoxes, Box3DMode, CameraInstance3DBoxes, Coord3DMode, DepthInstance3DBoxes, LiDARInstance3DBoxes, get_box_type, limit_period, - mono_cam_box2vis, points_cam2img, xywhr2xyxyr) + mono_cam_box2vis, points_cam2img, points_img2cam, + xywhr2xyxyr) from .transforms import bbox3d2result, bbox3d2roi, bbox3d_mapping_back __all__ = [ @@ -25,5 +26,5 @@ 'LiDARInstance3DBoxes', 'CameraInstance3DBoxes', 'bbox3d2roi', 'bbox3d2result', 'DepthInstance3DBoxes', 'BaseInstance3DBoxes', 'bbox3d_mapping_back', 'xywhr2xyxyr', 'limit_period', 'points_cam2img', - 'get_box_type', 'Coord3DMode', 'mono_cam_box2vis' + 'points_img2cam', 'get_box_type', 'Coord3DMode', 'mono_cam_box2vis' ] diff --git a/mmdet3d/core/bbox/structures/__init__.py b/mmdet3d/core/bbox/structures/__init__.py index 58c111ef43..460035a533 100644 --- a/mmdet3d/core/bbox/structures/__init__.py +++ b/mmdet3d/core/bbox/structures/__init__.py @@ -6,12 +6,13 @@ from .depth_box3d import DepthInstance3DBoxes from .lidar_box3d import LiDARInstance3DBoxes from .utils import (get_box_type, get_proj_mat_by_coord_type, limit_period, - mono_cam_box2vis, points_cam2img, rotation_3d_in_axis, - xywhr2xyxyr) + mono_cam_box2vis, points_cam2img, points_img2cam, + rotation_3d_in_axis, xywhr2xyxyr) __all__ = [ 'Box3DMode', 'BaseInstance3DBoxes', 'LiDARInstance3DBoxes', 'CameraInstance3DBoxes', 'DepthInstance3DBoxes', 'xywhr2xyxyr', 'get_box_type', 'rotation_3d_in_axis', 'limit_period', 'points_cam2img', - 'Coord3DMode', 'mono_cam_box2vis', 'get_proj_mat_by_coord_type' + 'points_img2cam', 'Coord3DMode', 'mono_cam_box2vis', + 'get_proj_mat_by_coord_type' ] diff --git a/mmdet3d/core/bbox/structures/utils.py b/mmdet3d/core/bbox/structures/utils.py index 64483506cc..4c61755636 100644 --- a/mmdet3d/core/bbox/structures/utils.py +++ b/mmdet3d/core/bbox/structures/utils.py @@ -213,6 +213,40 @@ def points_cam2img(points_3d, proj_mat, with_depth=False): return point_2d_res +@array_converter(apply_to=('points', 'cam2img')) +def points_img2cam(points, cam2img): + """Project points in image coordinates to camera coordinates. + + Args: + points (torch.Tensor): 2.5D points in 2D images, [N, 3], + 3 corresponds with x, y in the image and depth. + cam2img (torch.Tensor): Camera instrinsic matrix. The shape can be + [3, 3], [3, 4] or [4, 4]. + + Returns: + torch.Tensor: points in 3D space. [N, 3], + 3 corresponds with x, y, z in 3D space. + """ + assert cam2img.shape[0] <= 4 + assert cam2img.shape[1] <= 4 + assert points.shape[1] == 3 + + xys = points[:, :2] + depths = points[:, 2].view(-1, 1) + unnormed_xys = torch.cat([xys * depths, depths], dim=1) + + pad_cam2img = torch.eye(4, dtype=xys.dtype, device=xys.device) + pad_cam2img[:cam2img.shape[0], :cam2img.shape[1]] = cam2img + inv_pad_cam2img = torch.inverse(pad_cam2img).transpose(0, 1) + + # Do operation in homogenous coordinates. + num_points = unnormed_xys.shape[0] + homo_xys = torch.cat([unnormed_xys, xys.new_ones((num_points, 1))], dim=1) + points3D = torch.mm(homo_xys, inv_pad_cam2img)[:, :3] + + return points3D + + def mono_cam_box2vis(cam_box): """This is a post-processing function on the bboxes from Mono-3D task. If we want to perform projection visualization, we need to: diff --git a/mmdet3d/models/dense_heads/fcos_mono3d_head.py b/mmdet3d/models/dense_heads/fcos_mono3d_head.py index c0b15a4581..ba2db7982c 100644 --- a/mmdet3d/models/dense_heads/fcos_mono3d_head.py +++ b/mmdet3d/models/dense_heads/fcos_mono3d_head.py @@ -1,11 +1,13 @@ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch +from logging import warning from mmcv.cnn import Scale from mmcv.runner import force_fp32 from torch import nn as nn -from mmdet3d.core import box3d_multiclass_nms, limit_period, xywhr2xyxyr +from mmdet3d.core import (box3d_multiclass_nms, limit_period, points_img2cam, + xywhr2xyxyr) from mmdet.core import multi_apply from mmdet.models.builder import HEADS, build_loss from .anchor_free_mono3d_head import AnchorFreeMono3DHead @@ -639,7 +641,7 @@ def _get_bboxes_single(self, if rescale: bbox_pred[:, :2] /= bbox_pred[:, :2].new_tensor(scale_factor) pred_center2d = bbox_pred[:, :3].clone() - bbox_pred[:, :3] = self.pts2Dto3D(bbox_pred[:, :3], view) + bbox_pred[:, :3] = points_img2cam(bbox_pred[:, :3], view) mlvl_centers2d.append(pred_center2d) mlvl_bboxes.append(bbox_pred) mlvl_scores.append(scores) @@ -708,6 +710,10 @@ def pts2Dto3D(points, view): torch.Tensor: points in 3D space. [N, 3], 3 corresponds with x, y, z in 3D space. """ + warning.warn('DeprecationWarning: This static method has been moved ' + 'out of this class to mmdet3d/core. The function ' + 'pts2Dto3D will be deprecated.') + assert view.shape[0] <= 4 assert view.shape[1] <= 4 assert points.shape[1] == 3 diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 0aa9732c90..3d8259f7e2 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -3,7 +3,7 @@ import pytest import torch -from mmdet3d.core import array_converter, draw_heatmap_gaussian +from mmdet3d.core import array_converter, draw_heatmap_gaussian, points_img2cam def test_gaussian(): @@ -178,3 +178,13 @@ def test_func_9(container, array_a, array_b=1): with pytest.raises(TypeError): new_array_a, new_array_b = test_func_9(container, [True, np.array([3.0])]) + + +def test_points_img2cam(): + points = torch.tensor([[0.5764, 0.9109, 0.7576], [0.6656, 0.5498, 0.9813]]) + cam2img = torch.tensor([[700., 0., 450., 0.], [0., 700., 200., 0.], + [0., 0., 1., 0.]]) + xyzs = points_img2cam(points, cam2img) + expected_xyzs = torch.tensor([[-0.4864, -0.2155, 0.7576], + [-0.6299, -0.2796, 0.9813]]) + assert torch.allclose(xyzs, expected_xyzs, atol=1e-3) From 911a3333c150c1e816cbc283496770d90688dfe8 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 15 Sep 2021 21:31:40 +0800 Subject: [PATCH 30/51] [Feature] FCOS3D BBox Coder (#940) * FCOS3D BBox Coder * Add unit tests * Change the value from long to float/double * Rename bbox_out as bbox * Add comments to forward returns --- configs/_base_/models/fcos3d.py | 1 + mmdet3d/core/bbox/coders/__init__.py | 3 +- mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py | 126 ++++++++++++++++++ .../models/dense_heads/fcos_mono3d_head.py | 53 +++----- tests/test_utils/test_bbox_coders.py | 83 ++++++++++++ 5 files changed, 232 insertions(+), 34 deletions(-) create mode 100644 mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py diff --git a/configs/_base_/models/fcos3d.py b/configs/_base_/models/fcos3d.py index 1465b81a72..a46ed9cd60 100644 --- a/configs/_base_/models/fcos3d.py +++ b/configs/_base_/models/fcos3d.py @@ -55,6 +55,7 @@ type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + bbox_coder=dict(type='FCOS3DBBoxCoder', code_size=9), norm_on_bbox=True, centerness_on_reg=True, center_sampling=True, diff --git a/mmdet3d/core/bbox/coders/__init__.py b/mmdet3d/core/bbox/coders/__init__.py index 08fbb410b3..0e44042212 100644 --- a/mmdet3d/core/bbox/coders/__init__.py +++ b/mmdet3d/core/bbox/coders/__init__.py @@ -3,6 +3,7 @@ from .anchor_free_bbox_coder import AnchorFreeBBoxCoder from .centerpoint_bbox_coders import CenterPointBBoxCoder from .delta_xyzwhlr_bbox_coder import DeltaXYZWLHRBBoxCoder +from .fcos3d_bbox_coder import FCOS3DBBoxCoder from .groupfree3d_bbox_coder import GroupFree3DBBoxCoder from .partial_bin_based_bbox_coder import PartialBinBasedBBoxCoder from .point_xyzwhlr_bbox_coder import PointXYZWHLRBBoxCoder @@ -10,5 +11,5 @@ __all__ = [ 'build_bbox_coder', 'DeltaXYZWLHRBBoxCoder', 'PartialBinBasedBBoxCoder', 'CenterPointBBoxCoder', 'AnchorFreeBBoxCoder', 'GroupFree3DBBoxCoder', - 'PointXYZWHLRBBoxCoder' + 'PointXYZWHLRBBoxCoder', 'FCOS3DBBoxCoder' ] diff --git a/mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py b/mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py new file mode 100644 index 0000000000..7245a57e09 --- /dev/null +++ b/mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py @@ -0,0 +1,126 @@ +import numpy as np +import torch + +from mmdet.core.bbox import BaseBBoxCoder +from mmdet.core.bbox.builder import BBOX_CODERS +from ..structures import limit_period + + +@BBOX_CODERS.register_module() +class FCOS3DBBoxCoder(BaseBBoxCoder): + """Bounding box coder for FCOS3D. + + Args: + base_depths (tuple[tuple[float]]): Depth references for decode box + depth. Defaults to None. + base_dims (tuple[tuple[float]]): Dimension references for decode box + dimension. Defaults to None. + code_size (int): The dimension of boxes to be encoded. Defaults to 7. + norm_on_bbox (bool): Whether to apply normalization on the bounding + box 2D attributes. Defaults to True. + """ + + def __init__(self, + base_depths=None, + base_dims=None, + code_size=7, + norm_on_bbox=True): + super(FCOS3DBBoxCoder, self).__init__() + self.base_depths = base_depths + self.base_dims = base_dims + self.bbox_code_size = code_size + self.norm_on_bbox = norm_on_bbox + + def encode(self, gt_bboxes_3d, gt_labels_3d, gt_bboxes, gt_labels): + # TODO: refactor the encoder in the FCOS3D and PGD head + pass + + def decode(self, bbox, scale, stride, training, cls_score=None): + """Decode regressed results into 3D predictions. + + Note that offsets are not transformed to the projected 3D centers. + + Args: + bbox (torch.Tensor): Raw bounding box predictions in shape + [N, C, H, W]. + scale (tuple[`Scale`]): Learnable scale parameters. + stride (tuple[int]): Stride for a specific feature level. + training (bool): Whether the decoding is in the training + procedure. + cls_score (torch.Tensor): Classification score map for deciding + which base depth or dim is used. Defaults to None. + + Returns: + torch.Tensor: Decoded boxes. + """ + # scale the bbox of different level + # only apply to offset, depth and size prediction + scale_offset, scale_depth, scale_size = scale[0:3] + + clone_bbox = bbox.clone() + bbox[:, :2] = scale_offset(clone_bbox[:, :2]).float() + bbox[:, 2] = scale_depth(clone_bbox[:, 2]).float() + bbox[:, 3:6] = scale_size(clone_bbox[:, 3:6]).float() + + if self.base_depths is None: + bbox[:, 2] = bbox[:, 2].exp() + elif len(self.base_depths) == 1: # only single prior + mean = self.base_depths[0][0] + std = self.base_depths[0][1] + bbox[:, 2] = mean + bbox.clone()[:, 2] * std + else: # multi-class priors + assert len(self.base_depths) == cls_score.shape[1], \ + 'The number of multi-class depth priors should be equal to ' \ + 'the number of categories.' + indices = cls_score.max(dim=1)[1] + depth_priors = cls_score.new_tensor( + self.base_depths)[indices, :].permute(0, 3, 1, 2) + mean = depth_priors[:, 0] + std = depth_priors[:, 1] + bbox[:, 2] = mean + bbox.clone()[:, 2] * std + + bbox[:, 3:6] = bbox[:, 3:6].exp() + if self.base_dims is not None: + assert len(self.base_dims) == cls_score.shape[1], \ + 'The number of anchor sizes should be equal to the number ' \ + 'of categories.' + indices = cls_score.max(dim=1)[1] + size_priors = cls_score.new_tensor( + self.base_dims)[indices, :].permute(0, 3, 1, 2) + bbox[:, 3:6] = size_priors * bbox.clone()[:, 3:6] + + assert self.norm_on_bbox is True, 'Setting norm_on_bbox to False '\ + 'has not been thoroughly tested for FCOS3D.' + if self.norm_on_bbox: + if not training: + # Note that this line is conducted only when testing + bbox[:, :2] *= stride + + return bbox + + @staticmethod + def decode_yaw(bbox, centers2d, dir_cls, dir_offset, cam2img): + """Decode yaw angle and change it from local to global.i. + + Args: + bbox (torch.Tensor): Bounding box predictions in shape + [N, C] with yaws to be decoded. + centers2d (torch.Tensor): Projected 3D-center on the image planes + corresponding to the box predictions. + dir_cls (torch.Tensor): Predicted direction classes. + dir_offset (float): Direction offset before dividing all the + directions into several classes. + cam2img (torch.Tensor): Camera intrinsic matrix in shape [4, 4]. + + Returns: + torch.Tensor: Bounding boxes with decoded yaws. + """ + if bbox.shape[0] > 0: + dir_rot = limit_period(bbox[..., 6] - dir_offset, 0, np.pi) + bbox[..., 6] = \ + dir_rot + dir_offset + np.pi * dir_cls.to(bbox.dtype) + + bbox[:, 6] = torch.atan2(centers2d[:, 0] - cam2img[0, 2], + cam2img[0, 0]) + bbox[:, 6] + + return bbox diff --git a/mmdet3d/models/dense_heads/fcos_mono3d_head.py b/mmdet3d/models/dense_heads/fcos_mono3d_head.py index ba2db7982c..1dba45d200 100644 --- a/mmdet3d/models/dense_heads/fcos_mono3d_head.py +++ b/mmdet3d/models/dense_heads/fcos_mono3d_head.py @@ -9,6 +9,7 @@ from mmdet3d.core import (box3d_multiclass_nms, limit_period, points_img2cam, xywhr2xyxyr) from mmdet.core import multi_apply +from mmdet.core.bbox.builder import build_bbox_coder from mmdet.models.builder import HEADS, build_loss from .anchor_free_mono3d_head import AnchorFreeMono3DHead @@ -75,6 +76,7 @@ def __init__(self, type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + bbox_coder=dict(type='FCOS3DBBoxCoder', code_size=9), norm_cfg=dict(type='GN', num_groups=32, requires_grad=True), centerness_branch=(64, ), init_cfg=None, @@ -97,6 +99,8 @@ def __init__(self, init_cfg=init_cfg, **kwargs) self.loss_centerness = build_loss(loss_centerness) + bbox_coder['code_size'] = self.bbox_code_size + self.bbox_coder = build_bbox_coder(bbox_coder) if init_cfg is None: self.init_cfg = dict( type='Normal', @@ -112,9 +116,11 @@ def _init_layers(self): conv_channels=self.centerness_branch, conv_strides=(1, ) * len(self.centerness_branch)) self.conv_centerness = nn.Conv2d(self.centerness_branch[-1], 1, 1) + self.scale_dim = 3 # only for offset, depth and size regression self.scales = nn.ModuleList([ - nn.ModuleList([Scale(1.0) for _ in range(3)]) for _ in self.strides - ]) # only for offset, depth and size regression + nn.ModuleList([Scale(1.0) for _ in range(self.scale_dim)]) + for _ in self.strides + ]) def forward(self, feats): """Forward features from the upstream network. @@ -140,8 +146,9 @@ def forward(self, feats): centernesses (list[Tensor]): Centerness for each scale level, each is a 4D-tensor, the channel number is num_points * 1. """ + # Note: we use [:5] to filter feats and only return predictions return multi_apply(self.forward_single, feats, self.scales, - self.strides) + self.strides)[:5] def forward_single(self, x, scale, stride): """Forward features of a single scale levle. @@ -171,26 +178,12 @@ def forward_single(self, x, scale, stride): for conv_centerness_prev_layer in self.conv_centerness_prev: clone_cls_feat = conv_centerness_prev_layer(clone_cls_feat) centerness = self.conv_centerness(clone_cls_feat) - # scale the bbox_pred of different level - # only apply to offset, depth and size prediction - scale_offset, scale_depth, scale_size = scale[0:3] - clone_bbox_pred = bbox_pred.clone() - bbox_pred[:, :2] = scale_offset(clone_bbox_pred[:, :2]).float() - bbox_pred[:, 2] = scale_depth(clone_bbox_pred[:, 2]).float() - bbox_pred[:, 3:6] = scale_size(clone_bbox_pred[:, 3:6]).float() + bbox_pred = self.bbox_coder.decode(bbox_pred, scale, stride, + self.training, cls_score) - bbox_pred[:, 2] = bbox_pred[:, 2].exp() - bbox_pred[:, 3:6] = bbox_pred[:, 3:6].exp() + 1e-6 # avoid size=0 - - assert self.norm_on_bbox is True, 'Setting norm_on_bbox to False '\ - 'has not been thoroughly tested for FCOS3D.' - if self.norm_on_bbox: - if not self.training: - # Note that this line is conducted only when testing - bbox_pred[:, :2] *= stride - - return cls_score, bbox_pred, dir_cls_pred, attr_pred, centerness + return cls_score, bbox_pred, dir_cls_pred, attr_pred, centerness, \ + cls_feat, reg_feat @staticmethod def add_sin_difference(boxes1, boxes2): @@ -654,19 +647,13 @@ def _get_bboxes_single(self, mlvl_dir_scores = torch.cat(mlvl_dir_scores) # change local yaw to global yaw for 3D nms - if mlvl_bboxes.shape[0] > 0: - dir_rot = limit_period(mlvl_bboxes[..., 6] - self.dir_offset, 0, - np.pi) - mlvl_bboxes[..., 6] = ( - dir_rot + self.dir_offset + - np.pi * mlvl_dir_scores.to(mlvl_bboxes.dtype)) - - cam_intrinsic = mlvl_centers2d.new_zeros((4, 4)) - cam_intrinsic[:view.shape[0], :view.shape[1]] = \ + cam2img = mlvl_centers2d.new_zeros((4, 4)) + cam2img[:view.shape[0], :view.shape[1]] = \ mlvl_centers2d.new_tensor(view) - mlvl_bboxes[:, 6] = torch.atan2( - mlvl_centers2d[:, 0] - cam_intrinsic[0, 2], - cam_intrinsic[0, 0]) + mlvl_bboxes[:, 6] + mlvl_bboxes = self.bbox_coder.decode_yaw(mlvl_bboxes, mlvl_centers2d, + mlvl_dir_scores, + self.dir_offset, cam2img) + mlvl_bboxes_for_nms = xywhr2xyxyr(input_meta['box_type_3d']( mlvl_bboxes, box_dim=self.bbox_code_size, origin=(0.5, 0.5, 0.5)).bev) diff --git a/tests/test_utils/test_bbox_coders.py b/tests/test_utils/test_bbox_coders.py index 1e9e236d81..385f27609d 100644 --- a/tests/test_utils/test_bbox_coders.py +++ b/tests/test_utils/test_bbox_coders.py @@ -1,5 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. import torch +from mmcv.cnn import Scale +from torch import nn as nn from mmdet3d.core.bbox import DepthInstance3DBoxes, LiDARInstance3DBoxes from mmdet.core import build_bbox_coder @@ -382,3 +384,84 @@ def test_point_xyzwhlr_bbox_coder(): # test decode bbox3d_out = boxcoder.decode(bbox_target, points, gt_labels_3d) assert torch.allclose(bbox3d_out, gt_bboxes_3d, atol=1e-4) + + +def test_fcos3d_bbox_coder(): + # test a config without priors + bbox_coder_cfg = dict( + type='FCOS3DBBoxCoder', + base_depths=None, + base_dims=None, + code_size=7, + norm_on_bbox=True) + bbox_coder = build_bbox_coder(bbox_coder_cfg) + + # test decode + # [2, 7, 1, 1] + batch_bbox_out = torch.tensor([[[[0.3130]], [[0.7094]], [[0.8743]], + [[0.0570]], [[0.5579]], [[0.1593]], + [[0.4553]]], + [[[0.7758]], [[0.2298]], [[0.3925]], + [[0.6307]], [[0.4377]], [[0.3339]], + [[0.1966]]]]) + batch_scale = nn.ModuleList([Scale(1.0) for _ in range(3)]) + stride = 2 + training = False + cls_score = torch.randn([2, 2, 1, 1]).sigmoid() + decode_bbox_out = bbox_coder.decode(batch_bbox_out, batch_scale, stride, + training, cls_score) + + expected_bbox_out = torch.tensor([[[[0.6261]], [[1.4188]], [[2.3971]], + [[1.0586]], [[1.7470]], [[1.1727]], + [[0.4553]]], + [[[1.5516]], [[0.4596]], [[1.4806]], + [[1.8790]], [[1.5492]], [[1.3965]], + [[0.1966]]]]) + assert torch.allclose(decode_bbox_out, expected_bbox_out, atol=1e-3) + + # test a config with priors + prior_bbox_coder_cfg = dict( + type='FCOS3DBBoxCoder', + base_depths=((28., 13.), (25., 12.)), + base_dims=((2., 3., 1.), (1., 2., 3.)), + code_size=7, + norm_on_bbox=True) + prior_bbox_coder = build_bbox_coder(prior_bbox_coder_cfg) + + # test decode + batch_bbox_out = torch.tensor([[[[0.3130]], [[0.7094]], [[0.8743]], + [[0.0570]], [[0.5579]], [[0.1593]], + [[0.4553]]], + [[[0.7758]], [[0.2298]], [[0.3925]], + [[0.6307]], [[0.4377]], [[0.3339]], + [[0.1966]]]]) + batch_scale = nn.ModuleList([Scale(1.0) for _ in range(3)]) + stride = 2 + training = False + cls_score = torch.tensor([[[[0.5811]], [[0.6198]]], [[[0.4889]], + [[0.8142]]]]) + decode_bbox_out = prior_bbox_coder.decode(batch_bbox_out, batch_scale, + stride, training, cls_score) + expected_bbox_out = torch.tensor([[[[0.6260]], [[1.4188]], [[35.4916]], + [[1.0587]], [[3.4940]], [[3.5181]], + [[0.4553]]], + [[[1.5516]], [[0.4596]], [[29.7100]], + [[1.8789]], [[3.0983]], [[4.1892]], + [[0.1966]]]]) + assert torch.allclose(decode_bbox_out, expected_bbox_out, atol=1e-3) + + # test decode_yaw + decode_bbox_out = decode_bbox_out.permute(0, 2, 3, 1).view(-1, 7) + batch_centers2d = torch.tensor([[100., 150.], [200., 100.]]) + batch_dir_cls = torch.tensor([0., 1.]) + dir_offset = 0.7854 + cam2img = torch.tensor([[700., 0., 450., 0.], [0., 700., 200., 0.], + [0., 0., 1., 0.], [0., 0., 0., 1.]]) + decode_bbox_out = prior_bbox_coder.decode_yaw(decode_bbox_out, + batch_centers2d, + batch_dir_cls, dir_offset, + cam2img) + expected_bbox_out = torch.tensor( + [[0.6260, 1.4188, 35.4916, 1.0587, 3.4940, 3.5181, 3.1332], + [1.5516, 0.4596, 29.7100, 1.8789, 3.0983, 4.1892, 6.1368]]) + assert torch.allclose(decode_bbox_out, expected_bbox_out, atol=1e-3) From 0899bad2b9980e35fac83f56917a3b087b658f2f Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 22 Sep 2021 12:46:56 +0800 Subject: [PATCH 31/51] Support PGD BBox Coder --- mmdet3d/core/bbox/coders/__init__.py | 3 +- mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py | 2 +- mmdet3d/core/bbox/coders/pgd_bbox_coder.py | 125 +++++++++++++ tests/test_utils/test_bbox_coders.py | 173 ++++++++++++++---- 4 files changed, 264 insertions(+), 39 deletions(-) create mode 100644 mmdet3d/core/bbox/coders/pgd_bbox_coder.py diff --git a/mmdet3d/core/bbox/coders/__init__.py b/mmdet3d/core/bbox/coders/__init__.py index 0e44042212..e2a9d9f7ff 100644 --- a/mmdet3d/core/bbox/coders/__init__.py +++ b/mmdet3d/core/bbox/coders/__init__.py @@ -6,10 +6,11 @@ from .fcos3d_bbox_coder import FCOS3DBBoxCoder from .groupfree3d_bbox_coder import GroupFree3DBBoxCoder from .partial_bin_based_bbox_coder import PartialBinBasedBBoxCoder +from .pgd_bbox_coder import PGDBBoxCoder from .point_xyzwhlr_bbox_coder import PointXYZWHLRBBoxCoder __all__ = [ 'build_bbox_coder', 'DeltaXYZWLHRBBoxCoder', 'PartialBinBasedBBoxCoder', 'CenterPointBBoxCoder', 'AnchorFreeBBoxCoder', 'GroupFree3DBBoxCoder', - 'PointXYZWHLRBBoxCoder', 'FCOS3DBBoxCoder' + 'PointXYZWHLRBBoxCoder', 'FCOS3DBBoxCoder', 'PGDBBoxCoder' ] diff --git a/mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py b/mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py index 7245a57e09..ae90c5d60b 100644 --- a/mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py @@ -44,7 +44,7 @@ def decode(self, bbox, scale, stride, training, cls_score=None): bbox (torch.Tensor): Raw bounding box predictions in shape [N, C, H, W]. scale (tuple[`Scale`]): Learnable scale parameters. - stride (tuple[int]): Stride for a specific feature level. + stride (int): Stride for a specific feature level. training (bool): Whether the decoding is in the training procedure. cls_score (torch.Tensor): Classification score map for deciding diff --git a/mmdet3d/core/bbox/coders/pgd_bbox_coder.py b/mmdet3d/core/bbox/coders/pgd_bbox_coder.py new file mode 100644 index 0000000000..1051d2146f --- /dev/null +++ b/mmdet3d/core/bbox/coders/pgd_bbox_coder.py @@ -0,0 +1,125 @@ +import numpy as np +from torch.nn import functional as F + +from mmdet.core.bbox.builder import BBOX_CODERS +from .fcos3d_bbox_coder import FCOS3DBBoxCoder + + +@BBOX_CODERS.register_module() +class PGDBBoxCoder(FCOS3DBBoxCoder): + """Bounding box coder for PGD.""" + + def encode(self, gt_bboxes_3d, gt_labels_3d, gt_bboxes, gt_labels): + # TODO: refactor the encoder codes in the FCOS3D and PGD head + pass + + def decode_2d(self, + bbox, + scale, + stride, + max_regress_range, + training, + pred_keypoints=False, + pred_bbox2d=True): + """Decode regressed 2D attributes. + + Args: + bbox (torch.Tensor): Raw bounding box predictions in shape + [N, C, H, W]. + scale (tuple[`Scale`]): Learnable scale parameters. + stride (int): Stride for a specific feature level. + max_regress_range (int): Maximum regression range for a specific + feature level. + training (bool): Whether the decoding is in the training + procedure. + pred_keypoints (bool): Whether to predict keypoints. Defaults to + False. + pred_bbox2d (bool): Whether to predict 2D bounding boxes. Defaults + to False. + + Returns: + torch.Tensor: Decoded boxes. + """ + clone_bbox = bbox.clone() + if pred_keypoints: + scale_kpts = scale[3] + # 2 dimension of offsets x 8 corners of a 3D bbox + bbox[:, self.bbox_code_size:self.bbox_code_size + 16] = \ + scale_kpts(clone_bbox[ + :, self.bbox_code_size:self.bbox_code_size + 16]).float() + if pred_bbox2d: + scale_bbox2d = scale[-1] + # The last four dimensions are offsets to four sides of a 2D bbox + bbox[:, -4:] = scale_bbox2d(clone_bbox[:, -4:]).float() + + if self.norm_on_bbox: + if pred_bbox2d: + bbox[:, -4:] = F.relu(bbox.clone()[:, -4:]) + if not training: + if pred_keypoints: + bbox[ + :, self.bbox_code_size:self.bbox_code_size + 16] *= \ + max_regress_range + if pred_bbox2d: + bbox[:, -4:] *= stride + else: + if pred_bbox2d: + bbox[:, -4:] = bbox.clone()[:, -4:].exp() + return bbox + + def decode_prob_depth(self, depth_cls_preds, depth_range, depth_unit, + division, num_depth_cls): + """Decode probabilistic depth map. + + Args: + depth_cls_preds (torch.Tensor): Depth probabilistic map in shape + [..., self.num_depth_cls] (raw output before softmax). + depth_range (tuple[float]): Range of depth estimation. + depth_unit (int): Unit of depth range division. + division (str): Depth division method. Options include 'uniform', + 'linear', 'log', 'loguniform'. + num_depth_cls (int): Number of depth classes. + + Returns: + torch.Tensor: Decoded probabilistic depth estimation. + """ + if division == 'uniform': + depth_multiplier = depth_unit * \ + depth_cls_preds.new_tensor( + list(range(num_depth_cls))).reshape([1, -1]) + prob_depth_preds = (F.softmax(depth_cls_preds.clone(), dim=-1) * + depth_multiplier).sum(dim=-1) + return prob_depth_preds + elif division == 'linear': + split_pts = depth_cls_preds.new_tensor(list( + range(num_depth_cls))).reshape([1, -1]) + depth_multiplier = depth_range[0] + ( + depth_range[1] - depth_range[0]) / \ + (num_depth_cls * (num_depth_cls - 1)) * \ + (split_pts * (split_pts+1)) + prob_depth_preds = (F.softmax(depth_cls_preds.clone(), dim=-1) * + depth_multiplier).sum(dim=-1) + return prob_depth_preds + elif division == 'log': + split_pts = depth_cls_preds.new_tensor(list( + range(num_depth_cls))).reshape([1, -1]) + start = max(depth_range[0], 1) + end = depth_range[1] + depth_multiplier = (np.log(start) + + split_pts * np.log(end / start) / + (num_depth_cls - 1)).exp() + prob_depth_preds = (F.softmax(depth_cls_preds.clone(), dim=-1) * + depth_multiplier).sum(dim=-1) + return prob_depth_preds + elif division == 'loguniform': + split_pts = depth_cls_preds.new_tensor(list( + range(num_depth_cls))).reshape([1, -1]) + start = max(depth_range[0], 1) + end = depth_range[1] + log_multiplier = np.log(start) + \ + split_pts * np.log(end / start) / (num_depth_cls - 1) + prob_depth_preds = (F.softmax(depth_cls_preds.clone(), dim=-1) * + log_multiplier).sum(dim=-1).exp() + return prob_depth_preds + else: + raise NotImplementedError diff --git a/tests/test_utils/test_bbox_coders.py b/tests/test_utils/test_bbox_coders.py index 385f27609d..e9928bc1cf 100644 --- a/tests/test_utils/test_bbox_coders.py +++ b/tests/test_utils/test_bbox_coders.py @@ -398,26 +398,24 @@ def test_fcos3d_bbox_coder(): # test decode # [2, 7, 1, 1] - batch_bbox_out = torch.tensor([[[[0.3130]], [[0.7094]], [[0.8743]], - [[0.0570]], [[0.5579]], [[0.1593]], - [[0.4553]]], - [[[0.7758]], [[0.2298]], [[0.3925]], - [[0.6307]], [[0.4377]], [[0.3339]], - [[0.1966]]]]) + batch_bbox = torch.tensor([[[[0.3130]], [[0.7094]], [[0.8743]], [[0.0570]], + [[0.5579]], [[0.1593]], [[0.4553]]], + [[[0.7758]], [[0.2298]], [[0.3925]], [[0.6307]], + [[0.4377]], [[0.3339]], [[0.1966]]]]) batch_scale = nn.ModuleList([Scale(1.0) for _ in range(3)]) stride = 2 training = False cls_score = torch.randn([2, 2, 1, 1]).sigmoid() - decode_bbox_out = bbox_coder.decode(batch_bbox_out, batch_scale, stride, - training, cls_score) + decode_bbox = bbox_coder.decode(batch_bbox, batch_scale, stride, training, + cls_score) - expected_bbox_out = torch.tensor([[[[0.6261]], [[1.4188]], [[2.3971]], - [[1.0586]], [[1.7470]], [[1.1727]], - [[0.4553]]], - [[[1.5516]], [[0.4596]], [[1.4806]], - [[1.8790]], [[1.5492]], [[1.3965]], - [[0.1966]]]]) - assert torch.allclose(decode_bbox_out, expected_bbox_out, atol=1e-3) + expected_bbox = torch.tensor([[[[0.6261]], [[1.4188]], [[2.3971]], + [[1.0586]], [[1.7470]], [[1.1727]], + [[0.4553]]], + [[[1.5516]], [[0.4596]], [[1.4806]], + [[1.8790]], [[1.5492]], [[1.3965]], + [[0.1966]]]]) + assert torch.allclose(decode_bbox, expected_bbox, atol=1e-3) # test a config with priors prior_bbox_coder_cfg = dict( @@ -429,39 +427,140 @@ def test_fcos3d_bbox_coder(): prior_bbox_coder = build_bbox_coder(prior_bbox_coder_cfg) # test decode - batch_bbox_out = torch.tensor([[[[0.3130]], [[0.7094]], [[0.8743]], - [[0.0570]], [[0.5579]], [[0.1593]], - [[0.4553]]], - [[[0.7758]], [[0.2298]], [[0.3925]], - [[0.6307]], [[0.4377]], [[0.3339]], - [[0.1966]]]]) + batch_bbox = torch.tensor([[[[0.3130]], [[0.7094]], [[0.8743]], [[0.0570]], + [[0.5579]], [[0.1593]], [[0.4553]]], + [[[0.7758]], [[0.2298]], [[0.3925]], [[0.6307]], + [[0.4377]], [[0.3339]], [[0.1966]]]]) batch_scale = nn.ModuleList([Scale(1.0) for _ in range(3)]) stride = 2 training = False cls_score = torch.tensor([[[[0.5811]], [[0.6198]]], [[[0.4889]], [[0.8142]]]]) - decode_bbox_out = prior_bbox_coder.decode(batch_bbox_out, batch_scale, - stride, training, cls_score) - expected_bbox_out = torch.tensor([[[[0.6260]], [[1.4188]], [[35.4916]], - [[1.0587]], [[3.4940]], [[3.5181]], - [[0.4553]]], - [[[1.5516]], [[0.4596]], [[29.7100]], - [[1.8789]], [[3.0983]], [[4.1892]], - [[0.1966]]]]) - assert torch.allclose(decode_bbox_out, expected_bbox_out, atol=1e-3) + decode_bbox = prior_bbox_coder.decode(batch_bbox, batch_scale, stride, + training, cls_score) + expected_bbox = torch.tensor([[[[0.6260]], [[1.4188]], [[35.4916]], + [[1.0587]], [[3.4940]], [[3.5181]], + [[0.4553]]], + [[[1.5516]], [[0.4596]], [[29.7100]], + [[1.8789]], [[3.0983]], [[4.1892]], + [[0.1966]]]]) + assert torch.allclose(decode_bbox, expected_bbox, atol=1e-3) # test decode_yaw - decode_bbox_out = decode_bbox_out.permute(0, 2, 3, 1).view(-1, 7) + decode_bbox = decode_bbox.permute(0, 2, 3, 1).view(-1, 7) batch_centers2d = torch.tensor([[100., 150.], [200., 100.]]) batch_dir_cls = torch.tensor([0., 1.]) dir_offset = 0.7854 cam2img = torch.tensor([[700., 0., 450., 0.], [0., 700., 200., 0.], [0., 0., 1., 0.], [0., 0., 0., 1.]]) - decode_bbox_out = prior_bbox_coder.decode_yaw(decode_bbox_out, - batch_centers2d, - batch_dir_cls, dir_offset, - cam2img) - expected_bbox_out = torch.tensor( + decode_bbox = prior_bbox_coder.decode_yaw(decode_bbox, batch_centers2d, + batch_dir_cls, dir_offset, + cam2img) + expected_bbox = torch.tensor( [[0.6260, 1.4188, 35.4916, 1.0587, 3.4940, 3.5181, 3.1332], [1.5516, 0.4596, 29.7100, 1.8789, 3.0983, 4.1892, 6.1368]]) - assert torch.allclose(decode_bbox_out, expected_bbox_out, atol=1e-3) + assert torch.allclose(decode_bbox, expected_bbox, atol=1e-3) + + +def test_pgd_bbox_coder(): + # test a config without priors + bbox_coder_cfg = dict( + type='PGDBBoxCoder', + base_depths=None, + base_dims=None, + code_size=7, + norm_on_bbox=True) + bbox_coder = build_bbox_coder(bbox_coder_cfg) + + # test decode_2d + # [2, 27, 1, 1] + batch_bbox = torch.tensor([[[[0.0103]], [[0.7394]], [[0.3296]], [[0.4708]], + [[0.1439]], [[0.0778]], [[0.9399]], [[0.8366]], + [[0.1264]], [[0.3030]], [[0.1898]], [[0.0714]], + [[0.4144]], [[0.4341]], [[0.6442]], [[0.2951]], + [[0.2890]], [[0.4486]], [[0.2848]], [[0.1071]], + [[0.9530]], [[0.9460]], [[0.3822]], [[0.9320]], + [[0.2611]], [[0.5580]], [[0.0397]]], + [[[0.8612]], [[0.1680]], [[0.5167]], [[0.8502]], + [[0.0377]], [[0.3615]], [[0.9550]], [[0.5219]], + [[0.1402]], [[0.6843]], [[0.2121]], [[0.9468]], + [[0.6238]], [[0.7918]], [[0.1646]], [[0.0500]], + [[0.6290]], [[0.3956]], [[0.2901]], [[0.4612]], + [[0.7333]], [[0.1194]], [[0.6999]], [[0.3980]], + [[0.3262]], [[0.7185]], [[0.4474]]]]) + batch_scale = nn.ModuleList([Scale(1.0) for _ in range(5)]) + stride = 2 + training = False + cls_score = torch.randn([2, 2, 1, 1]).sigmoid() + decode_bbox = bbox_coder.decode(batch_bbox, batch_scale, stride, training, + cls_score) + max_regress_range = 16 + pred_keypoints = True + pred_bbox2d = True + decode_bbox_w2d = bbox_coder.decode_2d(decode_bbox, batch_scale, stride, + max_regress_range, training, + pred_keypoints, pred_bbox2d) + expected_decode_bbox_w2d = torch.tensor( + [[[[0.0206]], [[1.4788]], [[1.3904]], [[1.6013]], [[1.1548]], + [[1.0809]], [[0.9399]], [[13.3856]], [[2.0224]], [[4.8480]], + [[3.0368]], [[1.1424]], [[6.6304]], [[6.9456]], [[10.3072]], + [[4.7216]], [[4.6240]], [[7.1776]], [[4.5568]], [[1.7136]], + [[15.2480]], [[15.1360]], [[6.1152]], [[1.8640]], [[0.5222]], + [[1.1160]], [[0.0794]]], + [[[1.7224]], [[0.3360]], [[1.6765]], [[2.3401]], [[1.0384]], + [[1.4355]], [[0.9550]], [[8.3504]], [[2.2432]], [[10.9488]], + [[3.3936]], [[15.1488]], [[9.9808]], [[12.6688]], [[2.6336]], + [[0.8000]], [[10.0640]], [[6.3296]], [[4.6416]], [[7.3792]], + [[11.7328]], [[1.9104]], [[11.1984]], [[0.7960]], [[0.6524]], + [[1.4370]], [[0.8948]]]]) + assert torch.allclose(expected_decode_bbox_w2d, decode_bbox_w2d, atol=1e-3) + + # test decode_prob_depth + # [10, 8] + depth_cls_preds = torch.tensor([ + [-0.4383, 0.7207, -0.4092, 0.4649, 0.8526, 0.6186, -1.4312, -0.7150], + [0.0621, 0.2369, 0.5170, 0.8484, -0.1099, 0.1829, -0.0072, 1.0618], + [-1.6114, -0.1057, 0.5721, -0.5986, -2.0471, 0.8140, -0.8385, -0.4822], + [0.0742, -0.3261, 0.4607, 1.8155, -0.3571, -0.0234, 0.3787, 2.3251], + [1.0492, -0.6881, -0.0136, -1.8291, 0.8460, -1.0171, 2.5691, -0.8114], + [0.0968, -0.5601, 1.0458, 0.2560, 1.3018, 0.1635, 0.0680, -1.0263], + [-0.0765, 0.1498, -2.7321, 1.0047, -0.2505, 0.0871, -0.4820, -0.3003], + [-0.4123, 0.2298, -0.1330, -0.6008, 0.6526, 0.7118, 0.9728, -0.7793], + [1.6940, 0.3355, 1.4661, 0.5477, 0.8667, 0.0527, -0.9975, -0.0689], + [0.4724, -0.3632, -0.0654, 0.4034, -0.3494, -0.7548, 0.7297, 1.2754] + ]) + depth_range = (0, 70) + depth_unit = 10 + num_depth_cls = 8 + uniform_prob_depth_preds = bbox_coder.decode_prob_depth( + depth_cls_preds, depth_range, depth_unit, 'uniform', num_depth_cls) + expected_preds = torch.tensor([ + 32.0441, 38.4689, 36.1831, 48.2096, 46.1560, 32.7973, 33.2155, 39.9822, + 21.9905, 43.0161 + ]) + assert torch.allclose(uniform_prob_depth_preds, expected_preds, atol=1e-3) + + linear_prob_depth_preds = bbox_coder.decode_prob_depth( + depth_cls_preds, depth_range, depth_unit, 'linear', num_depth_cls) + expected_preds = torch.tensor([ + 21.1431, 30.2421, 25.8964, 41.6116, 38.6234, 21.4582, 23.2993, 30.1111, + 13.9273, 36.8419 + ]) + assert torch.allclose(linear_prob_depth_preds, expected_preds, atol=1e-3) + + log_prob_depth_preds = bbox_coder.decode_prob_depth( + depth_cls_preds, depth_range, depth_unit, 'log', num_depth_cls) + expected_preds = torch.tensor([ + 12.6458, 24.2487, 17.4015, 36.9375, 27.5982, 12.5510, 15.6635, 19.8408, + 9.1605, 31.3765 + ]) + assert torch.allclose(log_prob_depth_preds, expected_preds, atol=1e-3) + + loguniform_prob_depth_preds = bbox_coder.decode_prob_depth( + depth_cls_preds, depth_range, depth_unit, 'loguniform', num_depth_cls) + expected_preds = torch.tensor([ + 6.9925, 10.3273, 8.9895, 18.6524, 16.4667, 7.3196, 7.5078, 11.3207, + 3.7987, 13.6095 + ]) + assert torch.allclose( + loguniform_prob_depth_preds, expected_preds, atol=1e-3) From b217f7ad3ec636efc8367ace0ed7c634e64842ff Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 22 Sep 2021 13:30:59 +0800 Subject: [PATCH 32/51] Refine docstring --- mmdet3d/core/bbox/coders/pgd_bbox_coder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mmdet3d/core/bbox/coders/pgd_bbox_coder.py b/mmdet3d/core/bbox/coders/pgd_bbox_coder.py index 1051d2146f..5e42128189 100644 --- a/mmdet3d/core/bbox/coders/pgd_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/pgd_bbox_coder.py @@ -32,10 +32,10 @@ def decode_2d(self, feature level. training (bool): Whether the decoding is in the training procedure. - pred_keypoints (bool): Whether to predict keypoints. Defaults to - False. - pred_bbox2d (bool): Whether to predict 2D bounding boxes. Defaults - to False. + pred_keypoints (bool, optional): Whether to predict keypoints. + Defaults to False. + pred_bbox2d (bool, optional): Whether to predict 2D bounding + boxes. Defaults to False. Returns: torch.Tensor: Decoded boxes. From d28a8b513e700ac853f947288bbf162d560843c5 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 22 Sep 2021 15:25:29 +0800 Subject: [PATCH 33/51] Add uncertain l1 loss and its unit tests --- mmdet3d/models/losses/__init__.py | 3 +- .../models/losses/uncertain_smooth_l1_loss.py | 174 ++++++++++++++++++ tests/test_metrics/test_losses.py | 37 ++++ 3 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 mmdet3d/models/losses/uncertain_smooth_l1_loss.py diff --git a/mmdet3d/models/losses/__init__.py b/mmdet3d/models/losses/__init__.py index 7d4703aef9..0efbcb780d 100644 --- a/mmdet3d/models/losses/__init__.py +++ b/mmdet3d/models/losses/__init__.py @@ -3,9 +3,10 @@ from .axis_aligned_iou_loss import AxisAlignedIoULoss, axis_aligned_iou_loss from .chamfer_distance import ChamferDistance, chamfer_distance from .paconv_regularization_loss import PAConvRegularizationLoss +from .uncertain_smooth_l1_loss import UncertainL1Loss, UncertainSmoothL1Loss __all__ = [ 'FocalLoss', 'SmoothL1Loss', 'binary_cross_entropy', 'ChamferDistance', 'chamfer_distance', 'axis_aligned_iou_loss', 'AxisAlignedIoULoss', - 'PAConvRegularizationLoss' + 'PAConvRegularizationLoss', 'UncertainL1Loss', 'UncertainSmoothL1Loss' ] diff --git a/mmdet3d/models/losses/uncertain_smooth_l1_loss.py b/mmdet3d/models/losses/uncertain_smooth_l1_loss.py new file mode 100644 index 0000000000..f5f0c91c6a --- /dev/null +++ b/mmdet3d/models/losses/uncertain_smooth_l1_loss.py @@ -0,0 +1,174 @@ +import mmcv +import torch +from torch import nn as nn + +from mmdet.models.builder import LOSSES +from mmdet.models.losses.utils import weighted_loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def uncertain_smooth_l1_loss(pred, target, sigma, alpha=1.0, beta=1.0): + """Smooth L1 loss with uncertainty. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + sigma (torch.Tensor): The sigma for uncertainty. + alpha (float, optional): The coefficient of log(sigma). + Defaults to 1.0. + beta (float, optional): The threshold in the piecewise function. + Defaults to 1.0. + + Returns: + torch.Tensor: Calculated loss + """ + assert beta > 0 + assert pred.size() == target.size() == sigma.size() and \ + target.numel() > 0 + diff = torch.abs(pred - target) + loss = torch.where(diff < beta, 0.5 * diff * diff / beta, + diff - 0.5 * beta) + loss = torch.exp(-sigma) * loss + alpha * sigma + + return loss + + +@mmcv.jit(derivate=True, coderize=True) +@weighted_loss +def uncertain_l1_loss(pred, target, sigma, alpha=1.0): + """L1 loss with uncertainty. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + sigma (torch.Tensor): The sigma for uncertainty. + alpha (float, optional): The coefficient of log(sigma). + Defaults to 1.0. + + Returns: + torch.Tensor: Calculated loss + """ + assert pred.size() == target.size() == sigma.size() and \ + target.numel() > 0 + loss = torch.abs(pred - target) + loss = torch.exp(-sigma) * loss + alpha * sigma + return loss + + +@LOSSES.register_module() +class UncertainSmoothL1Loss(nn.Module): + r"""Smooth L1 loss with uncertainty. + + Please refer to `PGD `_ and + `Multi-Task Learning Using Uncertainty to Weigh Losses for Scene Geometry + and Semantics `_ for more details. + + Args: + alpha (float, optional): The coefficient of log(sigma). + Defaults to 1.0. + beta (float, optional): The threshold in the piecewise function. + Defaults to 1.0. + reduction (str, optional): The method to reduce the loss. + Options are "none", "mean" and "sum". Defaults to "mean". + loss_weight (float, optional): The weight of loss. + """ + + def __init__(self, alpha=1.0, beta=1.0, reduction='mean', loss_weight=1.0): + super(UncertainSmoothL1Loss, self).__init__() + assert reduction in ['none', 'sum', 'mean'] + self.alpha = alpha + self.beta = beta + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + sigma, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + sigma (torch.Tensor): The sigma for uncertainty. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss_bbox = self.loss_weight * uncertain_smooth_l1_loss( + pred, + target, + weight, + sigma=sigma, + alpha=self.alpha, + beta=self.beta, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss_bbox + + +@LOSSES.register_module() +class UncertainL1Loss(nn.Module): + """L1 loss with uncertainty. + + Args: + alpha (float, optional): The coefficient of log(sigma). + Defaults to 1.0. + reduction (str, optional): The method to reduce the loss. + Options are "none", "mean" and "sum". + loss_weight (float, optional): The weight of loss. + """ + + def __init__(self, alpha=1.0, reduction='mean', loss_weight=1.0): + super(UncertainL1Loss, self).__init__() + assert reduction in ['none', 'sum', 'mean'] + self.alpha = alpha + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + sigma, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + sigma (torch.Tensor): The sigma for uncertainty. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss_bbox = self.loss_weight * uncertain_l1_loss( + pred, + target, + weight, + sigma=sigma, + alpha=self.alpha, + reduction=reduction, + avg_factor=avg_factor) + return loss_bbox diff --git a/tests/test_metrics/test_losses.py b/tests/test_metrics/test_losses.py index 08cdb6275c..d8eca2f961 100644 --- a/tests/test_metrics/test_losses.py +++ b/tests/test_metrics/test_losses.py @@ -3,6 +3,8 @@ import torch from torch import nn as nn +from mmdet.models import build_loss + def test_chamfer_disrance(): from mmdet3d.models.losses import ChamferDistance, chamfer_distance @@ -109,3 +111,38 @@ def __init__(self): model.modules(), reduction_override='none') assert none_corr_loss.shape[0] == 3 assert torch.allclose(none_corr_loss.mean(), mean_corr_loss) + + +def test_uncertain_smooth_l1_loss(): + from mmdet3d.models.losses import UncertainL1Loss, UncertainSmoothL1Loss + + # reduction shoule be in ['none', 'mean', 'sum'] + with pytest.raises(AssertionError): + uncertain_l1_loss = UncertainL1Loss(reduction='l2') + with pytest.raises(AssertionError): + uncertain_smooth_l1_loss = UncertainSmoothL1Loss(reduction='l2') + + pred = torch.tensor([1.5783, 0.5972, 1.4821, 0.9488]) + target = torch.tensor([1.0813, -0.3466, -1.1404, -0.9665]) + sigma = torch.tensor([-1.0053, 0.4710, -1.7784, -0.8603]) + + # test uncertain l1 loss + uncertain_l1_loss_cfg = dict( + type='UncertainL1Loss', alpha=1.0, reduction='mean', loss_weight=1.0) + uncertain_l1_loss = build_loss(uncertain_l1_loss_cfg) + mean_l1_loss = uncertain_l1_loss(pred, target, sigma) + expected_l1_loss = torch.tensor(4.7069) + assert torch.allclose(mean_l1_loss, expected_l1_loss, atol=1e-4) + + # test uncertain smooth l1 loss + uncertain_smooth_l1_loss_cfg = dict( + type='UncertainSmoothL1Loss', + alpha=1.0, + beta=0.5, + reduction='mean', + loss_weight=1.0) + uncertain_smooth_l1_loss = build_loss(uncertain_smooth_l1_loss_cfg) + mean_smooth_l1_loss = uncertain_smooth_l1_loss(pred, target, sigma) + expected_smooth_l1_loss = torch.tensor(3.9795) + assert torch.allclose( + mean_smooth_l1_loss, expected_smooth_l1_loss, atol=1e-4) From 506f92940cb0223cae2a821acbf7ac29994076f5 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 22 Sep 2021 17:17:27 +0800 Subject: [PATCH 34/51] [Feature] PGD BBox Coder (#948) * Support PGD BBox Coder * Refine docstring --- mmdet3d/core/bbox/coders/__init__.py | 3 +- mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py | 2 +- mmdet3d/core/bbox/coders/pgd_bbox_coder.py | 125 +++++++++++++ tests/test_utils/test_bbox_coders.py | 173 ++++++++++++++---- 4 files changed, 264 insertions(+), 39 deletions(-) create mode 100644 mmdet3d/core/bbox/coders/pgd_bbox_coder.py diff --git a/mmdet3d/core/bbox/coders/__init__.py b/mmdet3d/core/bbox/coders/__init__.py index 0e44042212..e2a9d9f7ff 100644 --- a/mmdet3d/core/bbox/coders/__init__.py +++ b/mmdet3d/core/bbox/coders/__init__.py @@ -6,10 +6,11 @@ from .fcos3d_bbox_coder import FCOS3DBBoxCoder from .groupfree3d_bbox_coder import GroupFree3DBBoxCoder from .partial_bin_based_bbox_coder import PartialBinBasedBBoxCoder +from .pgd_bbox_coder import PGDBBoxCoder from .point_xyzwhlr_bbox_coder import PointXYZWHLRBBoxCoder __all__ = [ 'build_bbox_coder', 'DeltaXYZWLHRBBoxCoder', 'PartialBinBasedBBoxCoder', 'CenterPointBBoxCoder', 'AnchorFreeBBoxCoder', 'GroupFree3DBBoxCoder', - 'PointXYZWHLRBBoxCoder', 'FCOS3DBBoxCoder' + 'PointXYZWHLRBBoxCoder', 'FCOS3DBBoxCoder', 'PGDBBoxCoder' ] diff --git a/mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py b/mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py index 7245a57e09..ae90c5d60b 100644 --- a/mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/fcos3d_bbox_coder.py @@ -44,7 +44,7 @@ def decode(self, bbox, scale, stride, training, cls_score=None): bbox (torch.Tensor): Raw bounding box predictions in shape [N, C, H, W]. scale (tuple[`Scale`]): Learnable scale parameters. - stride (tuple[int]): Stride for a specific feature level. + stride (int): Stride for a specific feature level. training (bool): Whether the decoding is in the training procedure. cls_score (torch.Tensor): Classification score map for deciding diff --git a/mmdet3d/core/bbox/coders/pgd_bbox_coder.py b/mmdet3d/core/bbox/coders/pgd_bbox_coder.py new file mode 100644 index 0000000000..5e42128189 --- /dev/null +++ b/mmdet3d/core/bbox/coders/pgd_bbox_coder.py @@ -0,0 +1,125 @@ +import numpy as np +from torch.nn import functional as F + +from mmdet.core.bbox.builder import BBOX_CODERS +from .fcos3d_bbox_coder import FCOS3DBBoxCoder + + +@BBOX_CODERS.register_module() +class PGDBBoxCoder(FCOS3DBBoxCoder): + """Bounding box coder for PGD.""" + + def encode(self, gt_bboxes_3d, gt_labels_3d, gt_bboxes, gt_labels): + # TODO: refactor the encoder codes in the FCOS3D and PGD head + pass + + def decode_2d(self, + bbox, + scale, + stride, + max_regress_range, + training, + pred_keypoints=False, + pred_bbox2d=True): + """Decode regressed 2D attributes. + + Args: + bbox (torch.Tensor): Raw bounding box predictions in shape + [N, C, H, W]. + scale (tuple[`Scale`]): Learnable scale parameters. + stride (int): Stride for a specific feature level. + max_regress_range (int): Maximum regression range for a specific + feature level. + training (bool): Whether the decoding is in the training + procedure. + pred_keypoints (bool, optional): Whether to predict keypoints. + Defaults to False. + pred_bbox2d (bool, optional): Whether to predict 2D bounding + boxes. Defaults to False. + + Returns: + torch.Tensor: Decoded boxes. + """ + clone_bbox = bbox.clone() + if pred_keypoints: + scale_kpts = scale[3] + # 2 dimension of offsets x 8 corners of a 3D bbox + bbox[:, self.bbox_code_size:self.bbox_code_size + 16] = \ + scale_kpts(clone_bbox[ + :, self.bbox_code_size:self.bbox_code_size + 16]).float() + if pred_bbox2d: + scale_bbox2d = scale[-1] + # The last four dimensions are offsets to four sides of a 2D bbox + bbox[:, -4:] = scale_bbox2d(clone_bbox[:, -4:]).float() + + if self.norm_on_bbox: + if pred_bbox2d: + bbox[:, -4:] = F.relu(bbox.clone()[:, -4:]) + if not training: + if pred_keypoints: + bbox[ + :, self.bbox_code_size:self.bbox_code_size + 16] *= \ + max_regress_range + if pred_bbox2d: + bbox[:, -4:] *= stride + else: + if pred_bbox2d: + bbox[:, -4:] = bbox.clone()[:, -4:].exp() + return bbox + + def decode_prob_depth(self, depth_cls_preds, depth_range, depth_unit, + division, num_depth_cls): + """Decode probabilistic depth map. + + Args: + depth_cls_preds (torch.Tensor): Depth probabilistic map in shape + [..., self.num_depth_cls] (raw output before softmax). + depth_range (tuple[float]): Range of depth estimation. + depth_unit (int): Unit of depth range division. + division (str): Depth division method. Options include 'uniform', + 'linear', 'log', 'loguniform'. + num_depth_cls (int): Number of depth classes. + + Returns: + torch.Tensor: Decoded probabilistic depth estimation. + """ + if division == 'uniform': + depth_multiplier = depth_unit * \ + depth_cls_preds.new_tensor( + list(range(num_depth_cls))).reshape([1, -1]) + prob_depth_preds = (F.softmax(depth_cls_preds.clone(), dim=-1) * + depth_multiplier).sum(dim=-1) + return prob_depth_preds + elif division == 'linear': + split_pts = depth_cls_preds.new_tensor(list( + range(num_depth_cls))).reshape([1, -1]) + depth_multiplier = depth_range[0] + ( + depth_range[1] - depth_range[0]) / \ + (num_depth_cls * (num_depth_cls - 1)) * \ + (split_pts * (split_pts+1)) + prob_depth_preds = (F.softmax(depth_cls_preds.clone(), dim=-1) * + depth_multiplier).sum(dim=-1) + return prob_depth_preds + elif division == 'log': + split_pts = depth_cls_preds.new_tensor(list( + range(num_depth_cls))).reshape([1, -1]) + start = max(depth_range[0], 1) + end = depth_range[1] + depth_multiplier = (np.log(start) + + split_pts * np.log(end / start) / + (num_depth_cls - 1)).exp() + prob_depth_preds = (F.softmax(depth_cls_preds.clone(), dim=-1) * + depth_multiplier).sum(dim=-1) + return prob_depth_preds + elif division == 'loguniform': + split_pts = depth_cls_preds.new_tensor(list( + range(num_depth_cls))).reshape([1, -1]) + start = max(depth_range[0], 1) + end = depth_range[1] + log_multiplier = np.log(start) + \ + split_pts * np.log(end / start) / (num_depth_cls - 1) + prob_depth_preds = (F.softmax(depth_cls_preds.clone(), dim=-1) * + log_multiplier).sum(dim=-1).exp() + return prob_depth_preds + else: + raise NotImplementedError diff --git a/tests/test_utils/test_bbox_coders.py b/tests/test_utils/test_bbox_coders.py index 385f27609d..e9928bc1cf 100644 --- a/tests/test_utils/test_bbox_coders.py +++ b/tests/test_utils/test_bbox_coders.py @@ -398,26 +398,24 @@ def test_fcos3d_bbox_coder(): # test decode # [2, 7, 1, 1] - batch_bbox_out = torch.tensor([[[[0.3130]], [[0.7094]], [[0.8743]], - [[0.0570]], [[0.5579]], [[0.1593]], - [[0.4553]]], - [[[0.7758]], [[0.2298]], [[0.3925]], - [[0.6307]], [[0.4377]], [[0.3339]], - [[0.1966]]]]) + batch_bbox = torch.tensor([[[[0.3130]], [[0.7094]], [[0.8743]], [[0.0570]], + [[0.5579]], [[0.1593]], [[0.4553]]], + [[[0.7758]], [[0.2298]], [[0.3925]], [[0.6307]], + [[0.4377]], [[0.3339]], [[0.1966]]]]) batch_scale = nn.ModuleList([Scale(1.0) for _ in range(3)]) stride = 2 training = False cls_score = torch.randn([2, 2, 1, 1]).sigmoid() - decode_bbox_out = bbox_coder.decode(batch_bbox_out, batch_scale, stride, - training, cls_score) + decode_bbox = bbox_coder.decode(batch_bbox, batch_scale, stride, training, + cls_score) - expected_bbox_out = torch.tensor([[[[0.6261]], [[1.4188]], [[2.3971]], - [[1.0586]], [[1.7470]], [[1.1727]], - [[0.4553]]], - [[[1.5516]], [[0.4596]], [[1.4806]], - [[1.8790]], [[1.5492]], [[1.3965]], - [[0.1966]]]]) - assert torch.allclose(decode_bbox_out, expected_bbox_out, atol=1e-3) + expected_bbox = torch.tensor([[[[0.6261]], [[1.4188]], [[2.3971]], + [[1.0586]], [[1.7470]], [[1.1727]], + [[0.4553]]], + [[[1.5516]], [[0.4596]], [[1.4806]], + [[1.8790]], [[1.5492]], [[1.3965]], + [[0.1966]]]]) + assert torch.allclose(decode_bbox, expected_bbox, atol=1e-3) # test a config with priors prior_bbox_coder_cfg = dict( @@ -429,39 +427,140 @@ def test_fcos3d_bbox_coder(): prior_bbox_coder = build_bbox_coder(prior_bbox_coder_cfg) # test decode - batch_bbox_out = torch.tensor([[[[0.3130]], [[0.7094]], [[0.8743]], - [[0.0570]], [[0.5579]], [[0.1593]], - [[0.4553]]], - [[[0.7758]], [[0.2298]], [[0.3925]], - [[0.6307]], [[0.4377]], [[0.3339]], - [[0.1966]]]]) + batch_bbox = torch.tensor([[[[0.3130]], [[0.7094]], [[0.8743]], [[0.0570]], + [[0.5579]], [[0.1593]], [[0.4553]]], + [[[0.7758]], [[0.2298]], [[0.3925]], [[0.6307]], + [[0.4377]], [[0.3339]], [[0.1966]]]]) batch_scale = nn.ModuleList([Scale(1.0) for _ in range(3)]) stride = 2 training = False cls_score = torch.tensor([[[[0.5811]], [[0.6198]]], [[[0.4889]], [[0.8142]]]]) - decode_bbox_out = prior_bbox_coder.decode(batch_bbox_out, batch_scale, - stride, training, cls_score) - expected_bbox_out = torch.tensor([[[[0.6260]], [[1.4188]], [[35.4916]], - [[1.0587]], [[3.4940]], [[3.5181]], - [[0.4553]]], - [[[1.5516]], [[0.4596]], [[29.7100]], - [[1.8789]], [[3.0983]], [[4.1892]], - [[0.1966]]]]) - assert torch.allclose(decode_bbox_out, expected_bbox_out, atol=1e-3) + decode_bbox = prior_bbox_coder.decode(batch_bbox, batch_scale, stride, + training, cls_score) + expected_bbox = torch.tensor([[[[0.6260]], [[1.4188]], [[35.4916]], + [[1.0587]], [[3.4940]], [[3.5181]], + [[0.4553]]], + [[[1.5516]], [[0.4596]], [[29.7100]], + [[1.8789]], [[3.0983]], [[4.1892]], + [[0.1966]]]]) + assert torch.allclose(decode_bbox, expected_bbox, atol=1e-3) # test decode_yaw - decode_bbox_out = decode_bbox_out.permute(0, 2, 3, 1).view(-1, 7) + decode_bbox = decode_bbox.permute(0, 2, 3, 1).view(-1, 7) batch_centers2d = torch.tensor([[100., 150.], [200., 100.]]) batch_dir_cls = torch.tensor([0., 1.]) dir_offset = 0.7854 cam2img = torch.tensor([[700., 0., 450., 0.], [0., 700., 200., 0.], [0., 0., 1., 0.], [0., 0., 0., 1.]]) - decode_bbox_out = prior_bbox_coder.decode_yaw(decode_bbox_out, - batch_centers2d, - batch_dir_cls, dir_offset, - cam2img) - expected_bbox_out = torch.tensor( + decode_bbox = prior_bbox_coder.decode_yaw(decode_bbox, batch_centers2d, + batch_dir_cls, dir_offset, + cam2img) + expected_bbox = torch.tensor( [[0.6260, 1.4188, 35.4916, 1.0587, 3.4940, 3.5181, 3.1332], [1.5516, 0.4596, 29.7100, 1.8789, 3.0983, 4.1892, 6.1368]]) - assert torch.allclose(decode_bbox_out, expected_bbox_out, atol=1e-3) + assert torch.allclose(decode_bbox, expected_bbox, atol=1e-3) + + +def test_pgd_bbox_coder(): + # test a config without priors + bbox_coder_cfg = dict( + type='PGDBBoxCoder', + base_depths=None, + base_dims=None, + code_size=7, + norm_on_bbox=True) + bbox_coder = build_bbox_coder(bbox_coder_cfg) + + # test decode_2d + # [2, 27, 1, 1] + batch_bbox = torch.tensor([[[[0.0103]], [[0.7394]], [[0.3296]], [[0.4708]], + [[0.1439]], [[0.0778]], [[0.9399]], [[0.8366]], + [[0.1264]], [[0.3030]], [[0.1898]], [[0.0714]], + [[0.4144]], [[0.4341]], [[0.6442]], [[0.2951]], + [[0.2890]], [[0.4486]], [[0.2848]], [[0.1071]], + [[0.9530]], [[0.9460]], [[0.3822]], [[0.9320]], + [[0.2611]], [[0.5580]], [[0.0397]]], + [[[0.8612]], [[0.1680]], [[0.5167]], [[0.8502]], + [[0.0377]], [[0.3615]], [[0.9550]], [[0.5219]], + [[0.1402]], [[0.6843]], [[0.2121]], [[0.9468]], + [[0.6238]], [[0.7918]], [[0.1646]], [[0.0500]], + [[0.6290]], [[0.3956]], [[0.2901]], [[0.4612]], + [[0.7333]], [[0.1194]], [[0.6999]], [[0.3980]], + [[0.3262]], [[0.7185]], [[0.4474]]]]) + batch_scale = nn.ModuleList([Scale(1.0) for _ in range(5)]) + stride = 2 + training = False + cls_score = torch.randn([2, 2, 1, 1]).sigmoid() + decode_bbox = bbox_coder.decode(batch_bbox, batch_scale, stride, training, + cls_score) + max_regress_range = 16 + pred_keypoints = True + pred_bbox2d = True + decode_bbox_w2d = bbox_coder.decode_2d(decode_bbox, batch_scale, stride, + max_regress_range, training, + pred_keypoints, pred_bbox2d) + expected_decode_bbox_w2d = torch.tensor( + [[[[0.0206]], [[1.4788]], [[1.3904]], [[1.6013]], [[1.1548]], + [[1.0809]], [[0.9399]], [[13.3856]], [[2.0224]], [[4.8480]], + [[3.0368]], [[1.1424]], [[6.6304]], [[6.9456]], [[10.3072]], + [[4.7216]], [[4.6240]], [[7.1776]], [[4.5568]], [[1.7136]], + [[15.2480]], [[15.1360]], [[6.1152]], [[1.8640]], [[0.5222]], + [[1.1160]], [[0.0794]]], + [[[1.7224]], [[0.3360]], [[1.6765]], [[2.3401]], [[1.0384]], + [[1.4355]], [[0.9550]], [[8.3504]], [[2.2432]], [[10.9488]], + [[3.3936]], [[15.1488]], [[9.9808]], [[12.6688]], [[2.6336]], + [[0.8000]], [[10.0640]], [[6.3296]], [[4.6416]], [[7.3792]], + [[11.7328]], [[1.9104]], [[11.1984]], [[0.7960]], [[0.6524]], + [[1.4370]], [[0.8948]]]]) + assert torch.allclose(expected_decode_bbox_w2d, decode_bbox_w2d, atol=1e-3) + + # test decode_prob_depth + # [10, 8] + depth_cls_preds = torch.tensor([ + [-0.4383, 0.7207, -0.4092, 0.4649, 0.8526, 0.6186, -1.4312, -0.7150], + [0.0621, 0.2369, 0.5170, 0.8484, -0.1099, 0.1829, -0.0072, 1.0618], + [-1.6114, -0.1057, 0.5721, -0.5986, -2.0471, 0.8140, -0.8385, -0.4822], + [0.0742, -0.3261, 0.4607, 1.8155, -0.3571, -0.0234, 0.3787, 2.3251], + [1.0492, -0.6881, -0.0136, -1.8291, 0.8460, -1.0171, 2.5691, -0.8114], + [0.0968, -0.5601, 1.0458, 0.2560, 1.3018, 0.1635, 0.0680, -1.0263], + [-0.0765, 0.1498, -2.7321, 1.0047, -0.2505, 0.0871, -0.4820, -0.3003], + [-0.4123, 0.2298, -0.1330, -0.6008, 0.6526, 0.7118, 0.9728, -0.7793], + [1.6940, 0.3355, 1.4661, 0.5477, 0.8667, 0.0527, -0.9975, -0.0689], + [0.4724, -0.3632, -0.0654, 0.4034, -0.3494, -0.7548, 0.7297, 1.2754] + ]) + depth_range = (0, 70) + depth_unit = 10 + num_depth_cls = 8 + uniform_prob_depth_preds = bbox_coder.decode_prob_depth( + depth_cls_preds, depth_range, depth_unit, 'uniform', num_depth_cls) + expected_preds = torch.tensor([ + 32.0441, 38.4689, 36.1831, 48.2096, 46.1560, 32.7973, 33.2155, 39.9822, + 21.9905, 43.0161 + ]) + assert torch.allclose(uniform_prob_depth_preds, expected_preds, atol=1e-3) + + linear_prob_depth_preds = bbox_coder.decode_prob_depth( + depth_cls_preds, depth_range, depth_unit, 'linear', num_depth_cls) + expected_preds = torch.tensor([ + 21.1431, 30.2421, 25.8964, 41.6116, 38.6234, 21.4582, 23.2993, 30.1111, + 13.9273, 36.8419 + ]) + assert torch.allclose(linear_prob_depth_preds, expected_preds, atol=1e-3) + + log_prob_depth_preds = bbox_coder.decode_prob_depth( + depth_cls_preds, depth_range, depth_unit, 'log', num_depth_cls) + expected_preds = torch.tensor([ + 12.6458, 24.2487, 17.4015, 36.9375, 27.5982, 12.5510, 15.6635, 19.8408, + 9.1605, 31.3765 + ]) + assert torch.allclose(log_prob_depth_preds, expected_preds, atol=1e-3) + + loguniform_prob_depth_preds = bbox_coder.decode_prob_depth( + depth_cls_preds, depth_range, depth_unit, 'loguniform', num_depth_cls) + expected_preds = torch.tensor([ + 6.9925, 10.3273, 8.9895, 18.6524, 16.4667, 7.3196, 7.5078, 11.3207, + 3.7987, 13.6095 + ]) + assert torch.allclose( + loguniform_prob_depth_preds, expected_preds, atol=1e-3) From 38f75f58019fb78319327cf976c2a7afc0940098 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 22 Sep 2021 20:41:01 +0800 Subject: [PATCH 35/51] PGD Head initialized --- mmdet3d/models/dense_heads/__init__.py | 3 +- mmdet3d/models/dense_heads/pgd_head.py | 1185 ++++++++++++++++++++++++ 2 files changed, 1187 insertions(+), 1 deletion(-) create mode 100644 mmdet3d/models/dense_heads/pgd_head.py diff --git a/mmdet3d/models/dense_heads/__init__.py b/mmdet3d/models/dense_heads/__init__.py index d2283786db..2e7e69bcaf 100644 --- a/mmdet3d/models/dense_heads/__init__.py +++ b/mmdet3d/models/dense_heads/__init__.py @@ -8,6 +8,7 @@ from .free_anchor3d_head import FreeAnchor3DHead from .groupfree3d_head import GroupFree3DHead from .parta2_rpn_head import PartA2RPNHead +from .pgd_head import PGDHead from .shape_aware_head import ShapeAwareHead from .ssd_3d_head import SSD3DHead from .vote_head import VoteHead @@ -16,5 +17,5 @@ 'Anchor3DHead', 'FreeAnchor3DHead', 'PartA2RPNHead', 'VoteHead', 'SSD3DHead', 'BaseConvBboxHead', 'CenterHead', 'ShapeAwareHead', 'BaseMono3DDenseHead', 'AnchorFreeMono3DHead', 'FCOSMono3DHead', - 'GroupFree3DHead' + 'GroupFree3DHead', 'PGDHead' ] diff --git a/mmdet3d/models/dense_heads/pgd_head.py b/mmdet3d/models/dense_heads/pgd_head.py new file mode 100644 index 0000000000..ef796a375a --- /dev/null +++ b/mmdet3d/models/dense_heads/pgd_head.py @@ -0,0 +1,1185 @@ +import numpy as np +import torch +from mmcv.cnn import Scale, bias_init_with_prob, normal_init +from mmcv.runner import force_fp32 +from torch import nn as nn +from torch.nn import functional as F + +from mmdet3d.core import box3d_multiclass_nms, xywhr2xyxyr +from mmdet3d.core.bbox import points_cam2img, points_img2cam +from mmdet.core import distance2bbox, multi_apply +from mmdet.models.builder import HEADS, build_loss +from .fcos_mono3d_head import FCOSMono3DHead + + +@HEADS.register_module() +class PGDHead(FCOSMono3DHead): + r"""Anchor-free head used in `PGD `_. + + Args: + use_depth_classifer (bool, optional): Whether to use depth classifier. + Defaults to True. + use_only_reg_proj (bool, optional): Whether to use only direct + regressed depth in the re-projection (to make the network easier + to learn). Defaults to False. + weight_dim (int, optional): Dimension of the location-aware weight + map. Defaults to -1. + weight_branch (tuple[tuple[int]], optional): Feature map channels of + the convolutional branch for weight map. Defaults to ((256, ), ). + depth_branch (tuple[int], optional): Feature map channels of the + branch for probabilistic depth estimation. Defaults to (64, ), + depth_range (tuple[float], optional): Range of depth estimation. + Defaults to (0, 70), + depth_unit (int, optional): Unit of depth range division. Defaults to + 10. + division (str, optional): Depth division method. Options include + 'uniform', 'linear', 'log', 'loguniform'. Defaults to 'uniform'. + depth_bins (int, optional): Discrete bins of depth division. Defaults + to 8. + loss_depth (dict, optional): Depth loss. Defaults to dict( + type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0). + loss_bbox2d (dict, optional): Loss for 2D box estimation. Defaults to + dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0). + loss_consistency (dict, optional): Consistency loss. Defaults to + dict(type='GIoULoss', loss_weight=1.0), + pred_velo (bool, optional): Whether to predict velocity. Defaults to + False. + pred_bbox2d (bool, optional): Whether to predict 2D bounding boxes. + Defaults to True. + pred_keypoints (bool, optional): Whether to predict keypoints. + Defaults to False, + bbox_coder (dict, optional): Bounding box coder. Defaults to + dict(type='PGDBBoxCoder', base_depths=((28.01, 16.32), ), + base_dims=((0.8, 1.73, 0.6), (1.76, 1.73, 0.6), (3.9, 1.56, 1.6)), + code_size=7). + """ + + def __init__(self, + use_depth_classifier=True, + use_onlyreg_proj=False, + weight_dim=-1, + weight_branch=((256, ), ), + depth_branch=(64, ), + depth_range=(0, 70), + depth_unit=10, + division='uniform', + depth_bins=8, + loss_depth=dict( + type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0), + loss_bbox2d=dict( + type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0), + loss_consistency=dict(type='GIoULoss', loss_weight=1.0), + pred_velo=False, + pred_bbox2d=True, + pred_keypoints=False, + bbox_coder=dict( + type='PGDBBoxCoder', + base_depths=((28.01, 16.32), ), + base_dims=((0.8, 1.73, 0.6), (1.76, 1.73, 0.6), + (3.9, 1.56, 1.6)), + code_size=7, + depth_range=(0, 70), + depth_unit=10, + division='uniform', + depth_bins=8), + **kwargs): + self.use_depth_classifier = use_depth_classifier + self.use_onlyreg_proj = use_onlyreg_proj + self.depth_branch = depth_branch + self.pred_bbox2d = pred_bbox2d + self.pred_keypoints = pred_keypoints + self.weight_dim = weight_dim + self.weight_branch = weight_branch + self.weight_out_channels = [] + for weight_branch_channels in weight_branch: + if len(weight_branch_channels) > 0: + self.weight_out_channels.append(weight_branch_channels[-1]) + else: + self.weight_out_channels.append(-1) + self.depth_range = depth_range + self.depth_unit = depth_unit + self.division = division + if self.division == 'uniform': + self.num_depth_cls = int( + (depth_range[1] - depth_range[0]) / depth_unit) + 1 + if self.num_depth_cls != depth_bins: + print('Warning: The number of bins computed from ' + + 'depth_unit is different from given paramter! ' + + 'Depth_unit will be considered with priority in ' + + 'Uniform Division.') + else: + self.num_depth_cls = depth_bins + super().__init__(bbox_coder=bbox_coder, **kwargs) + self.loss_depth = build_loss(loss_depth) + if self.pred_bbox2d: + self.loss_bbox2d = build_loss(loss_bbox2d) + self.loss_consistency = build_loss(loss_consistency) + if self.pred_keypoints: + self.kpts_start = 9 if self.pred_velo else 7 + + def _init_layers(self): + """Initialize layers of the head.""" + super()._init_layers() + if self.pred_bbox2d: + self.scale_dim += 1 + if self.pred_keypoints: + self.scale_dim += 1 + self.scales = nn.ModuleList([ + nn.ModuleList([Scale(1.0) for _ in range(self.scale_dim)]) + for _ in self.strides + ]) + + def _init_predictor(self): + """Initialize predictor layers of the head.""" + super()._init_predictor() + + if self.use_depth_classifier: + self.conv_depth_cls_prev = self._init_branch( + conv_channels=self.depth_branch, + conv_strides=(1, ) * len(self.depth_branch)) + self.conv_depth_cls = nn.Conv2d(self.depth_branch[-1], + self.num_depth_cls, 1) + self.bld_alpha = nn.Parameter(torch.tensor(10e-5)) + + if self.weight_dim != -1: + self.conv_weight_prevs = nn.ModuleList() + self.conv_weights = nn.ModuleList() + for i in range(self.weight_dim): + weight_branch_channels = self.weight_branch[i] + weight_out_channel = self.weight_out_channels[i] + if len(weight_branch_channels) > 0: + self.conv_weight_prevs.append( + self._init_branch( + conv_channels=weight_branch_channels, + conv_strides=(1, ) * len(weight_branch_channels))) + self.conv_weights.append( + nn.Conv2d(weight_out_channel, 1, 1)) + else: + self.conv_weight_prevs.append(None) + self.conv_weights.append( + nn.Conv2d(self.feat_channels, 1, 1)) + + def init_weights(self): + """Initialize weights of the head.""" + super().init_weights() + + bias_cls = bias_init_with_prob(0.01) + if self.use_depth_classifier: + for m in self.conv_depth_cls_prev: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + normal_init(self.conv_depth_cls, std=0.01, bias=bias_cls) + + if self.weight_dim != -1: + for conv_weight_prev in self.conv_weight_prevs: + if conv_weight_prev is None: + continue + for m in conv_weight_prev: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + for conv_weight in self.conv_weights: + normal_init(conv_weight, std=0.01) + + def forward(self, feats): + """Forward features from the upstream network. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + tuple: + cls_scores (list[Tensor]): Box scores for each scale level, + each is a 4D-tensor, the channel number is + num_points * num_classes. + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level, each is a 4D-tensor, the channel number is + num_points * bbox_code_size. + dir_cls_preds (list[Tensor]): Box scores for direction class + predictions on each scale level, each is a 4D-tensor, + the channel number is num_points * 2. (bin = 2). + depth_cls_preds (list[Tensor]): Box scores for depth class + predictions on each scale level, each is a 4D-tensor, + the channel number is num_points * self.num_depth_cls. + attr_preds (list[Tensor]): Attribute scores for each scale + level, each is a 4D-tensor, the channel number is + num_points * num_attrs. + centernesses (list[Tensor]): Centerness for each scale level, + each is a 4D-tensor, the channel number is num_points * 1. + """ + return multi_apply(self.forward_single, feats, self.scales, + self.strides) + + def forward_single(self, x, scale, stride): + """Forward features of a single scale levle. + + Args: + x (Tensor): FPN feature maps of the specified stride. + scale (:obj: `mmcv.cnn.Scale`): Learnable scale module to resize + the bbox prediction. + stride (int): The corresponding stride for feature maps, only + used to normalize the bbox prediction when self.norm_on_bbox + is True. + + Returns: + tuple: scores for each class, bbox and direction class \ + predictions, centerness predictions of input feature maps. + """ + cls_score, bbox_pred, dir_cls_pred, attr_pred, centerness, cls_feat, \ + reg_feat = super().forward_single(x, scale, stride) + + max_regress_range = stride * self.regress_ranges[0][1] / \ + self.strides[0] + bbox_pred = self.bbox_coder.decode_2d(bbox_pred, scale, stride, + max_regress_range, self.training, + self.pred_keypoints, + self.pred_bbox2d) + + depth_cls_pred = None + if self.use_depth_classifier: + clone_reg_feat = reg_feat.clone() + for conv_depth_cls_prev_layer in self.conv_depth_cls_prev: + clone_reg_feat = conv_depth_cls_prev_layer(clone_reg_feat) + depth_cls_pred = self.conv_depth_cls(clone_reg_feat) + + weight = None + if self.weight_dim != -1: + weight = [] + for i in range(self.weight_dim): + clone_reg_feat = reg_feat.clone() + if len(self.weight_branch[i]) > 0: + for conv_weight_prev_layer in self.conv_weight_prevs[i]: + clone_reg_feat = conv_weight_prev_layer(clone_reg_feat) + weight.append(self.conv_weights[i](clone_reg_feat)) + weight = torch.cat(weight, dim=1) + + return cls_score, bbox_pred, dir_cls_pred, depth_cls_pred, weight, \ + attr_pred, centerness + + def get_proj_bbox2d(self, + bbox_preds, + pos_dir_cls_preds, + labels_3d, + bbox_targets_3d, + pos_points, + pos_inds, + img_metas, + pos_depth_cls_preds=None, + pos_weights=None, + pos_cls_scores=None, + with_kpts=False): + """Add prob depth and geo depth related operations.""" + views = [np.array(img_meta['cam2img']) for img_meta in img_metas] + num_imgs = len(img_metas) + img_idx = [] + for label in labels_3d: + for idx in range(num_imgs): + img_idx.append( + labels_3d[0].new_ones(int(len(label) / num_imgs)) * idx) + img_idx = torch.cat(img_idx) + pos_img_idx = img_idx[pos_inds] + + flatten_strided_bbox_preds = [] + flatten_strided_bbox2d_preds = [] + flatten_bbox_targets_3d = [] + flatten_strides = [] + + for stride_idx, bbox_pred in enumerate(bbox_preds): + flatten_bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape( + -1, sum(self.group_reg_dims)) + flatten_bbox_pred[:, :2] *= self.strides[stride_idx] + flatten_bbox_pred[:, -4:] *= self.strides[stride_idx] + flatten_strided_bbox_preds.append( + flatten_bbox_pred[:, :self.bbox_coder.bbox_code_size]) + flatten_strided_bbox2d_preds.append(flatten_bbox_pred[:, -4:]) + + bbox_target_3d = bbox_targets_3d[stride_idx].clone() + bbox_target_3d[:, :2] *= self.strides[stride_idx] + bbox_target_3d[:, -4:] *= self.strides[stride_idx] + flatten_bbox_targets_3d.append(bbox_target_3d) + + flatten_stride = flatten_bbox_pred.new_ones( + *flatten_bbox_pred.shape[:-1], 1) * self.strides[stride_idx] + flatten_strides.append(flatten_stride) + + flatten_strided_bbox_preds = torch.cat(flatten_strided_bbox_preds) + flatten_strided_bbox2d_preds = torch.cat(flatten_strided_bbox2d_preds) + flatten_bbox_targets_3d = torch.cat(flatten_bbox_targets_3d) + flatten_strides = torch.cat(flatten_strides) + pos_strided_bbox_preds = flatten_strided_bbox_preds[pos_inds] + pos_strided_bbox2d_preds = flatten_strided_bbox2d_preds[pos_inds] + pos_bbox_targets_3d = flatten_bbox_targets_3d[pos_inds] + pos_strides = flatten_strides[pos_inds] + + pos_decoded_bbox2d_preds = distance2bbox(pos_points, + pos_strided_bbox2d_preds) + + pos_strided_bbox_preds[:, :2] = \ + pos_points - pos_strided_bbox_preds[:, :2] + pos_bbox_targets_3d[:, :2] = \ + pos_points - pos_bbox_targets_3d[:, :2] + + if self.use_depth_classifier and (not self.use_onlyreg_proj): + pos_prob_depth_preds = self.bbox_coder.decode_prob_depth( + pos_depth_cls_preds, self.depth_range, self.depth_unit, + self.division, self.num_depth_cls) + sig_alpha = torch.sigmoid(self.bld_alpha) + pos_strided_bbox_preds[:, 2] = \ + sig_alpha * pos_strided_bbox_preds.clone()[:, 2] + \ + (1 - sig_alpha) * pos_prob_depth_preds + + box_corners_in_image = pos_strided_bbox_preds.new_zeros( + (*pos_strided_bbox_preds.shape[:-1], 8, 2)) + box_corners_in_image_gt = pos_strided_bbox_preds.new_zeros( + (*pos_strided_bbox_preds.shape[:-1], 8, 2)) + + for idx in range(num_imgs): + mask = (pos_img_idx == idx) + if pos_strided_bbox_preds[mask].shape[0] == 0: + continue + cam2img = pos_strided_bbox_preds.new_zeros((4, 4)) + view_shape = views[idx].shape + cam2img[:view_shape[0], :view_shape[1]] = \ + pos_strided_bbox_preds.new_tensor(views[idx]) + + centers2d_preds = pos_strided_bbox_preds.clone()[mask, :2] + centers2d_targets = pos_bbox_targets_3d.clone()[mask, :2] + centers3d_targets = points_img2cam(pos_bbox_targets_3d[mask, :3], + views[idx]) + + # use predicted depth to re-project the 2.5D centers + pos_strided_bbox_preds[mask, :3] = points_img2cam( + pos_strided_bbox_preds[mask, :3], views[idx]) + pos_bbox_targets_3d[mask, :3] = centers3d_targets + + # depth fixed + pos_strided_bbox_preds[mask, 2] = \ + pos_bbox_targets_3d.clone()[mask, 2] + if self.use_direction_classifier: + pos_dir_cls_scores = torch.max( + pos_dir_cls_preds[mask], dim=-1)[1] + pos_strided_bbox_preds[mask] = self.bbox_coder.decode_yaw( + pos_strided_bbox_preds[mask], centers2d_preds, + pos_dir_cls_scores, self.dir_offset, cam2img) + pos_bbox_targets_3d[mask, 6] = torch.atan2( + centers2d_targets[:, 0] - cam2img[0, 2], + cam2img[0, 0]) + pos_bbox_targets_3d[mask, 6] + + corners = img_metas[0]['box_type_3d']( + pos_strided_bbox_preds[mask], + box_dim=self.bbox_coder.bbox_code_size, + origin=(0.5, 0.5, 0.5)).corners + box_corners_in_image[mask] = points_cam2img(corners, cam2img) + + corners_gt = img_metas[0]['box_type_3d']( + pos_bbox_targets_3d[mask, :self.bbox_code_size], + box_dim=self.bbox_coder.bbox_code_size, + origin=(0.5, 0.5, 0.5)).corners + box_corners_in_image_gt[mask] = points_cam2img(corners_gt, cam2img) + + minxy = torch.min(box_corners_in_image, dim=1)[0] + maxxy = torch.max(box_corners_in_image, dim=1)[0] + proj_bbox2d_preds = torch.cat([minxy, maxxy], dim=1) + + outputs = (proj_bbox2d_preds, pos_decoded_bbox2d_preds) + + if with_kpts: + norm_strides = pos_strides * self.regress_ranges[0][1] / \ + self.strides[0] + kpts_targets = box_corners_in_image_gt - pos_points[..., None, :] + kpts_targets = kpts_targets.view( + (*pos_strided_bbox_preds.shape[:-1], 16)) + kpts_targets /= norm_strides + + outputs += (kpts_targets, ) + + return outputs + + def get_pos_predictions(self, bbox_preds, dir_cls_preds, depth_cls_preds, + weights, attr_preds, centernesses, pos_inds, + img_metas): + """Flatten predictions and get positive ones. + + Args: + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level, each is a 4D-tensor, the channel number is + num_points * bbox_code_size. + dir_cls_preds (list[Tensor]): Box scores for direction class + predictions on each scale level, each is a 4D-tensor, + the channel number is num_points * 2. (bin = 2) + depth_cls_preds (list[Tensor]): Box scores for direction class + predictions on each scale level, each is a 4D-tensor, + the channel number is num_points * self.num_depth_cls. + attr_preds (list[Tensor]): Attribute scores for each scale level, + each is a 4D-tensor, the channel number is + num_points * num_attrs. + centernesses (list[Tensor]): Centerness for each scale level, each + is a 4D-tensor, the channel number is num_points * 1. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + flatten_bbox_preds = [ + bbox_pred.permute(0, 2, 3, 1).reshape(-1, sum(self.group_reg_dims)) + for bbox_pred in bbox_preds + ] + flatten_dir_cls_preds = [ + dir_cls_pred.permute(0, 2, 3, 1).reshape(-1, 2) + for dir_cls_pred in dir_cls_preds + ] + flatten_centerness = [ + centerness.permute(0, 2, 3, 1).reshape(-1) + for centerness in centernesses + ] + flatten_bbox_preds = torch.cat(flatten_bbox_preds) + flatten_dir_cls_preds = torch.cat(flatten_dir_cls_preds) + flatten_centerness = torch.cat(flatten_centerness) + pos_bbox_preds = flatten_bbox_preds[pos_inds] + pos_dir_cls_preds = flatten_dir_cls_preds[pos_inds] + pos_centerness = flatten_centerness[pos_inds] + + pos_depth_cls_preds = None + if self.use_depth_classifier: + flatten_depth_cls_preds = [ + depth_cls_pred.permute(0, 2, 3, + 1).reshape(-1, self.num_depth_cls) + for depth_cls_pred in depth_cls_preds + ] + flatten_depth_cls_preds = torch.cat(flatten_depth_cls_preds) + pos_depth_cls_preds = flatten_depth_cls_preds[pos_inds] + + pos_weights = None + if self.weight_dim != -1: + flatten_weights = [ + weight.permute(0, 2, 3, 1).reshape(-1, self.weight_dim) + for weight in weights + ] + flatten_weights = torch.cat(flatten_weights) + pos_weights = flatten_weights[pos_inds] + + pos_attr_preds = None + if self.pred_attrs: + flatten_attr_preds = [ + attr_pred.permute(0, 2, 3, 1).reshape(-1, self.num_attrs) + for attr_pred in attr_preds + ] + flatten_attr_preds = torch.cat(flatten_attr_preds) + pos_attr_preds = flatten_attr_preds[pos_inds] + + return pos_bbox_preds, pos_dir_cls_preds, pos_depth_cls_preds, \ + pos_weights, pos_attr_preds, pos_centerness + + @force_fp32( + apply_to=('cls_scores', 'bbox_preds', 'dir_cls_preds', + 'depth_cls_preds', 'weights', 'attr_preds', 'centernesses')) + def loss(self, + cls_scores, + bbox_preds, + dir_cls_preds, + depth_cls_preds, + weights, + attr_preds, + centernesses, + gt_bboxes, + gt_labels, + gt_bboxes_3d, + gt_labels_3d, + centers2d, + depths, + attr_labels, + img_metas, + gt_bboxes_ignore=None): + """Compute loss of the head. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level, + each is a 4D-tensor, the channel number is + num_points * num_classes. + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level, each is a 4D-tensor, the channel number is + num_points * bbox_code_size. + dir_cls_preds (list[Tensor]): Box scores for direction class + predictions on each scale level, each is a 4D-tensor, + the channel number is num_points * 2. (bin = 2) + depth_cls_preds (list[Tensor]): Box scores for direction class + predictions on each scale level, each is a 4D-tensor, + the channel number is num_points * self.num_depth_cls. + attr_preds (list[Tensor]): Attribute scores for each scale level, + each is a 4D-tensor, the channel number is + num_points * num_attrs. + centernesses (list[Tensor]): Centerness for each scale level, each + is a 4D-tensor, the channel number is num_points * 1. + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box + gt_bboxes_3d (list[Tensor]): 3D boxes ground truth with shape of + (num_gts, code_size). + gt_labels_3d (list[Tensor]): same as gt_labels + centers2d (list[Tensor]): 2D centers on the image with shape of + (num_gts, 2). + depths (list[Tensor]): Depth ground truth with shape of + (num_gts, ). + attr_labels (list[Tensor]): Attributes indices of each box. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. + + Returns: + dict[str, Tensor]: A dictionary of loss components. + """ + assert len(cls_scores) == len(bbox_preds) == len(dir_cls_preds) == \ + len(depth_cls_preds) == len(weights) == len(centernesses) == \ + len(attr_preds) + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + all_level_points = self.get_points(featmap_sizes, bbox_preds[0].dtype, + bbox_preds[0].device) + labels_3d, bbox_targets_3d, centerness_targets, attr_targets = \ + self.get_targets( + all_level_points, gt_bboxes, gt_labels, gt_bboxes_3d, + gt_labels_3d, centers2d, depths, attr_labels) + + num_imgs = cls_scores[0].size(0) + # flatten cls_scores and targets + flatten_cls_scores = [ + cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) + for cls_score in cls_scores + ] + flatten_cls_scores = torch.cat(flatten_cls_scores) + flatten_labels_3d = torch.cat(labels_3d) + flatten_bbox_targets_3d = torch.cat(bbox_targets_3d) + flatten_centerness_targets = torch.cat(centerness_targets) + flatten_points = torch.cat( + [points.repeat(num_imgs, 1) for points in all_level_points]) + if self.pred_attrs: + flatten_attr_targets = torch.cat(attr_targets) + + # FG cat_id: [0, num_classes -1], BG cat_id: num_classes + bg_class_ind = self.num_classes + pos_inds = ((flatten_labels_3d >= 0) + & (flatten_labels_3d < bg_class_ind)).nonzero().reshape(-1) + num_pos = len(pos_inds) + + loss_cls = self.loss_cls( + flatten_cls_scores, + flatten_labels_3d, + avg_factor=num_pos + num_imgs) # avoid num_pos is 0 + + pos_bbox_preds, pos_dir_cls_preds, pos_depth_cls_preds, pos_weights, \ + pos_attr_preds, pos_centerness = self.get_pos_predictions( + bbox_preds, dir_cls_preds, depth_cls_preds, weights, + attr_preds, centernesses, pos_inds, img_metas) + + if num_pos > 0: + pos_bbox_targets_3d = flatten_bbox_targets_3d[pos_inds] + pos_centerness_targets = flatten_centerness_targets[pos_inds] + pos_points = flatten_points[pos_inds] + if self.pred_attrs: + pos_attr_targets = flatten_attr_targets[pos_inds] + if self.use_direction_classifier: + pos_dir_cls_targets = self.get_direction_target( + pos_bbox_targets_3d, self.dir_offset, one_hot=False) + + bbox_weights = pos_centerness_targets.new_ones( + len(pos_centerness_targets), sum(self.group_reg_dims)) + equal_weights = pos_centerness_targets.new_ones( + pos_centerness_targets.shape) + code_weight = self.train_cfg.get('code_weight', None) + if code_weight: + assert len(code_weight) == sum(self.group_reg_dims) + bbox_weights = bbox_weights * bbox_weights.new_tensor( + code_weight) + + if self.diff_rad_by_sin: + pos_bbox_preds, pos_bbox_targets_3d = self.add_sin_difference( + pos_bbox_preds, pos_bbox_targets_3d) + + loss_offset = self.loss_bbox( + pos_bbox_preds[:, :2], + pos_bbox_targets_3d[:, :2], + weight=bbox_weights[:, :2], + avg_factor=equal_weights.sum()) + loss_depth = self.loss_bbox( + pos_bbox_preds[:, 2], + pos_bbox_targets_3d[:, 2], + weight=bbox_weights[:, 2], + avg_factor=equal_weights.sum()) + loss_size = self.loss_bbox( + pos_bbox_preds[:, 3:6], + pos_bbox_targets_3d[:, 3:6], + weight=bbox_weights[:, 3:6], + avg_factor=equal_weights.sum()) + loss_rotsin = self.loss_bbox( + pos_bbox_preds[:, 6], + pos_bbox_targets_3d[:, 6], + weight=bbox_weights[:, 6], + avg_factor=equal_weights.sum()) + if self.pred_velo: + loss_velo = self.loss_bbox( + pos_bbox_preds[:, 7:9], + pos_bbox_targets_3d[:, 7:9], + weight=bbox_weights[:, 7:9], + avg_factor=equal_weights.sum()) + + proj_bbox2d_inputs = (bbox_preds, pos_dir_cls_preds, labels_3d, + bbox_targets_3d, pos_points, pos_inds, + img_metas) + + # direction classification loss + # TODO: add more check for use_direction_classifier + if self.use_direction_classifier: + loss_dir = self.loss_dir( + pos_dir_cls_preds, + pos_dir_cls_targets, + equal_weights, + avg_factor=equal_weights.sum()) + + # depth classification loss + if self.use_depth_classifier: + pos_prob_depth_preds = self.bbox_coder.decode_prob_depth( + pos_depth_cls_preds, self.depth_range, self.depth_unit, + self.division, self.num_depth_cls) + sig_alpha = torch.sigmoid(self.bld_alpha) + if self.weight_dim != -1: + loss_depth_bld = self.loss_depth( + sig_alpha * pos_bbox_preds[:, 2] + + (1 - sig_alpha) * pos_prob_depth_preds, + pos_bbox_targets_3d[:, 2], + sigma=pos_weights[:, 0], + weight=bbox_weights[:, 2], + avg_factor=equal_weights.sum()) + else: + loss_depth_bld = self.loss_depth( + sig_alpha * pos_bbox_preds[:, 2] + + (1 - sig_alpha) * pos_prob_depth_preds, + pos_bbox_targets_3d[:, 2], + weight=bbox_weights[:, 2], + avg_factor=equal_weights.sum()) + + proj_bbox2d_inputs += (pos_depth_cls_preds, ) + + if self.pred_keypoints: + # use smoothL1 to compute consistency loss for keypoints + # normalize the offsets with strides + proj_bbox2d_preds, pos_decoded_bbox2d_preds, kpts_targets = \ + self.get_proj_bbox2d(*proj_bbox2d_inputs, with_kpts=True) + loss_kpts = self.loss_bbox( + pos_bbox_preds[:, self.kpts_start:self.kpts_start + 16], + kpts_targets, + weight=bbox_weights[:, + self.kpts_start:self.kpts_start + 16], + avg_factor=equal_weights.sum()) + + if self.pred_bbox2d: + loss_bbox2d = self.loss_bbox2d( + pos_bbox_preds[:, -4:], + pos_bbox_targets_3d[:, -4:], + weight=bbox_weights[:, -4:], + avg_factor=equal_weights.sum()) + if not self.pred_keypoints: + proj_bbox2d_preds, pos_decoded_bbox2d_preds = \ + self.get_proj_bbox2d(*proj_bbox2d_inputs) + loss_consistency = self.loss_consistency( + proj_bbox2d_preds, + pos_decoded_bbox2d_preds, + weight=bbox_weights[:, -4:], + avg_factor=equal_weights.sum()) + + loss_centerness = self.loss_centerness(pos_centerness, + pos_centerness_targets) + + # attribute classification loss + if self.pred_attrs: + loss_attr = self.loss_attr( + pos_attr_preds, + pos_attr_targets, + pos_centerness_targets, + avg_factor=pos_centerness_targets.sum()) + + else: + # need absolute due to possible negative delta x/y + loss_offset = pos_bbox_preds[:, :2].sum() + loss_depth = pos_bbox_preds[:, 2].sum() + loss_size = pos_bbox_preds[:, 3:6].sum() + loss_rotsin = pos_bbox_preds[:, 6].sum() + if self.pred_velo: + loss_velo = pos_bbox_preds[:, 7:9].sum() + if self.pred_keypoints: + loss_kpts = pos_bbox_preds[:, self.kpts_start:self.kpts_start + + 16].sum() + if self.pred_bbox2d: + loss_bbox2d = pos_bbox_preds[:, -4:].sum() + loss_consistency = pos_bbox_preds[:, -4:].sum() + loss_centerness = pos_centerness.sum() + if self.use_direction_classifier: + loss_dir = pos_dir_cls_preds.sum() + if self.use_depth_classifier: + sig_alpha = torch.sigmoid(self.bld_alpha) + if self.weight_dim != -1: + loss_depth_bld *= torch.exp(-pos_weights[:, 0].sum()) + else: + loss_depth_bld = \ + sig_alpha * pos_bbox_preds[:, 2].sum() + \ + (1 - sig_alpha) * pos_depth_cls_preds.sum() + if self.pred_attrs: + loss_attr = pos_attr_preds.sum() + + loss_dict = dict( + loss_cls=loss_cls, + loss_offset=loss_offset, + loss_depth=loss_depth, + loss_size=loss_size, + loss_rotsin=loss_rotsin, + loss_centerness=loss_centerness) + + if self.pred_velo: + loss_dict['loss_velo'] = loss_velo + + if self.pred_keypoints: + loss_dict['loss_kpts'] = loss_kpts + + if self.pred_bbox2d: + loss_dict['loss_bbox2d'] = loss_bbox2d + loss_dict['loss_consistency'] = loss_consistency + + if self.use_direction_classifier: + loss_dict['loss_dir'] = loss_dir + + if self.use_depth_classifier: + loss_dict['loss_depth'] = loss_depth_bld + + if self.pred_attrs: + loss_dict['loss_attr'] = loss_attr + + return loss_dict + + @force_fp32( + apply_to=('cls_scores', 'bbox_preds', 'dir_cls_preds', + 'depth_cls_preds', 'weights', 'attr_preds', 'centernesses')) + def get_bboxes(self, + cls_scores, + bbox_preds, + dir_cls_preds, + depth_cls_preds, + weights, + attr_preds, + centernesses, + img_metas, + cfg=None, + rescale=None): + """Transform network output for a batch into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box scores for each scale level + Has shape (N, num_points * num_classes, H, W) + bbox_preds (list[Tensor]): Box energies / deltas for each scale + level with shape (N, num_points * 4, H, W) + dir_cls_preds (list[Tensor]): Box scores for direction class + predictions on each scale level, each is a 4D-tensor, + the channel number is num_points * 2. (bin = 2) + depth_cls_preds (list[Tensor]): Box scores for direction class + predictions on each scale level, each is a 4D-tensor, + the channel number is num_points * self.num_depth_cls. + attr_preds (list[Tensor]): Attribute scores for each scale level + Has shape (N, num_points * num_attrs, H, W) + centernesses (list[Tensor]): Centerness for each scale level with + shape (N, num_points * 1, H, W) + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used + rescale (bool): If True, return boxes in original image space + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. \ + The first item is an (n, 5) tensor, where the first 4 columns \ + are bounding box positions (tl_x, tl_y, br_x, br_y) and the \ + 5-th column is a score between 0 and 1. The second item is a \ + (n,) tensor where each item is the predicted class label of \ + the corresponding box. + """ + assert len(cls_scores) == len(bbox_preds) == len(dir_cls_preds) == \ + len(depth_cls_preds) == len(weights) == len(centernesses) == \ + len(attr_preds) + num_levels = len(cls_scores) + + featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] + mlvl_points = self.get_points(featmap_sizes, bbox_preds[0].dtype, + bbox_preds[0].device) + result_list = [] + for img_id in range(len(img_metas)): + cls_score_list = [ + cls_scores[i][img_id].detach() for i in range(num_levels) + ] + bbox_pred_list = [ + bbox_preds[i][img_id].detach() for i in range(num_levels) + ] + if self.use_direction_classifier: + dir_cls_pred_list = [ + dir_cls_preds[i][img_id].detach() + for i in range(num_levels) + ] + else: + dir_cls_pred_list = [ + cls_scores[i][img_id].new_full( + [2, *cls_scores[i][img_id].shape[1:]], 0).detach() + for i in range(num_levels) + ] + if self.use_depth_classifier: + depth_cls_pred_list = [ + depth_cls_preds[i][img_id].detach() + for i in range(num_levels) + ] + else: + depth_cls_pred_list = [ + cls_scores[i][img_id].new_full( + [self.num_depth_cls, *cls_scores[i][img_id].shape[1:]], + 0).detach() for i in range(num_levels) + ] + if self.weight_dim != -1: + weight_list = [ + weights[i][img_id].detach() for i in range(num_levels) + ] + else: + weight_list = [ + cls_scores[i][img_id].new_full( + [1, *cls_scores[i][img_id].shape[1:]], 0).detach() + for i in range(num_levels) + ] + if self.pred_attrs: + attr_pred_list = [ + attr_preds[i][img_id].detach() for i in range(num_levels) + ] + else: + attr_pred_list = [ + cls_scores[i][img_id].new_full( + [self.num_attrs, *cls_scores[i][img_id].shape[1:]], + self.attr_background_label).detach() + for i in range(num_levels) + ] + centerness_pred_list = [ + centernesses[i][img_id].detach() for i in range(num_levels) + ] + input_meta = img_metas[img_id] + det_bboxes = self._get_bboxes_single( + cls_score_list, bbox_pred_list, dir_cls_pred_list, + depth_cls_pred_list, weight_list, attr_pred_list, + centerness_pred_list, mlvl_points, input_meta, cfg, rescale) + result_list.append(det_bboxes) + return result_list + + def _get_bboxes_single(self, + cls_scores, + bbox_preds, + dir_cls_preds, + depth_cls_preds, + weights, + attr_preds, + centernesses, + mlvl_points, + input_meta, + cfg, + rescale=False): + """Transform outputs for a single batch item into bbox predictions. + + Args: + cls_scores (list[Tensor]): Box scores for a single scale level + Has shape (num_points * num_classes, H, W). + bbox_preds (list[Tensor]): Box energies / deltas for a single scale + level with shape (num_points * bbox_code_size, H, W). + dir_cls_preds (list[Tensor]): Box scores for direction class + predictions on a single scale level with shape \ + (num_points * 2, H, W) + depth_cls_preds (list[Tensor]): Box scores for direction class + predictions on a single scale level with shape \ + (num_points * self.num_depth_cls, H, W) + attr_preds (list[Tensor]): Attribute scores for each scale level + Has shape (N, num_points * num_attrs, H, W) + centernesses (list[Tensor]): Centerness for a single scale level + with shape (num_points, H, W). + mlvl_points (list[Tensor]): Box reference for a single scale level + with shape (num_total_points, 2). + input_meta (dict): Metadata of input image. + cfg (mmcv.Config): Test / postprocessing configuration, + if None, test_cfg would be used. + rescale (bool): If True, return boxes in original image space. + + Returns: + tuples[Tensor]: Predicted 3D boxes, scores, labels and attributes. + """ + view = np.array(input_meta['cam2img']) + scale_factor = input_meta['scale_factor'] + cfg = self.test_cfg if cfg is None else cfg + assert len(cls_scores) == len(bbox_preds) == len(mlvl_points) + mlvl_centers2d = [] + mlvl_bboxes = [] + mlvl_scores = [] + mlvl_dir_scores = [] + mlvl_attr_scores = [] + mlvl_centerness = [] + mlvl_depth_cls_scores = [] + mlvl_depth_uncertainty = [] + mlvl_bboxes2d = None + if self.pred_bbox2d: + mlvl_bboxes2d = [] + + for cls_score, bbox_pred, dir_cls_pred, depth_cls_pred, weight, \ + attr_pred, centerness, points in zip( + cls_scores, bbox_preds, dir_cls_preds, depth_cls_preds, + weights, attr_preds, centernesses, mlvl_points): + assert cls_score.size()[-2:] == bbox_pred.size()[-2:] + scores = cls_score.permute(1, 2, 0).reshape( + -1, self.cls_out_channels).sigmoid() + dir_cls_pred = dir_cls_pred.permute(1, 2, 0).reshape(-1, 2) + dir_cls_score = torch.max(dir_cls_pred, dim=-1)[1] + depth_cls_pred = depth_cls_pred.permute(1, 2, 0).reshape( + -1, self.num_depth_cls) + depth_cls_score = F.softmax( + depth_cls_pred, dim=-1).topk( + k=2, dim=-1)[0].mean(dim=-1) + if self.weight_dim != -1: + weight = weight.permute(1, 2, 0).reshape(-1, self.weight_dim) + else: + weight = weight.permute(1, 2, 0).reshape(-1, 1) + depth_uncertainty = torch.exp(-weight[:, -1]) + attr_pred = attr_pred.permute(1, 2, 0).reshape(-1, self.num_attrs) + attr_score = torch.max(attr_pred, dim=-1)[1] + centerness = centerness.permute(1, 2, 0).reshape(-1).sigmoid() + + bbox_pred = bbox_pred.permute(1, 2, + 0).reshape(-1, + sum(self.group_reg_dims)) + bbox_pred3d = bbox_pred[:, :self.bbox_coder.bbox_code_size] + if self.pred_bbox2d: + bbox_pred2d = bbox_pred[:, -4:] + nms_pre = cfg.get('nms_pre', -1) + if nms_pre > 0 and scores.shape[0] > nms_pre: + merged_scores = scores * centerness[:, None] + if self.use_depth_classifier: + merged_scores *= depth_cls_score[:, None] + if self.weight_dim != -1: + merged_scores *= depth_uncertainty[:, None] + max_scores, _ = merged_scores.max(dim=1) + _, topk_inds = max_scores.topk(nms_pre) + points = points[topk_inds, :] + bbox_pred3d = bbox_pred3d[topk_inds, :] + scores = scores[topk_inds, :] + dir_cls_pred = dir_cls_pred[topk_inds, :] + depth_cls_pred = depth_cls_pred[topk_inds, :] + centerness = centerness[topk_inds] + dir_cls_score = dir_cls_score[topk_inds] + depth_cls_score = depth_cls_score[topk_inds] + depth_uncertainty = depth_uncertainty[topk_inds] + attr_score = attr_score[topk_inds] + if self.pred_bbox2d: + bbox_pred2d = bbox_pred2d[topk_inds, :] + # change the offset to actual center predictions + bbox_pred3d[:, :2] = points - bbox_pred3d[:, :2] + if rescale: + bbox_pred3d[:, :2] /= bbox_pred3d[:, :2].new_tensor( + scale_factor) + if self.pred_bbox2d: + bbox_pred2d /= bbox_pred2d.new_tensor(scale_factor) + if self.use_depth_classifier: + prob_depth_pred = self.bbox_coder.decode_prob_depth( + depth_cls_pred, self.depth_range, self.depth_unit, + self.division, self.num_depth_cls) + sig_alpha = torch.sigmoid(self.bld_alpha) + bbox_pred3d[:, 2] = sig_alpha * bbox_pred3d[:, 2] + \ + (1 - sig_alpha) * prob_depth_pred + pred_center2d = bbox_pred3d[:, :3].clone() + bbox_pred3d[:, :3] = points_img2cam(bbox_pred3d[:, :3], view) + mlvl_centers2d.append(pred_center2d) + mlvl_bboxes.append(bbox_pred3d) + mlvl_scores.append(scores) + mlvl_dir_scores.append(dir_cls_score) + mlvl_depth_cls_scores.append(depth_cls_score) + mlvl_attr_scores.append(attr_score) + mlvl_centerness.append(centerness) + mlvl_depth_uncertainty.append(depth_uncertainty) + if self.pred_bbox2d: + bbox_pred2d = distance2bbox( + points, bbox_pred2d, max_shape=input_meta['img_shape']) + mlvl_bboxes2d.append(bbox_pred2d) + + mlvl_centers2d = torch.cat(mlvl_centers2d) + mlvl_bboxes = torch.cat(mlvl_bboxes) + mlvl_dir_scores = torch.cat(mlvl_dir_scores) + if self.pred_bbox2d: + mlvl_bboxes2d = torch.cat(mlvl_bboxes2d) + + # change local yaw to global yaw for 3D nms + cam2img = mlvl_centers2d.new_zeros((4, 4)) + cam2img[:view.shape[0], :view.shape[1]] = \ + mlvl_centers2d.new_tensor(view) + mlvl_bboxes = self.bbox_coder.decode_yaw(mlvl_bboxes, mlvl_centers2d, + mlvl_dir_scores, + self.dir_offset, cam2img) + + mlvl_bboxes_for_nms = xywhr2xyxyr(input_meta['box_type_3d']( + mlvl_bboxes, + box_dim=self.bbox_coder.bbox_code_size, + origin=(0.5, 0.5, 0.5)).bev) + + mlvl_scores = torch.cat(mlvl_scores) + padding = mlvl_scores.new_zeros(mlvl_scores.shape[0], 1) + # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 + # BG cat_id: num_class + mlvl_scores = torch.cat([mlvl_scores, padding], dim=1) + mlvl_attr_scores = torch.cat(mlvl_attr_scores) + mlvl_centerness = torch.cat(mlvl_centerness) + # no scale_factors in box3d_multiclass_nms + # Then we multiply it from outside + mlvl_nms_scores = mlvl_scores * mlvl_centerness[:, None] + if self.use_depth_classifier: # multiply the depth confidence + mlvl_depth_cls_scores = torch.cat(mlvl_depth_cls_scores) + mlvl_nms_scores *= mlvl_depth_cls_scores[:, None] + if self.weight_dim != -1: + mlvl_depth_uncertainty = torch.cat(mlvl_depth_uncertainty) + mlvl_nms_scores *= mlvl_depth_uncertainty[:, None] + results = box3d_multiclass_nms(mlvl_bboxes, mlvl_bboxes_for_nms, + mlvl_nms_scores, cfg.score_thr, + cfg.max_per_img, cfg, mlvl_dir_scores, + mlvl_attr_scores, mlvl_bboxes2d) + bboxes, scores, labels, dir_scores, attrs = results[0:5] + attrs = attrs.to(labels.dtype) # change data type to int + bboxes = input_meta['box_type_3d']( + bboxes, + box_dim=self.bbox_coder.bbox_code_size, + origin=(0.5, 0.5, 0.5)) + # Note that the predictions use origin (0.5, 0.5, 0.5) + # Due to the ground truth centers2d are the gravity center of objects + # v0.10.0 fix inplace operation to the input tensor of cam_box3d + # So here we also need to add origin=(0.5, 0.5, 0.5) + if not self.pred_attrs: + attrs = None + + outputs = (bboxes, scores, labels, attrs) + if self.pred_bbox2d: + bboxes2d = results[-1] + bboxes2d = torch.cat([bboxes2d, scores[:, None]], dim=1) + outputs = outputs + (bboxes2d, ) + + return outputs + + def get_targets(self, points, gt_bboxes_list, gt_labels_list, + gt_bboxes_3d_list, gt_labels_3d_list, centers2d_list, + depths_list, attr_labels_list): + """Compute regression, classification and centerss targets for points + in multiple images. + + Args: + points (list[Tensor]): Points of each fpn level, each has shape + (num_points, 2). + gt_bboxes_list (list[Tensor]): Ground truth bboxes of each image, + each has shape (num_gt, 4). + gt_labels_list (list[Tensor]): Ground truth labels of each box, + each has shape (num_gt,). + gt_bboxes_3d_list (list[Tensor]): 3D Ground truth bboxes of each + image, each has shape (num_gt, bbox_code_size). + gt_labels_3d_list (list[Tensor]): 3D Ground truth labels of each + box, each has shape (num_gt,). + centers2d_list (list[Tensor]): Projected 3D centers onto 2D image, + each has shape (num_gt, 2). + depths_list (list[Tensor]): Depth of projected 3D centers onto 2D + image, each has shape (num_gt, 1). + attr_labels_list (list[Tensor]): Attribute labels of each box, + each has shape (num_gt,). + + Returns: + tuple: + concat_lvl_labels (list[Tensor]): Labels of each level. \ + concat_lvl_bbox_targets (list[Tensor]): BBox targets of each \ + level. + """ + assert len(points) == len(self.regress_ranges) + num_levels = len(points) + # expand regress ranges to align with points + expanded_regress_ranges = [ + points[i].new_tensor(self.regress_ranges[i])[None].expand_as( + points[i]) for i in range(num_levels) + ] + # concat all levels points and regress ranges + concat_regress_ranges = torch.cat(expanded_regress_ranges, dim=0) + concat_points = torch.cat(points, dim=0) + + # the number of points per img, per lvl + num_points = [center.size(0) for center in points] + + if attr_labels_list is None: + attr_labels_list = [ + gt_labels.new_full(gt_labels.shape, self.attr_background_label) + for gt_labels in gt_labels_list + ] + + # get labels and bbox_targets of each image + _, bbox_targets_list, labels_3d_list, bbox_targets_3d_list, \ + centerness_targets_list, attr_targets_list = multi_apply( + self._get_target_single, + gt_bboxes_list, + gt_labels_list, + gt_bboxes_3d_list, + gt_labels_3d_list, + centers2d_list, + depths_list, + attr_labels_list, + points=concat_points, + regress_ranges=concat_regress_ranges, + num_points_per_lvl=num_points) + + # split to per img, per level + bbox_targets_list = [ + bbox_targets.split(num_points, 0) + for bbox_targets in bbox_targets_list + ] + labels_3d_list = [ + labels_3d.split(num_points, 0) for labels_3d in labels_3d_list + ] + bbox_targets_3d_list = [ + bbox_targets_3d.split(num_points, 0) + for bbox_targets_3d in bbox_targets_3d_list + ] + centerness_targets_list = [ + centerness_targets.split(num_points, 0) + for centerness_targets in centerness_targets_list + ] + attr_targets_list = [ + attr_targets.split(num_points, 0) + for attr_targets in attr_targets_list + ] + + # concat per level image + concat_lvl_labels_3d = [] + concat_lvl_bbox_targets_3d = [] + concat_lvl_centerness_targets = [] + concat_lvl_attr_targets = [] + for i in range(num_levels): + concat_lvl_labels_3d.append( + torch.cat([labels[i] for labels in labels_3d_list])) + concat_lvl_centerness_targets.append( + torch.cat([ + centerness_targets[i] + for centerness_targets in centerness_targets_list + ])) + bbox_targets_3d = torch.cat([ + bbox_targets_3d[i] for bbox_targets_3d in bbox_targets_3d_list + ]) + if self.pred_bbox2d: + bbox_targets = torch.cat( + [bbox_targets[i] for bbox_targets in bbox_targets_list]) + bbox_targets_3d = torch.cat([bbox_targets_3d, bbox_targets], + dim=1) + concat_lvl_attr_targets.append( + torch.cat( + [attr_targets[i] for attr_targets in attr_targets_list])) + if self.norm_on_bbox: + bbox_targets_3d[:, :2] = \ + bbox_targets_3d[:, :2] / self.strides[i] + if self.pred_bbox2d: + bbox_targets_3d[:, -4:] = \ + bbox_targets_3d[:, -4:] / self.strides[i] + concat_lvl_bbox_targets_3d.append(bbox_targets_3d) + return concat_lvl_labels_3d, concat_lvl_bbox_targets_3d, \ + concat_lvl_centerness_targets, concat_lvl_attr_targets From 89be05c5d5b1bbf0253508f34ae7d30c539e32a9 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 22 Sep 2021 21:33:52 +0800 Subject: [PATCH 36/51] Refactor init methods, fix legacy variable names --- .../dense_heads/anchor_free_mono3d_head.py | 35 +++++++++++++------ mmdet3d/models/dense_heads/pgd_head.py | 30 +++++----------- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py b/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py index a8131fc21e..e7d5c059b1 100644 --- a/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py +++ b/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py @@ -1,7 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. import torch from abc import abstractmethod -from mmcv.cnn import ConvModule, bias_init_with_prob, normal_init +from mmcv.cnn import ConvModule from mmcv.runner import force_fp32 from torch import nn as nn @@ -181,8 +181,29 @@ def __init__( type='Normal', layer='Conv2d', std=0.01, - override=dict( - type='Normal', name='conv_cls', std=0.01, bias_prob=0.01)) + override=[ + dict( + type='Normal', + name='conv_cls', + std=0.01, + bias_prob=0.01) + ]) + if self.use_direction_classifier: + self.init_cfg['override'].append( + dict( + type='Normal', + name='conv_dir_cls', + std=0.01, + bias_prob=0.01)) + if self.pred_attrs: + self.init_cfg['override'].append( + dict( + type='Normal', + name='conv_attr', + std=0.01, + bias_prob=0.01)) + else: + self.init_cfg = init_cfg def _init_layers(self): """Initialize layers of the head.""" @@ -287,14 +308,6 @@ def _init_predictor(self): conv_strides=(1, ) * len(self.attr_branch)) self.conv_attr = nn.Conv2d(self.attr_branch[-1], self.num_attrs, 1) - def init_weights(self): - super().init_weights() - bias_cls = bias_init_with_prob(0.01) - if self.use_direction_classifier: - normal_init(self.conv_dir_cls, std=0.01, bias=bias_cls) - if self.pred_attrs: - normal_init(self.conv_attr, std=0.01, bias=bias_cls) - def forward(self, feats): """Forward features from the upstream network. diff --git a/mmdet3d/models/dense_heads/pgd_head.py b/mmdet3d/models/dense_heads/pgd_head.py index ef796a375a..d60f46fd84 100644 --- a/mmdet3d/models/dense_heads/pgd_head.py +++ b/mmdet3d/models/dense_heads/pgd_head.py @@ -1,6 +1,6 @@ import numpy as np import torch -from mmcv.cnn import Scale, bias_init_with_prob, normal_init +from mmcv.cnn import Scale from mmcv.runner import force_fp32 from torch import nn as nn from torch.nn import functional as F @@ -116,6 +116,13 @@ def __init__(self, self.loss_consistency = build_loss(loss_consistency) if self.pred_keypoints: self.kpts_start = 9 if self.pred_velo else 7 + if self.use_depth_classifier: + self.init_cfg['override'].append( + dict( + type='Normal', + name='conv_depth_cls', + std=0.01, + bias_prob=0.01)) def _init_layers(self): """Initialize layers of the head.""" @@ -159,27 +166,6 @@ def _init_predictor(self): self.conv_weights.append( nn.Conv2d(self.feat_channels, 1, 1)) - def init_weights(self): - """Initialize weights of the head.""" - super().init_weights() - - bias_cls = bias_init_with_prob(0.01) - if self.use_depth_classifier: - for m in self.conv_depth_cls_prev: - if isinstance(m.conv, nn.Conv2d): - normal_init(m.conv, std=0.01) - normal_init(self.conv_depth_cls, std=0.01, bias=bias_cls) - - if self.weight_dim != -1: - for conv_weight_prev in self.conv_weight_prevs: - if conv_weight_prev is None: - continue - for m in conv_weight_prev: - if isinstance(m.conv, nn.Conv2d): - normal_init(m.conv, std=0.01) - for conv_weight in self.conv_weights: - normal_init(conv_weight, std=0.01) - def forward(self, feats): """Forward features from the upstream network. From 5be3d11de0eed860348a4a74c865bb0df7c6d742 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 22 Sep 2021 22:33:36 +0800 Subject: [PATCH 37/51] [Feature] Support Uncertain L1 Loss (#950) * Add uncertain l1 loss and its unit tests * Remove mmcv.jit and refine docstrings --- mmdet3d/models/losses/__init__.py | 3 +- .../models/losses/uncertain_smooth_l1_loss.py | 175 ++++++++++++++++++ tests/test_metrics/test_losses.py | 37 ++++ 3 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 mmdet3d/models/losses/uncertain_smooth_l1_loss.py diff --git a/mmdet3d/models/losses/__init__.py b/mmdet3d/models/losses/__init__.py index 7d4703aef9..0efbcb780d 100644 --- a/mmdet3d/models/losses/__init__.py +++ b/mmdet3d/models/losses/__init__.py @@ -3,9 +3,10 @@ from .axis_aligned_iou_loss import AxisAlignedIoULoss, axis_aligned_iou_loss from .chamfer_distance import ChamferDistance, chamfer_distance from .paconv_regularization_loss import PAConvRegularizationLoss +from .uncertain_smooth_l1_loss import UncertainL1Loss, UncertainSmoothL1Loss __all__ = [ 'FocalLoss', 'SmoothL1Loss', 'binary_cross_entropy', 'ChamferDistance', 'chamfer_distance', 'axis_aligned_iou_loss', 'AxisAlignedIoULoss', - 'PAConvRegularizationLoss' + 'PAConvRegularizationLoss', 'UncertainL1Loss', 'UncertainSmoothL1Loss' ] diff --git a/mmdet3d/models/losses/uncertain_smooth_l1_loss.py b/mmdet3d/models/losses/uncertain_smooth_l1_loss.py new file mode 100644 index 0000000000..60a5417223 --- /dev/null +++ b/mmdet3d/models/losses/uncertain_smooth_l1_loss.py @@ -0,0 +1,175 @@ +import torch +from torch import nn as nn + +from mmdet.models.builder import LOSSES +from mmdet.models.losses.utils import weighted_loss + + +@weighted_loss +def uncertain_smooth_l1_loss(pred, target, sigma, alpha=1.0, beta=1.0): + """Smooth L1 loss with uncertainty. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + sigma (torch.Tensor): The sigma for uncertainty. + alpha (float, optional): The coefficient of log(sigma). + Defaults to 1.0. + beta (float, optional): The threshold in the piecewise function. + Defaults to 1.0. + + Returns: + torch.Tensor: Calculated loss + """ + assert beta > 0 + assert target.numel() > 0 + assert pred.size() == target.size() == sigma.size(), 'The size of pred ' \ + f'{pred.size()}, target {target.size()}, and sigma {sigma.size()} ' \ + 'are inconsistent.' + diff = torch.abs(pred - target) + loss = torch.where(diff < beta, 0.5 * diff * diff / beta, + diff - 0.5 * beta) + loss = torch.exp(-sigma) * loss + alpha * sigma + + return loss + + +@weighted_loss +def uncertain_l1_loss(pred, target, sigma, alpha=1.0): + """L1 loss with uncertainty. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + sigma (torch.Tensor): The sigma for uncertainty. + alpha (float, optional): The coefficient of log(sigma). + Defaults to 1.0. + + Returns: + torch.Tensor: Calculated loss + """ + assert target.numel() > 0 + assert pred.size() == target.size() == sigma.size(), 'The size of pred ' \ + f'{pred.size()}, target {target.size()}, and sigma {sigma.size()} ' \ + 'are inconsistent.' + loss = torch.abs(pred - target) + loss = torch.exp(-sigma) * loss + alpha * sigma + return loss + + +@LOSSES.register_module() +class UncertainSmoothL1Loss(nn.Module): + r"""Smooth L1 loss with uncertainty. + + Please refer to `PGD `_ and + `Multi-Task Learning Using Uncertainty to Weigh Losses for Scene Geometry + and Semantics `_ for more details. + + Args: + alpha (float, optional): The coefficient of log(sigma). + Defaults to 1.0. + beta (float, optional): The threshold in the piecewise function. + Defaults to 1.0. + reduction (str, optional): The method to reduce the loss. + Options are 'none', 'mean' and 'sum'. Defaults to 'mean'. + loss_weight (float, optional): The weight of loss. Defaults to 1.0 + """ + + def __init__(self, alpha=1.0, beta=1.0, reduction='mean', loss_weight=1.0): + super(UncertainSmoothL1Loss, self).__init__() + assert reduction in ['none', 'sum', 'mean'] + self.alpha = alpha + self.beta = beta + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + sigma, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + sigma (torch.Tensor): The sigma for uncertainty. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss_bbox = self.loss_weight * uncertain_smooth_l1_loss( + pred, + target, + weight, + sigma=sigma, + alpha=self.alpha, + beta=self.beta, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss_bbox + + +@LOSSES.register_module() +class UncertainL1Loss(nn.Module): + """L1 loss with uncertainty. + + Args: + alpha (float, optional): The coefficient of log(sigma). + Defaults to 1.0. + reduction (str, optional): The method to reduce the loss. + Options are 'none', 'mean' and 'sum'. Defaults to 'mean'. + loss_weight (float, optional): The weight of loss. Defaults to 1.0. + """ + + def __init__(self, alpha=1.0, reduction='mean', loss_weight=1.0): + super(UncertainL1Loss, self).__init__() + assert reduction in ['none', 'sum', 'mean'] + self.alpha = alpha + self.reduction = reduction + self.loss_weight = loss_weight + + def forward(self, + pred, + target, + sigma, + weight=None, + avg_factor=None, + reduction_override=None): + """Forward function. + + Args: + pred (torch.Tensor): The prediction. + target (torch.Tensor): The learning target of the prediction. + sigma (torch.Tensor): The sigma for uncertainty. + weight (torch.Tensor, optional): The weight of loss for each + prediction. Defaults to None. + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + reduction_override (str, optional): The reduction method used to + override the original reduction method of the loss. + Defaults to None. + """ + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + loss_bbox = self.loss_weight * uncertain_l1_loss( + pred, + target, + weight, + sigma=sigma, + alpha=self.alpha, + reduction=reduction, + avg_factor=avg_factor) + return loss_bbox diff --git a/tests/test_metrics/test_losses.py b/tests/test_metrics/test_losses.py index 08cdb6275c..d8eca2f961 100644 --- a/tests/test_metrics/test_losses.py +++ b/tests/test_metrics/test_losses.py @@ -3,6 +3,8 @@ import torch from torch import nn as nn +from mmdet.models import build_loss + def test_chamfer_disrance(): from mmdet3d.models.losses import ChamferDistance, chamfer_distance @@ -109,3 +111,38 @@ def __init__(self): model.modules(), reduction_override='none') assert none_corr_loss.shape[0] == 3 assert torch.allclose(none_corr_loss.mean(), mean_corr_loss) + + +def test_uncertain_smooth_l1_loss(): + from mmdet3d.models.losses import UncertainL1Loss, UncertainSmoothL1Loss + + # reduction shoule be in ['none', 'mean', 'sum'] + with pytest.raises(AssertionError): + uncertain_l1_loss = UncertainL1Loss(reduction='l2') + with pytest.raises(AssertionError): + uncertain_smooth_l1_loss = UncertainSmoothL1Loss(reduction='l2') + + pred = torch.tensor([1.5783, 0.5972, 1.4821, 0.9488]) + target = torch.tensor([1.0813, -0.3466, -1.1404, -0.9665]) + sigma = torch.tensor([-1.0053, 0.4710, -1.7784, -0.8603]) + + # test uncertain l1 loss + uncertain_l1_loss_cfg = dict( + type='UncertainL1Loss', alpha=1.0, reduction='mean', loss_weight=1.0) + uncertain_l1_loss = build_loss(uncertain_l1_loss_cfg) + mean_l1_loss = uncertain_l1_loss(pred, target, sigma) + expected_l1_loss = torch.tensor(4.7069) + assert torch.allclose(mean_l1_loss, expected_l1_loss, atol=1e-4) + + # test uncertain smooth l1 loss + uncertain_smooth_l1_loss_cfg = dict( + type='UncertainSmoothL1Loss', + alpha=1.0, + beta=0.5, + reduction='mean', + loss_weight=1.0) + uncertain_smooth_l1_loss = build_loss(uncertain_smooth_l1_loss_cfg) + mean_smooth_l1_loss = uncertain_smooth_l1_loss(pred, target, sigma) + expected_smooth_l1_loss = torch.tensor(3.9795) + assert torch.allclose( + mean_smooth_l1_loss, expected_smooth_l1_loss, atol=1e-4) From 4a804bf39a3600210f653e903c0b93f0ff58f096 Mon Sep 17 00:00:00 2001 From: ChaimZhu Date: Wed, 22 Sep 2021 22:34:16 +0800 Subject: [PATCH 38/51] [Fix] Fix visualization in KITTI dataset (#956) * fix bug to support kitti vis * fix --- mmdet3d/core/visualizer/image_vis.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mmdet3d/core/visualizer/image_vis.py b/mmdet3d/core/visualizer/image_vis.py index 4144bb7e5f..f070d87cf8 100644 --- a/mmdet3d/core/visualizer/image_vis.py +++ b/mmdet3d/core/visualizer/image_vis.py @@ -192,7 +192,10 @@ def draw_camera_bbox3d_on_img(bboxes3d, points_3d = corners_3d.reshape(-1, 3) if not isinstance(cam2img, torch.Tensor): cam2img = torch.from_numpy(np.array(cam2img)) - cam2img = cam2img.reshape(3, 3).float().cpu() + + assert (cam2img.shape == torch.Size([3, 3]) + or cam2img.shape == torch.Size([4, 4])) + cam2img = cam2img.float().cpu() # project to 2d to get image coords (uv) uv_origin = points_cam2img(points_3d, cam2img) From 038c39d0370f59fbed0f9f94486815867440c133 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Thu, 23 Sep 2021 21:13:17 +0800 Subject: [PATCH 39/51] Refine variable names and docstrings --- .../models/dense_heads/fcos_mono3d_head.py | 7 -- mmdet3d/models/dense_heads/pgd_head.py | 105 +++++++++++++----- 2 files changed, 79 insertions(+), 33 deletions(-) diff --git a/mmdet3d/models/dense_heads/fcos_mono3d_head.py b/mmdet3d/models/dense_heads/fcos_mono3d_head.py index 1dba45d200..bc87359b02 100644 --- a/mmdet3d/models/dense_heads/fcos_mono3d_head.py +++ b/mmdet3d/models/dense_heads/fcos_mono3d_head.py @@ -101,13 +101,6 @@ def __init__(self, self.loss_centerness = build_loss(loss_centerness) bbox_coder['code_size'] = self.bbox_code_size self.bbox_coder = build_bbox_coder(bbox_coder) - if init_cfg is None: - self.init_cfg = dict( - type='Normal', - layer='Conv2d', - std=0.01, - override=dict( - type='Normal', name='conv_cls', std=0.01, bias_prob=0.01)) def _init_layers(self): """Initialize layers of the head.""" diff --git a/mmdet3d/models/dense_heads/pgd_head.py b/mmdet3d/models/dense_heads/pgd_head.py index d60f46fd84..f510d67bfd 100644 --- a/mmdet3d/models/dense_heads/pgd_head.py +++ b/mmdet3d/models/dense_heads/pgd_head.py @@ -146,7 +146,8 @@ def _init_predictor(self): conv_strides=(1, ) * len(self.depth_branch)) self.conv_depth_cls = nn.Conv2d(self.depth_branch[-1], self.num_depth_cls, 1) - self.bld_alpha = nn.Parameter(torch.tensor(10e-5)) + # Data-agnostic single param lambda for local depth fusion + self.fuse_lambda = nn.Parameter(torch.tensor(10e-5)) if self.weight_dim != -1: self.conv_weight_prevs = nn.ModuleList() @@ -184,6 +185,9 @@ def forward(self, feats): dir_cls_preds (list[Tensor]): Box scores for direction class predictions on each scale level, each is a 4D-tensor, the channel number is num_points * 2. (bin = 2). + weight (list[Tensor]): Location-aware weight maps on each + scale level, each is a 4D-tensor, the channel number is + num_points * 1. depth_cls_preds (list[Tensor]): Box scores for depth class predictions on each scale level, each is a 4D-tensor, the channel number is num_points * self.num_depth_cls. @@ -208,8 +212,9 @@ def forward_single(self, x, scale, stride): is True. Returns: - tuple: scores for each class, bbox and direction class \ - predictions, centerness predictions of input feature maps. + tuple: scores for each class, bbox and direction class + predictions, depth class predictions, location-aware weights, + attribute and centerness predictions of input feature maps. """ cls_score, bbox_pred, dir_cls_pred, attr_pred, centerness, cls_feat, \ reg_feat = super().forward_single(x, scale, stride) @@ -254,7 +259,41 @@ def get_proj_bbox2d(self, pos_weights=None, pos_cls_scores=None, with_kpts=False): - """Add prob depth and geo depth related operations.""" + """Decode box predictions and get projected 2D attributes. + + Args: + bbox_preds (list[Tensor]): Box predictions for each scale + level, each is a 4D-tensor, the channel number is + num_points * bbox_code_size. + pos_dir_cls_preds (Tensor): Box scores for direction class + predictions of positive boxes on all the scale levels in shape + (num_pos_points, 2). + labels_3d (list[Tensor]): 3D box category labels for each scale + level, each is a 4D-tensor. + bbox_targets_3d (list[Tensor]): 3D box targets for each scale + level, each is a 4D-tensor, the channel number is + num_points * bbox_code_size. + pos_points (Tensor): Foreground points. + pos_inds (Tensor): Index of foreground points from flattened + tensors. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + pos_depth_cls_preds (Tensor, optional): Probabilistic depth map of + positive boxes on all the scale levels in shape + (num_pos_points, self.num_depth_cls). Defaults to None. + pos_weights (Tensor, optional): Location-aware weights of positive + boxes in shape (num_pos_points, self.weight_dim). Defaults to + None. + pos_cls_scores (Tensor, optional): Classification scores of + positive boxes in shape (num_pos_points, self.num_classes). + Defaults to None. + with_kpts (bool, optional): Whether to output keypoints targets. + Defaults to False. + + Returns: + tuple[Tensor]: Exterior 2D boxes from projected 3D boxes, + predicted 2D boxes and keypoint targets (if necessary). + """ views = [np.array(img_meta['cam2img']) for img_meta in img_metas] num_imgs = len(img_metas) img_idx = [] @@ -309,7 +348,7 @@ def get_proj_bbox2d(self, pos_prob_depth_preds = self.bbox_coder.decode_prob_depth( pos_depth_cls_preds, self.depth_range, self.depth_unit, self.division, self.num_depth_cls) - sig_alpha = torch.sigmoid(self.bld_alpha) + sig_alpha = torch.sigmoid(self.fuse_lambda) pos_strided_bbox_preds[:, 2] = \ sig_alpha * pos_strided_bbox_preds.clone()[:, 2] + \ (1 - sig_alpha) * pos_prob_depth_preds @@ -338,9 +377,11 @@ def get_proj_bbox2d(self, pos_strided_bbox_preds[mask, :3], views[idx]) pos_bbox_targets_3d[mask, :3] = centers3d_targets - # depth fixed + # depth fixed when computing re-project 3D bboxes pos_strided_bbox_preds[mask, 2] = \ pos_bbox_targets_3d.clone()[mask, 2] + + # decode yaws if self.use_direction_classifier: pos_dir_cls_scores = torch.max( pos_dir_cls_preds[mask], dim=-1)[1] @@ -401,11 +442,15 @@ def get_pos_predictions(self, bbox_preds, dir_cls_preds, depth_cls_preds, num_points * num_attrs. centernesses (list[Tensor]): Centerness for each scale level, each is a 4D-tensor, the channel number is num_points * 1. + pos_inds (Tensor): Index of foreground points from flattened + tensors. img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. Returns: - dict[str, Tensor]: A dictionary of loss components. + tuple[Tensor]: Box predictions, direction classes, probabilistic + depth maps, location-aware weight maps, attributes and + centerness predictions. """ flatten_bbox_preds = [ bbox_pred.permute(0, 2, 3, 1).reshape(-1, sum(self.group_reg_dims)) @@ -492,6 +537,9 @@ def loss(self, depth_cls_preds (list[Tensor]): Box scores for direction class predictions on each scale level, each is a 4D-tensor, the channel number is num_points * self.num_depth_cls. + weights (list[Tensor]): Location-aware weights for each scale + level, each is a 4D-tensor, the channel number is + num_points * self.weight_dim. attr_preds (list[Tensor]): Attribute scores for each scale level, each is a 4D-tensor, the channel number is num_points * num_attrs. @@ -510,8 +558,8 @@ def loss(self, attr_labels (list[Tensor]): Attributes indices of each box. img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. - gt_bboxes_ignore (None | list[Tensor]): specify which bounding - boxes can be ignored when computing the loss. + gt_bboxes_ignore (list[Tensor]): specify which bounding boxes can + be ignored when computing the loss. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. @@ -627,7 +675,7 @@ def loss(self, pos_prob_depth_preds = self.bbox_coder.decode_prob_depth( pos_depth_cls_preds, self.depth_range, self.depth_unit, self.division, self.num_depth_cls) - sig_alpha = torch.sigmoid(self.bld_alpha) + sig_alpha = torch.sigmoid(self.fuse_lambda) if self.weight_dim != -1: loss_depth_bld = self.loss_depth( sig_alpha * pos_bbox_preds[:, 2] + @@ -702,7 +750,7 @@ def loss(self, if self.use_direction_classifier: loss_dir = pos_dir_cls_preds.sum() if self.use_depth_classifier: - sig_alpha = torch.sigmoid(self.bld_alpha) + sig_alpha = torch.sigmoid(self.fuse_lambda) if self.weight_dim != -1: loss_depth_bld *= torch.exp(-pos_weights[:, 0].sum()) else: @@ -768,23 +816,24 @@ def get_bboxes(self, depth_cls_preds (list[Tensor]): Box scores for direction class predictions on each scale level, each is a 4D-tensor, the channel number is num_points * self.num_depth_cls. + weights (list[Tensor]): Location-aware weights for each scale + level, each is a 4D-tensor, the channel number is + num_points * self.weight_dim. attr_preds (list[Tensor]): Attribute scores for each scale level Has shape (N, num_points * num_attrs, H, W) centernesses (list[Tensor]): Centerness for each scale level with shape (N, num_points * 1, H, W) img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. - cfg (mmcv.Config): Test / postprocessing configuration, - if None, test_cfg would be used - rescale (bool): If True, return boxes in original image space + cfg (mmcv.Config, optional): Test / postprocessing configuration, + if None, test_cfg would be used. Defaults to None. + rescale (bool, optional): If True, return boxes in original image + space. Defaults to None. Returns: - list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. \ - The first item is an (n, 5) tensor, where the first 4 columns \ - are bounding box positions (tl_x, tl_y, br_x, br_y) and the \ - 5-th column is a score between 0 and 1. The second item is a \ - (n,) tensor where each item is the predicted class label of \ - the corresponding box. + list[tuple[Tensor]]: Each item in result_list is a tuple, which + consists of predicted 3D boxes, scores, labels, attributes and + 2D boxes (if necessary). """ assert len(cls_scores) == len(bbox_preds) == len(dir_cls_preds) == \ len(depth_cls_preds) == len(weights) == len(centernesses) == \ @@ -876,11 +925,13 @@ def _get_bboxes_single(self, bbox_preds (list[Tensor]): Box energies / deltas for a single scale level with shape (num_points * bbox_code_size, H, W). dir_cls_preds (list[Tensor]): Box scores for direction class - predictions on a single scale level with shape \ + predictions on a single scale level with shape (num_points * 2, H, W) - depth_cls_preds (list[Tensor]): Box scores for direction class - predictions on a single scale level with shape \ + depth_cls_preds (list[Tensor]): Box scores for probabilistic depth + predictions on a single scale level with shape (num_points * self.num_depth_cls, H, W) + weights (list[Tensor]): Location-aware weight maps on a single + scale level with shape (num_points * self.weight_dim, H, W). attr_preds (list[Tensor]): Attribute scores for each scale level Has shape (N, num_points * num_attrs, H, W) centernesses (list[Tensor]): Centerness for a single scale level @@ -890,10 +941,12 @@ def _get_bboxes_single(self, input_meta (dict): Metadata of input image. cfg (mmcv.Config): Test / postprocessing configuration, if None, test_cfg would be used. - rescale (bool): If True, return boxes in original image space. + rescale (bool, optional): If True, return boxes in original image + space. Defaults to False. Returns: - tuples[Tensor]: Predicted 3D boxes, scores, labels and attributes. + tuples[Tensor]: Predicted 3D boxes, scores, labels, attributes and + 2D boxes (if necessary). """ view = np.array(input_meta['cam2img']) scale_factor = input_meta['scale_factor'] @@ -972,7 +1025,7 @@ def _get_bboxes_single(self, prob_depth_pred = self.bbox_coder.decode_prob_depth( depth_cls_pred, self.depth_range, self.depth_unit, self.division, self.num_depth_cls) - sig_alpha = torch.sigmoid(self.bld_alpha) + sig_alpha = torch.sigmoid(self.fuse_lambda) bbox_pred3d[:, 2] = sig_alpha * bbox_pred3d[:, 2] + \ (1 - sig_alpha) * prob_depth_pred pred_center2d = bbox_pred3d[:, :3].clone() From 0061d8955575c26b38f94314a60a0d8787ab9326 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Fri, 24 Sep 2021 14:26:09 +0800 Subject: [PATCH 40/51] Add unit tests and fix some minor bugs --- configs/_base_/datasets/kitti-mono3d.py | 89 ++++++++++++ configs/_base_/models/pgd.py | 55 ++++++++ ...1_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py | 127 ++++++++++++++++++ mmdet3d/models/dense_heads/pgd_head.py | 4 +- tests/test_models/test_heads/test_heads.py | 72 ++++++++++ 5 files changed, 345 insertions(+), 2 deletions(-) create mode 100644 configs/_base_/datasets/kitti-mono3d.py create mode 100644 configs/_base_/models/pgd.py create mode 100644 configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py diff --git a/configs/_base_/datasets/kitti-mono3d.py b/configs/_base_/datasets/kitti-mono3d.py new file mode 100644 index 0000000000..884c1109b5 --- /dev/null +++ b/configs/_base_/datasets/kitti-mono3d.py @@ -0,0 +1,89 @@ +dataset_type = 'KittiMonoDataset' +data_root = 'data/kitti/' +class_names = ['Pedestrian', 'Cyclist', 'Car'] +# Input modality for nuScenes dataset, this is consistent with the submission +# format which requires the information in input_modality. +input_modality = dict( + use_lidar=False, + use_camera=True, + use_radar=False, + use_map=False, + use_external=False) +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFileMono3D'), + dict( + type='LoadAnnotations3D', + with_bbox=True, + with_label=True, + with_attr_label=False, + with_bbox_3d=True, + with_label_3d=True, + with_bbox_depth=True), + dict(type='Resize', img_scale=(1242, 375), keep_ratio=True), + dict(type='RandomFlip3D', flip_ratio_bev_horizontal=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle3D', class_names=class_names), + dict( + type='Collect3D', + keys=[ + 'img', 'gt_bboxes', 'gt_labels', 'gt_bboxes_3d', 'gt_labels_3d', + 'centers2d', 'depths' + ]), +] +test_pipeline = [ + dict(type='LoadImageFromFileMono3D'), + dict( + type='MultiScaleFlipAug', + img_scale=(1242, 375), + flip=False, + transforms=[ + dict(type='RandomFlip3D'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict( + type='DefaultFormatBundle3D', + class_names=class_names, + with_label=False), + dict(type='Collect3D', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + data_root=data_root, + ann_file=data_root + 'kitti_infos_train_mono3d.coco.json', + info_file=data_root + 'kitti_infos_train.pkl', + img_prefix=data_root, + classes=class_names, + pipeline=train_pipeline, + modality=input_modality, + test_mode=False, + box_type_3d='Camera'), + val=dict( + type=dataset_type, + data_root=data_root, + ann_file=data_root + 'kitti_infos_val_mono3d.coco.json', + info_file=data_root + 'kitti_infos_val.pkl', + img_prefix=data_root, + classes=class_names, + pipeline=test_pipeline, + modality=input_modality, + test_mode=True, + box_type_3d='Camera'), + test=dict( + type=dataset_type, + data_root=data_root, + ann_file=data_root + 'kitti_infos_val_mono3d.coco.json', + info_file=data_root + 'kitti_infos_val.pkl', + img_prefix=data_root, + classes=class_names, + pipeline=test_pipeline, + modality=input_modality, + test_mode=True, + box_type_3d='Camera')) +evaluation = dict(interval=2) diff --git a/configs/_base_/models/pgd.py b/configs/_base_/models/pgd.py new file mode 100644 index 0000000000..e63fc1fceb --- /dev/null +++ b/configs/_base_/models/pgd.py @@ -0,0 +1,55 @@ +_base_ = './fcos3d.py' +# model settings +model = dict( + bbox_head=dict( + _delete_=True, + type='PGDHead', + num_classes=10, + in_channels=256, + stacked_convs=2, + feat_channels=256, + use_direction_classifier=True, + diff_rad_by_sin=True, + pred_attrs=True, + pred_velo=True, + pred_bbox2d=True, + pred_keypoints=False, + dir_offset=0.7854, # pi/4 + strides=[8, 16, 32, 64, 128], + group_reg_dims=(2, 1, 3, 1, 2), # offset, depth, size, rot, velo + cls_branch=(256, ), + reg_branch=( + (256, ), # offset + (256, ), # depth + (256, ), # size + (256, ), # rot + () # velo + ), + dir_branch=(256, ), + attr_branch=(256, ), + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0), + loss_dir=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_attr=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_centerness=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + norm_on_bbox=True, + centerness_on_reg=True, + center_sampling=True, + conv_bias=True, + dcn_on_last_conv=True, + use_depth_classifier=True, + depth_branch=(256, ), + depth_range=(0, 50), + depth_unit=10, + division='uniform', + depth_bins=6, + bbox_coder=dict(type='PGDBBoxCoder', code_size=9)), + test_cfg=dict(nms_pre=1000, nms_thr=0.8, score_thr=0.01, max_per_img=200)) diff --git a/configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py b/configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py new file mode 100644 index 0000000000..d77666af0f --- /dev/null +++ b/configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py @@ -0,0 +1,127 @@ +_base_ = [ + '../_base_/datasets/kitti-mono3d.py', '../_base_/models/pgd.py', + '../_base_/schedules/mmdet_schedule_1x.py', '../_base_/default_runtime.py' +] +# model settings +model = dict( + backbone=dict(frozen_stages=0), + neck=dict(start_level=0, num_outs=4), + bbox_head=dict( + num_classes=3, + bbox_code_size=7, + pred_attrs=False, + pred_velo=False, + pred_bbox2d=True, + use_onlyreg_proj=True, + strides=(4, 8, 16, 32), + regress_ranges=((-1, 64), (64, 128), (128, 256), (256, 1e8)), + group_reg_dims=(2, 1, 3, 1, 16, + 4), # offset, depth, size, rot, kpts, bbox2d + reg_branch=( + (256, ), # offset + (256, ), # depth + (256, ), # size + (256, ), # rot + (256, ), # kpts + (256, ) # bbox2d + ), + centerness_branch=(256, ), + loss_cls=dict( + type='FocalLoss', + use_sigmoid=True, + gamma=2.0, + alpha=0.25, + loss_weight=1.0), + loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0), + loss_dir=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_centerness=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + use_depth_classifier=True, + depth_branch=(256, ), + depth_range=(0, 70), + depth_unit=10, + division='uniform', + depth_bins=8, + pred_keypoints=True, + weight_dim=1, + loss_depth=dict( + type='UncertainSmoothL1Loss', + alpha=0.25, + beta=3.0, + loss_weight=1.0), + bbox_coder=dict( + type='PGDBBoxCoder', + base_depths=((28.01, 16.32), ), + base_dims=((0.8, 1.73, 0.6), (1.76, 1.73, 0.6), (3.9, 1.56, 1.6)), + code_size=7)), + train_cfg=dict(code_weight=[ + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, + 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 1.0, 1.0, 1.0, 1.0 + ]), + test_cfg=dict(nms_pre=100, nms_thr=0.05, score_thr=0.001, max_per_img=20)) + +class_names = ['Pedestrian', 'Cyclist', 'Car'] +img_norm_cfg = dict( + mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False) +train_pipeline = [ + dict(type='LoadImageFromFileMono3D'), + dict( + type='LoadAnnotations3D', + with_bbox=True, + with_label=True, + with_attr_label=False, + with_bbox_3d=True, + with_label_3d=True, + with_bbox_depth=True), + dict(type='Resize', img_scale=(1242, 375), keep_ratio=True), + dict(type='RandomFlip3D', flip_ratio_bev_horizontal=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle3D', class_names=class_names), + dict( + type='Collect3D', + keys=[ + 'img', 'gt_bboxes', 'gt_labels', 'gt_bboxes_3d', 'gt_labels_3d', + 'centers2d', 'depths' + ]), +] +test_pipeline = [ + dict(type='LoadImageFromFileMono3D'), + dict( + type='MultiScaleFlipAug', + scale_factor=1.0, + flip=False, + transforms=[ + dict(type='RandomFlip3D'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict( + type='DefaultFormatBundle3D', + class_names=class_names, + with_label=False), + dict(type='Collect3D', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=3, + workers_per_gpu=3, + train=dict(pipeline=train_pipeline), + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) +# optimizer +optimizer = dict( + lr=0.001, paramwise_cfg=dict(bias_lr_mult=2., bias_decay_mult=0.)) +optimizer_config = dict( + _delete_=True, grad_clip=dict(max_norm=35, norm_type=2)) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=1.0 / 3, + step=[32, 44]) +total_epochs = 48 +runner = dict(type='EpochBasedRunner', max_epochs=48) +evaluation = dict(interval=2) +checkpoint_config = dict(interval=2) diff --git a/mmdet3d/models/dense_heads/pgd_head.py b/mmdet3d/models/dense_heads/pgd_head.py index f510d67bfd..5184418e68 100644 --- a/mmdet3d/models/dense_heads/pgd_head.py +++ b/mmdet3d/models/dense_heads/pgd_head.py @@ -86,7 +86,6 @@ def __init__(self, self.use_depth_classifier = use_depth_classifier self.use_onlyreg_proj = use_onlyreg_proj self.depth_branch = depth_branch - self.pred_bbox2d = pred_bbox2d self.pred_keypoints = pred_keypoints self.weight_dim = weight_dim self.weight_branch = weight_branch @@ -109,7 +108,8 @@ def __init__(self, 'Uniform Division.') else: self.num_depth_cls = depth_bins - super().__init__(bbox_coder=bbox_coder, **kwargs) + super().__init__( + pred_bbox2d=pred_bbox2d, bbox_coder=bbox_coder, **kwargs) self.loss_depth = build_loss(loss_depth) if self.pred_bbox2d: self.loss_bbox2d = build_loss(loss_bbox2d) diff --git a/tests/test_models/test_heads/test_heads.py b/tests/test_models/test_heads/test_heads.py index 63cd1f8346..b7e41b818d 100644 --- a/tests/test_models/test_heads/test_heads.py +++ b/tests/test_models/test_heads/test_heads.py @@ -1222,3 +1222,75 @@ def test_groupfree3d_head(): assert results[0][0].tensor.shape[1] == 7 assert results[0][1].shape[0] >= 0 assert results[0][2].shape[0] >= 0 + + +def test_pgd_head(): + if not torch.cuda.is_available(): + pytest.skip('test requires GPU and torch+cuda') + _setup_seed(0) + pgd_head_cfg = _get_head_cfg( + 'pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py') + self = build_head(pgd_head_cfg).cuda() + + feats = [ + torch.rand([2, 256, 96, 312], dtype=torch.float32).cuda(), + torch.rand([2, 256, 48, 156], dtype=torch.float32).cuda(), + torch.rand([2, 256, 24, 78], dtype=torch.float32).cuda(), + torch.rand([2, 256, 12, 39], dtype=torch.float32).cuda(), + ] + + # test forward + ret_dict = self(feats) + assert len(ret_dict) == 7 + assert len(ret_dict[0]) == 4 + assert ret_dict[0][0].shape == torch.Size([2, 3, 96, 312]) + + # test loss + gt_bboxes = [ + torch.rand([3, 4], dtype=torch.float32).cuda(), + torch.rand([3, 4], dtype=torch.float32).cuda() + ] + gt_bboxes_3d = CameraInstance3DBoxes( + torch.rand([3, 7], device='cuda'), box_dim=7) + gt_labels = [torch.randint(0, 3, [3], device='cuda') for i in range(2)] + gt_labels_3d = gt_labels + centers2d = [ + torch.rand([3, 2], dtype=torch.float32).cuda(), + torch.rand([3, 2], dtype=torch.float32).cuda() + ] + depths = [ + torch.rand([3], dtype=torch.float32).cuda(), + torch.rand([3], dtype=torch.float32).cuda() + ] + attr_labels = None + img_metas = [ + dict( + img_shape=[384, 1248], + cam2img=[[721.5377, 0.0, 609.5593, 44.85728], + [0.0, 721.5377, 172.854, 0.2163791], + [0.0, 0.0, 1.0, 0.002745884], [0.0, 0.0, 0.0, 1.0]], + scale_factor=np.array([1., 1., 1., 1.], dtype=np.float32), + box_type_3d=CameraInstance3DBoxes) for i in range(2) + ] + losses = self.loss(*ret_dict, gt_bboxes, gt_labels, gt_bboxes_3d, + gt_labels_3d, centers2d, depths, attr_labels, img_metas) + assert losses['loss_cls'] >= 0 + assert losses['loss_offset'] >= 0 + assert losses['loss_depth'] >= 0 + assert losses['loss_size'] >= 0 + assert losses['loss_rotsin'] >= 0 + assert losses['loss_centerness'] >= 0 + assert losses['loss_kpts'] >= 0 + assert losses['loss_bbox2d'] >= 0 + assert losses['loss_consistency'] >= 0 + assert losses['loss_dir'] >= 0 + + # test get_boxes + results = self.get_bboxes(*ret_dict, img_metas) + assert len(results) == 2 + assert len(results[0]) == 5 + assert results[0][0].tensor.shape == torch.Size([20, 7]) + assert results[0][1].shape == torch.Size([20]) + assert results[0][2].shape == torch.Size([20]) + assert results[0][3] is None + assert results[0][4].shape == torch.Size([20, 5]) From 0e1f4ede7dbd0f76f462db3c6176cc1fe43b3e33 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Fri, 24 Sep 2021 14:37:36 +0800 Subject: [PATCH 41/51] Refine assertion messages --- mmdet3d/models/dense_heads/pgd_head.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mmdet3d/models/dense_heads/pgd_head.py b/mmdet3d/models/dense_heads/pgd_head.py index 5184418e68..1e990d308b 100644 --- a/mmdet3d/models/dense_heads/pgd_head.py +++ b/mmdet3d/models/dense_heads/pgd_head.py @@ -566,7 +566,11 @@ def loss(self, """ assert len(cls_scores) == len(bbox_preds) == len(dir_cls_preds) == \ len(depth_cls_preds) == len(weights) == len(centernesses) == \ - len(attr_preds) + len(attr_preds), 'The length of cls_scores, bbox_preds, ' \ + 'dir_cls_preds, depth_cls_preds, weights, centernesses, and' \ + f'attr_preds: {len(cls_scores)}, {len(bbox_preds)}, ' \ + f'{len(dir_cls_preds)}, {len(depth_cls_preds)}, {len(weights)}' \ + f'{len(centernesses)}, {len(attr_preds)} are inconsistent.' featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] all_level_points = self.get_points(featmap_sizes, bbox_preds[0].dtype, bbox_preds[0].device) @@ -837,7 +841,11 @@ def get_bboxes(self, """ assert len(cls_scores) == len(bbox_preds) == len(dir_cls_preds) == \ len(depth_cls_preds) == len(weights) == len(centernesses) == \ - len(attr_preds) + len(attr_preds), 'The length of cls_scores, bbox_preds, ' \ + 'dir_cls_preds, depth_cls_preds, weights, centernesses, and' \ + f'attr_preds: {len(cls_scores)}, {len(bbox_preds)}, ' \ + f'{len(dir_cls_preds)}, {len(depth_cls_preds)}, {len(weights)}' \ + f'{len(centernesses)}, {len(attr_preds)} are inconsistent.' num_levels = len(cls_scores) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] From a62e993d5246ab13d36f11ed1c41f2d563ef3170 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Fri, 24 Sep 2021 20:07:45 +0800 Subject: [PATCH 42/51] Fix typo in the docs_zh-CN --- docs_zh-CN/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs_zh-CN/index.rst b/docs_zh-CN/index.rst index d90fd7d6bd..b2ae80dd8d 100644 --- a/docs_zh-CN/index.rst +++ b/docs_zh-CN/index.rst @@ -83,7 +83,7 @@ Welcome to MMDetection3D's documentation! :maxdepth: 1 :caption: 接口文档（英文） - switch_language.md + api.rst .. toctree:: :maxdepth: 1 From efef7e9375d2874450b14a30a7ace53f5be61033 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Mon, 27 Sep 2021 19:27:53 +0800 Subject: [PATCH 43/51] Use Pretrain init and remove unused init_cfg in FCOS3D --- configs/_base_/models/fcos3d.py | 6 ++++-- mmdet3d/models/dense_heads/fcos_mono3d_head.py | 7 ------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/configs/_base_/models/fcos3d.py b/configs/_base_/models/fcos3d.py index a46ed9cd60..be83001d8f 100644 --- a/configs/_base_/models/fcos3d.py +++ b/configs/_base_/models/fcos3d.py @@ -1,6 +1,5 @@ model = dict( type='FCOSMono3D', - pretrained='open-mmlab://detectron2/resnet101_caffe', backbone=dict( type='ResNet', depth=101, @@ -9,7 +8,10 @@ frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True, - style='caffe'), + style='caffe', + init_cfg=dict( + type='Pretrained', + checkpoint='open-mmlab://detectron2/resnet101_caffe')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], diff --git a/mmdet3d/models/dense_heads/fcos_mono3d_head.py b/mmdet3d/models/dense_heads/fcos_mono3d_head.py index 1dba45d200..bc87359b02 100644 --- a/mmdet3d/models/dense_heads/fcos_mono3d_head.py +++ b/mmdet3d/models/dense_heads/fcos_mono3d_head.py @@ -101,13 +101,6 @@ def __init__(self, self.loss_centerness = build_loss(loss_centerness) bbox_coder['code_size'] = self.bbox_code_size self.bbox_coder = build_bbox_coder(bbox_coder) - if init_cfg is None: - self.init_cfg = dict( - type='Normal', - layer='Conv2d', - std=0.01, - override=dict( - type='Normal', name='conv_cls', std=0.01, bias_prob=0.01)) def _init_layers(self): """Initialize layers of the head.""" From 016fc2964a71347100fbf3d5aa6c5078dbecd8d4 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 29 Sep 2021 11:03:16 +0800 Subject: [PATCH 44/51] Fix the comments for the input_modality in the dataset config --- configs/_base_/datasets/kitti-mono3d.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/configs/_base_/datasets/kitti-mono3d.py b/configs/_base_/datasets/kitti-mono3d.py index 884c1109b5..73a0914fe1 100644 --- a/configs/_base_/datasets/kitti-mono3d.py +++ b/configs/_base_/datasets/kitti-mono3d.py @@ -1,14 +1,7 @@ dataset_type = 'KittiMonoDataset' data_root = 'data/kitti/' class_names = ['Pedestrian', 'Cyclist', 'Car'] -# Input modality for nuScenes dataset, this is consistent with the submission -# format which requires the information in input_modality. -input_modality = dict( - use_lidar=False, - use_camera=True, - use_radar=False, - use_map=False, - use_external=False) +input_modality = dict(use_lidar=False, use_camera=True) img_norm_cfg = dict( mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) train_pipeline = [ From b47ad7e8db5d9e5652e131a922a4035446c985d3 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Tue, 26 Oct 2021 20:53:37 +0800 Subject: [PATCH 45/51] Fix minor bugs in pgd_bbox_coder and incorrect setting for uncertain loss, use original init --- ...1_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py | 6 +- mmdet3d/core/bbox/coders/pgd_bbox_coder.py | 6 +- .../dense_heads/anchor_free_mono3d_head.py | 64 ++++++++++--------- .../models/dense_heads/fcos_mono3d_head.py | 15 ++++- mmdet3d/models/dense_heads/pgd_head.py | 43 ++++++++++--- 5 files changed, 88 insertions(+), 46 deletions(-) diff --git a/configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py b/configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py index d77666af0f..f863b642fa 100644 --- a/configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py +++ b/configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py @@ -46,9 +46,7 @@ pred_keypoints=True, weight_dim=1, loss_depth=dict( - type='UncertainSmoothL1Loss', - alpha=0.25, - beta=3.0, + type='UncertainSmoothL1Loss', alpha=1.0, beta=3.0, loss_weight=1.0), bbox_coder=dict( type='PGDBBoxCoder', @@ -124,4 +122,4 @@ total_epochs = 48 runner = dict(type='EpochBasedRunner', max_epochs=48) evaluation = dict(interval=2) -checkpoint_config = dict(interval=2) +checkpoint_config = dict(interval=8) diff --git a/mmdet3d/core/bbox/coders/pgd_bbox_coder.py b/mmdet3d/core/bbox/coders/pgd_bbox_coder.py index 5e42128189..70db450d25 100644 --- a/mmdet3d/core/bbox/coders/pgd_bbox_coder.py +++ b/mmdet3d/core/bbox/coders/pgd_bbox_coder.py @@ -1,4 +1,5 @@ import numpy as np +import torch from torch.nn import functional as F from mmdet.core.bbox.builder import BBOX_CODERS @@ -45,8 +46,9 @@ def decode_2d(self, scale_kpts = scale[3] # 2 dimension of offsets x 8 corners of a 3D bbox bbox[:, self.bbox_code_size:self.bbox_code_size + 16] = \ - scale_kpts(clone_bbox[ - :, self.bbox_code_size:self.bbox_code_size + 16]).float() + torch.tanh(scale_kpts(clone_bbox[ + :, self.bbox_code_size:self.bbox_code_size + 16]).float()) + if pred_bbox2d: scale_bbox2d = scale[-1] # The last four dimensions are offsets to four sides of a 2D bbox diff --git a/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py b/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py index e7d5c059b1..baefa0ee89 100644 --- a/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py +++ b/mmdet3d/models/dense_heads/anchor_free_mono3d_head.py @@ -1,7 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. import torch from abc import abstractmethod -from mmcv.cnn import ConvModule +from mmcv.cnn import ConvModule, bias_init_with_prob, normal_init from mmcv.runner import force_fp32 from torch import nn as nn @@ -176,34 +176,6 @@ def __init__( self.attr_branch = attr_branch self._init_layers() - if init_cfg is None: - self.init_cfg = dict( - type='Normal', - layer='Conv2d', - std=0.01, - override=[ - dict( - type='Normal', - name='conv_cls', - std=0.01, - bias_prob=0.01) - ]) - if self.use_direction_classifier: - self.init_cfg['override'].append( - dict( - type='Normal', - name='conv_dir_cls', - std=0.01, - bias_prob=0.01)) - if self.pred_attrs: - self.init_cfg['override'].append( - dict( - type='Normal', - name='conv_attr', - std=0.01, - bias_prob=0.01)) - else: - self.init_cfg = init_cfg def _init_layers(self): """Initialize layers of the head.""" @@ -308,6 +280,40 @@ def _init_predictor(self): conv_strides=(1, ) * len(self.attr_branch)) self.conv_attr = nn.Conv2d(self.attr_branch[-1], self.num_attrs, 1) + def init_weights(self): + """Initialize weights of the head. + + We currently still use the customized defined init_weights because the + default init of DCN triggered by the init_cfg will init + conv_offset.weight, which mistakenly affects the training stability. + """ + for modules in [self.cls_convs, self.reg_convs, self.conv_cls_prev]: + for m in modules: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + for conv_reg_prev in self.conv_reg_prevs: + if conv_reg_prev is None: + continue + for m in conv_reg_prev: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + if self.use_direction_classifier: + for m in self.conv_dir_cls_prev: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + if self.pred_attrs: + for m in self.conv_attr_prev: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + bias_cls = bias_init_with_prob(0.01) + normal_init(self.conv_cls, std=0.01, bias=bias_cls) + for conv_reg in self.conv_regs: + normal_init(conv_reg, std=0.01) + if self.use_direction_classifier: + normal_init(self.conv_dir_cls, std=0.01, bias=bias_cls) + if self.pred_attrs: + normal_init(self.conv_attr, std=0.01, bias=bias_cls) + def forward(self, feats): """Forward features from the upstream network. diff --git a/mmdet3d/models/dense_heads/fcos_mono3d_head.py b/mmdet3d/models/dense_heads/fcos_mono3d_head.py index bc87359b02..3c6e5c5cc9 100644 --- a/mmdet3d/models/dense_heads/fcos_mono3d_head.py +++ b/mmdet3d/models/dense_heads/fcos_mono3d_head.py @@ -2,7 +2,7 @@ import numpy as np import torch from logging import warning -from mmcv.cnn import Scale +from mmcv.cnn import Scale, normal_init from mmcv.runner import force_fp32 from torch import nn as nn @@ -115,6 +115,19 @@ def _init_layers(self): for _ in self.strides ]) + def init_weights(self): + """Initialize weights of the head. + + We currently still use the customized defined init_weights because the + default init of DCN triggered by the init_cfg will init + conv_offset.weight, which mistakenly affects the training stability. + """ + super().init_weights() + for m in self.conv_centerness_prev: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + normal_init(self.conv_centerness, std=0.01) + def forward(self, feats): """Forward features from the upstream network. diff --git a/mmdet3d/models/dense_heads/pgd_head.py b/mmdet3d/models/dense_heads/pgd_head.py index 1e990d308b..d48eca6db5 100644 --- a/mmdet3d/models/dense_heads/pgd_head.py +++ b/mmdet3d/models/dense_heads/pgd_head.py @@ -1,6 +1,6 @@ import numpy as np import torch -from mmcv.cnn import Scale +from mmcv.cnn import Scale, bias_init_with_prob, normal_init from mmcv.runner import force_fp32 from torch import nn as nn from torch.nn import functional as F @@ -116,13 +116,6 @@ def __init__(self, self.loss_consistency = build_loss(loss_consistency) if self.pred_keypoints: self.kpts_start = 9 if self.pred_velo else 7 - if self.use_depth_classifier: - self.init_cfg['override'].append( - dict( - type='Normal', - name='conv_depth_cls', - std=0.01, - bias_prob=0.01)) def _init_layers(self): """Initialize layers of the head.""" @@ -167,6 +160,32 @@ def _init_predictor(self): self.conv_weights.append( nn.Conv2d(self.feat_channels, 1, 1)) + def init_weights(self): + """Initialize weights of the head. + + We currently still use the customized defined init_weights because the + default init of DCN triggered by the init_cfg will init + conv_offset.weight, which mistakenly affects the training stability. + """ + super().init_weights() + + bias_cls = bias_init_with_prob(0.01) + if self.use_depth_classifier: + for m in self.conv_depth_cls_prev: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + normal_init(self.conv_depth_cls, std=0.01, bias=bias_cls) + + if self.weight_dim != -1: + for conv_weight_prev in self.conv_weight_prevs: + if conv_weight_prev is None: + continue + for m in conv_weight_prev: + if isinstance(m.conv, nn.Conv2d): + normal_init(m.conv, std=0.01) + for conv_weight in self.conv_weights: + normal_init(conv_weight, std=0.01) + def forward(self, feats): """Forward features from the upstream network. @@ -362,7 +381,10 @@ def get_proj_bbox2d(self, mask = (pos_img_idx == idx) if pos_strided_bbox_preds[mask].shape[0] == 0: continue - cam2img = pos_strided_bbox_preds.new_zeros((4, 4)) + cam2img = torch.eye( + 4, + dtype=pos_strided_bbox_preds.dtype, + device=pos_strided_bbox_preds.device) view_shape = views[idx].shape cam2img[:view_shape[0], :view_shape[1]] = \ pos_strided_bbox_preds.new_tensor(views[idx]) @@ -1058,7 +1080,8 @@ def _get_bboxes_single(self, mlvl_bboxes2d = torch.cat(mlvl_bboxes2d) # change local yaw to global yaw for 3D nms - cam2img = mlvl_centers2d.new_zeros((4, 4)) + cam2img = torch.eye( + 4, dtype=mlvl_centers2d.dtype, device=mlvl_centers2d.device) cam2img[:view.shape[0], :view.shape[1]] = \ mlvl_centers2d.new_tensor(view) mlvl_bboxes = self.bbox_coder.decode_yaw(mlvl_bboxes, mlvl_centers2d, From 3baa41af845f098a4c234124039b787d756bfd7a Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Tue, 26 Oct 2021 21:11:50 +0800 Subject: [PATCH 46/51] Add explanations for code_weights --- configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py b/configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py index f863b642fa..832b34e64d 100644 --- a/configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py +++ b/configs/pgd/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d.py @@ -53,6 +53,8 @@ base_depths=((28.01, 16.32), ), base_dims=((0.8, 1.73, 0.6), (1.76, 1.73, 0.6), (3.9, 1.56, 1.6)), code_size=7)), + # set weight 1.0 for base 7 dims (offset, depth, size, rot) + # 0.2 for 16-dim keypoint offsets and 1.0 for 4-dim 2D distance targets train_cfg=dict(code_weight=[ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 1.0, 1.0, 1.0, 1.0 From 9658a2bd8f029373401233240c2ca79c0514311e Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Tue, 26 Oct 2021 21:56:09 +0800 Subject: [PATCH 47/51] Adjust the unit test for pgd bbox coder --- tests/test_utils/test_bbox_coders.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/test_utils/test_bbox_coders.py b/tests/test_utils/test_bbox_coders.py index 3b42d64d86..d240456fa8 100644 --- a/tests/test_utils/test_bbox_coders.py +++ b/tests/test_utils/test_bbox_coders.py @@ -502,17 +502,17 @@ def test_pgd_bbox_coder(): max_regress_range, training, pred_keypoints, pred_bbox2d) expected_decode_bbox_w2d = torch.tensor( - [[[[0.0206]], [[1.4788]], [[1.3904]], [[1.6013]], [[1.1548]], - [[1.0809]], [[0.9399]], [[13.3856]], [[2.0224]], [[4.8480]], - [[3.0368]], [[1.1424]], [[6.6304]], [[6.9456]], [[10.3072]], - [[4.7216]], [[4.6240]], [[7.1776]], [[4.5568]], [[1.7136]], - [[15.2480]], [[15.1360]], [[6.1152]], [[1.8640]], [[0.5222]], - [[1.1160]], [[0.0794]]], + [[[[0.0206]], [[1.4788]], + [[1.3904]], [[1.6013]], [[1.1548]], [[1.0809]], [[0.9399]], + [[10.9441]], [[2.0117]], [[4.7049]], [[3.0009]], [[1.1405]], + [[6.2752]], [[6.5399]], [[9.0840]], [[4.5892]], [[4.4994]], + [[6.7320]], [[4.4375]], [[1.7071]], [[11.8582]], [[11.8075]], + [[5.8339]], [[1.8640]], [[0.5222]], [[1.1160]], [[0.0794]]], [[[1.7224]], [[0.3360]], [[1.6765]], [[2.3401]], [[1.0384]], - [[1.4355]], [[0.9550]], [[8.3504]], [[2.2432]], [[10.9488]], - [[3.3936]], [[15.1488]], [[9.9808]], [[12.6688]], [[2.6336]], - [[0.8000]], [[10.0640]], [[6.3296]], [[4.6416]], [[7.3792]], - [[11.7328]], [[1.9104]], [[11.1984]], [[0.7960]], [[0.6524]], + [[1.4355]], [[0.9550]], [[7.6666]], [[2.2286]], [[9.5089]], + [[3.3436]], [[11.8133]], [[8.8603]], [[10.5508]], [[2.6101]], + [[0.7993]], [[8.9178]], [[6.0188]], [[4.5156]], [[6.8970]], + [[10.0013]], [[1.9014]], [[9.6689]], [[0.7960]], [[0.6524]], [[1.4370]], [[0.8948]]]]) assert torch.allclose(expected_decode_bbox_w2d, decode_bbox_w2d, atol=1e-3) @@ -596,3 +596,7 @@ def test_smoke_bbox_coder(): locations = torch.tensor([[15., 2., 1.], [15., 2., -1.]]) orientations = bbox_coder._decode_orientation(ori_vector, locations) assert orientations.shape == torch.Size([2, 1]) + + +if __name__ == '__main__': + test_pgd_bbox_coder() From fa78b417153dbee627d9e1099f533a694deccd10 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Tue, 26 Oct 2021 22:03:25 +0800 Subject: [PATCH 48/51] Remove unused codes --- tests/test_utils/test_bbox_coders.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_utils/test_bbox_coders.py b/tests/test_utils/test_bbox_coders.py index d240456fa8..56a1e6af52 100644 --- a/tests/test_utils/test_bbox_coders.py +++ b/tests/test_utils/test_bbox_coders.py @@ -596,7 +596,3 @@ def test_smoke_bbox_coder(): locations = torch.tensor([[15., 2., 1.], [15., 2., -1.]]) orientations = bbox_coder._decode_orientation(ori_vector, locations) assert orientations.shape == torch.Size([2, 1]) - - -if __name__ == '__main__': - test_pgd_bbox_coder() From 4e086bc1cbbc72798a1e05fa01a0e607477cbd54 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Wed, 27 Oct 2021 11:08:35 +0800 Subject: [PATCH 49/51] Add mono3d metric into the gather_models and fix bugs --- .dev_scripts/gather_models.py | 37 ++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/.dev_scripts/gather_models.py b/.dev_scripts/gather_models.py index 38270a73b8..c4f355db00 100644 --- a/.dev_scripts/gather_models.py +++ b/.dev_scripts/gather_models.py @@ -3,6 +3,16 @@ Usage: python gather_models.py ${root_path} ${out_dir} + +Example: +python gather_models.py \ +work_dirs/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d \ +work_dirs/pgd_r101_caffe_fpn_gn-head_3x4_4x_kitti-mono3d + +Note that before running the above command, rename the directory with the +config name if you did not use the default directory name, create +a corresponding directory 'pgd' under the above path and put the used config +into it. """ import argparse @@ -36,16 +46,18 @@ RESULTS_LUT = { 'coco': ['bbox_mAP', 'segm_mAP'], 'nus': ['pts_bbox_NuScenes/NDS', 'NDS'], - 'kitti-3d-3class': [ - 'KITTI/Overall_3D_moderate', - 'Overall_3D_moderate', - ], + 'kitti-3d-3class': ['KITTI/Overall_3D_moderate', 'Overall_3D_moderate'], 'kitti-3d-car': ['KITTI/Car_3D_moderate_strict', 'Car_3D_moderate_strict'], 'lyft': ['score'], 'scannet_seg': ['miou'], 's3dis_seg': ['miou'], 'scannet': ['mAP_0.50'], - 'sunrgbd': ['mAP_0.50'] + 'sunrgbd': ['mAP_0.50'], + 'kitti-mono3d': [ + 'img_bbox/KITTI/Car_3D_AP40_moderate_strict', + 'Car_3D_AP40_moderate_strict' + ], + 'nus-mono3d': ['img_bbox_NuScenes/NDS', 'NDS'] } @@ -145,15 +157,13 @@ def main(): # and parse the best performance model_infos = [] for used_config in used_configs: - exp_dir = osp.join(models_root, used_config) - # get logs - log_json_path = glob.glob(osp.join(exp_dir, '*.log.json'))[0] - log_txt_path = glob.glob(osp.join(exp_dir, '*.log'))[0] + log_json_path = glob.glob(osp.join(models_root, '*.log.json'))[0] + log_txt_path = glob.glob(osp.join(models_root, '*.log'))[0] model_performance = get_best_results(log_json_path) final_epoch = model_performance['epoch'] final_model = 'epoch_{}.pth'.format(final_epoch) - model_path = osp.join(exp_dir, final_model) + model_path = osp.join(models_root, final_model) # skip if the model is still training if not osp.exists(model_path): @@ -182,7 +192,7 @@ def main(): model_name = model['config'].split('/')[-1].rstrip( '.py') + '_' + model['model_time'] publish_model_path = osp.join(model_publish_dir, model_name) - trained_model_path = osp.join(models_root, model['config'], + trained_model_path = osp.join(models_root, 'epoch_{}.pth'.format(model['epochs'])) # convert model @@ -191,11 +201,10 @@ def main(): # copy log shutil.copy( - osp.join(models_root, model['config'], model['log_json_path']), + osp.join(models_root, model['log_json_path']), osp.join(model_publish_dir, f'{model_name}.log.json')) shutil.copy( - osp.join(models_root, model['config'], - model['log_json_path'].rstrip('.json')), + osp.join(models_root, model['log_json_path'].rstrip('.json')), osp.join(model_publish_dir, f'{model_name}.log')) # copy config to guarantee reproducibility From 3f77afc40bf3fedba23d5cbf148db22b1bcc4a6e Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Thu, 28 Oct 2021 09:37:49 +0800 Subject: [PATCH 50/51] Involve the value assignment of loss_dict into the computing procedure --- .../models/dense_heads/fcos_mono3d_head.py | 6 +- mmdet3d/models/dense_heads/pgd_head.py | 79 ++++++++----------- 2 files changed, 36 insertions(+), 49 deletions(-) diff --git a/mmdet3d/models/dense_heads/fcos_mono3d_head.py b/mmdet3d/models/dense_heads/fcos_mono3d_head.py index 4074b791bd..ef1b53a02a 100644 --- a/mmdet3d/models/dense_heads/fcos_mono3d_head.py +++ b/mmdet3d/models/dense_heads/fcos_mono3d_head.py @@ -118,9 +118,9 @@ def _init_layers(self): def init_weights(self): """Initialize weights of the head. - We currently still use the customized defined init_weights because the - default init of DCN triggered by the init_cfg will init - conv_offset.weight, which mistakenly affects the training stability. + We currently still use the customized init_weights because the default + init of DCN triggered by the init_cfg will init conv_offset.weight, + which mistakenly affects the training stability. """ super().init_weights() for m in self.conv_centerness_prev: diff --git a/mmdet3d/models/dense_heads/pgd_head.py b/mmdet3d/models/dense_heads/pgd_head.py index d48eca6db5..206abc78ba 100644 --- a/mmdet3d/models/dense_heads/pgd_head.py +++ b/mmdet3d/models/dense_heads/pgd_head.py @@ -103,7 +103,7 @@ def __init__(self, (depth_range[1] - depth_range[0]) / depth_unit) + 1 if self.num_depth_cls != depth_bins: print('Warning: The number of bins computed from ' + - 'depth_unit is different from given paramter! ' + + 'depth_unit is different from given parameter! ' + 'Depth_unit will be considered with priority in ' + 'Uniform Division.') else: @@ -220,7 +220,7 @@ def forward(self, feats): self.strides) def forward_single(self, x, scale, stride): - """Forward features of a single scale levle. + """Forward features of a single scale level. Args: x (Tensor): FPN feature maps of the specified stride. @@ -632,6 +632,8 @@ def loss(self, bbox_preds, dir_cls_preds, depth_cls_preds, weights, attr_preds, centernesses, pos_inds, img_metas) + loss_dict = dict() + if num_pos > 0: pos_bbox_targets_3d = flatten_bbox_targets_3d[pos_inds] pos_centerness_targets = flatten_centerness_targets[pos_inds] @@ -677,7 +679,7 @@ def loss(self, weight=bbox_weights[:, 6], avg_factor=equal_weights.sum()) if self.pred_velo: - loss_velo = self.loss_bbox( + loss_dict['loss_velo'] = self.loss_bbox( pos_bbox_preds[:, 7:9], pos_bbox_targets_3d[:, 7:9], weight=bbox_weights[:, 7:9], @@ -690,7 +692,7 @@ def loss(self, # direction classification loss # TODO: add more check for use_direction_classifier if self.use_direction_classifier: - loss_dir = self.loss_dir( + loss_dict['loss_dir'] = self.loss_dir( pos_dir_cls_preds, pos_dir_cls_targets, equal_weights, @@ -703,7 +705,7 @@ def loss(self, self.division, self.num_depth_cls) sig_alpha = torch.sigmoid(self.fuse_lambda) if self.weight_dim != -1: - loss_depth_bld = self.loss_depth( + loss_fuse_depth = self.loss_depth( sig_alpha * pos_bbox_preds[:, 2] + (1 - sig_alpha) * pos_prob_depth_preds, pos_bbox_targets_3d[:, 2], @@ -711,12 +713,13 @@ def loss(self, weight=bbox_weights[:, 2], avg_factor=equal_weights.sum()) else: - loss_depth_bld = self.loss_depth( + loss_fuse_depth = self.loss_depth( sig_alpha * pos_bbox_preds[:, 2] + (1 - sig_alpha) * pos_prob_depth_preds, pos_bbox_targets_3d[:, 2], weight=bbox_weights[:, 2], avg_factor=equal_weights.sum()) + loss_dict['loss_depth'] = loss_fuse_depth proj_bbox2d_inputs += (pos_depth_cls_preds, ) @@ -725,7 +728,7 @@ def loss(self, # normalize the offsets with strides proj_bbox2d_preds, pos_decoded_bbox2d_preds, kpts_targets = \ self.get_proj_bbox2d(*proj_bbox2d_inputs, with_kpts=True) - loss_kpts = self.loss_bbox( + loss_dict['loss_kpts'] = self.loss_bbox( pos_bbox_preds[:, self.kpts_start:self.kpts_start + 16], kpts_targets, weight=bbox_weights[:, @@ -733,7 +736,7 @@ def loss(self, avg_factor=equal_weights.sum()) if self.pred_bbox2d: - loss_bbox2d = self.loss_bbox2d( + loss_dict['loss_bbox2d'] = self.loss_bbox2d( pos_bbox_preds[:, -4:], pos_bbox_targets_3d[:, -4:], weight=bbox_weights[:, -4:], @@ -741,7 +744,7 @@ def loss(self, if not self.pred_keypoints: proj_bbox2d_preds, pos_decoded_bbox2d_preds = \ self.get_proj_bbox2d(*proj_bbox2d_inputs) - loss_consistency = self.loss_consistency( + loss_dict['loss_consistency'] = self.loss_consistency( proj_bbox2d_preds, pos_decoded_bbox2d_preds, weight=bbox_weights[:, -4:], @@ -752,7 +755,7 @@ def loss(self, # attribute classification loss if self.pred_attrs: - loss_attr = self.loss_attr( + loss_dict['loss_attr'] = self.loss_attr( pos_attr_preds, pos_attr_targets, pos_centerness_targets, @@ -765,53 +768,37 @@ def loss(self, loss_size = pos_bbox_preds[:, 3:6].sum() loss_rotsin = pos_bbox_preds[:, 6].sum() if self.pred_velo: - loss_velo = pos_bbox_preds[:, 7:9].sum() + loss_dict['loss_velo'] = pos_bbox_preds[:, 7:9].sum() if self.pred_keypoints: - loss_kpts = pos_bbox_preds[:, self.kpts_start:self.kpts_start + - 16].sum() + loss_dict['loss_kpts'] = pos_bbox_preds[:, + self.kpts_start:self. + kpts_start + 16].sum() if self.pred_bbox2d: - loss_bbox2d = pos_bbox_preds[:, -4:].sum() - loss_consistency = pos_bbox_preds[:, -4:].sum() + loss_dict['loss_bbox2d'] = pos_bbox_preds[:, -4:].sum() + loss_dict['loss_consistency'] = pos_bbox_preds[:, -4:].sum() loss_centerness = pos_centerness.sum() if self.use_direction_classifier: - loss_dir = pos_dir_cls_preds.sum() + loss_dict['loss_dir'] = pos_dir_cls_preds.sum() if self.use_depth_classifier: sig_alpha = torch.sigmoid(self.fuse_lambda) if self.weight_dim != -1: - loss_depth_bld *= torch.exp(-pos_weights[:, 0].sum()) + loss_fuse_depth *= torch.exp(-pos_weights[:, 0].sum()) else: - loss_depth_bld = \ + loss_fuse_depth = \ sig_alpha * pos_bbox_preds[:, 2].sum() + \ (1 - sig_alpha) * pos_depth_cls_preds.sum() + loss_dict['loss_depth'] = loss_fuse_depth if self.pred_attrs: - loss_attr = pos_attr_preds.sum() - - loss_dict = dict( - loss_cls=loss_cls, - loss_offset=loss_offset, - loss_depth=loss_depth, - loss_size=loss_size, - loss_rotsin=loss_rotsin, - loss_centerness=loss_centerness) - - if self.pred_velo: - loss_dict['loss_velo'] = loss_velo - - if self.pred_keypoints: - loss_dict['loss_kpts'] = loss_kpts - - if self.pred_bbox2d: - loss_dict['loss_bbox2d'] = loss_bbox2d - loss_dict['loss_consistency'] = loss_consistency - - if self.use_direction_classifier: - loss_dict['loss_dir'] = loss_dir - - if self.use_depth_classifier: - loss_dict['loss_depth'] = loss_depth_bld - - if self.pred_attrs: - loss_dict['loss_attr'] = loss_attr + loss_dict['loss_attr'] = pos_attr_preds.sum() + + loss_dict.update( + dict( + loss_cls=loss_cls, + loss_offset=loss_offset, + loss_depth=loss_depth, + loss_size=loss_size, + loss_rotsin=loss_rotsin, + loss_centerness=loss_centerness)) return loss_dict From b8a46bcc625eb380c0efd1cb6822c8a0f36dcc82 Mon Sep 17 00:00:00 2001 From: Tai-Wang Date: Thu, 28 Oct 2021 09:47:46 +0800 Subject: [PATCH 51/51] Fix incorrect loss_depth --- mmdet3d/models/dense_heads/pgd_head.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mmdet3d/models/dense_heads/pgd_head.py b/mmdet3d/models/dense_heads/pgd_head.py index 206abc78ba..f4a9e87bae 100644 --- a/mmdet3d/models/dense_heads/pgd_head.py +++ b/mmdet3d/models/dense_heads/pgd_head.py @@ -663,11 +663,6 @@ def loss(self, pos_bbox_targets_3d[:, :2], weight=bbox_weights[:, :2], avg_factor=equal_weights.sum()) - loss_depth = self.loss_bbox( - pos_bbox_preds[:, 2], - pos_bbox_targets_3d[:, 2], - weight=bbox_weights[:, 2], - avg_factor=equal_weights.sum()) loss_size = self.loss_bbox( pos_bbox_preds[:, 3:6], pos_bbox_targets_3d[:, 3:6], @@ -698,6 +693,12 @@ def loss(self, equal_weights, avg_factor=equal_weights.sum()) + # init depth loss with the one computed from direct regression + loss_dict['loss_depth'] = self.loss_bbox( + pos_bbox_preds[:, 2], + pos_bbox_targets_3d[:, 2], + weight=bbox_weights[:, 2], + avg_factor=equal_weights.sum()) # depth classification loss if self.use_depth_classifier: pos_prob_depth_preds = self.bbox_coder.decode_prob_depth( @@ -764,9 +765,9 @@ def loss(self, else: # need absolute due to possible negative delta x/y loss_offset = pos_bbox_preds[:, :2].sum() - loss_depth = pos_bbox_preds[:, 2].sum() loss_size = pos_bbox_preds[:, 3:6].sum() loss_rotsin = pos_bbox_preds[:, 6].sum() + loss_dict['loss_depth'] = pos_bbox_preds[:, 2].sum() if self.pred_velo: loss_dict['loss_velo'] = pos_bbox_preds[:, 7:9].sum() if self.pred_keypoints: @@ -795,7 +796,6 @@ def loss(self, dict( loss_cls=loss_cls, loss_offset=loss_offset, - loss_depth=loss_depth, loss_size=loss_size, loss_rotsin=loss_rotsin, loss_centerness=loss_centerness))