From 5874e4108efc319466dbcfb42d7f20e160b527cc Mon Sep 17 00:00:00 2001 From: hanrui1sensetime <83800577+hanrui1sensetime@users.noreply.github.com> Date: Wed, 26 Oct 2022 19:49:33 +0800 Subject: [PATCH] [Feature] support mmyolo deployment (#79) * support mmyolo deployment * mv deploy place * remove unused configs * add deploy code * fix new register * fix comments * fix dependent codebase register * remove unused initialize * refact deploy config * credit return to triplemu * Add yolov5 head rewrite * refactor deploy * refactor deploy * Add yolov5 head rewrite * fix configs * refact config * fix comment * sync name after mmdeploy 1088 * fix mmyolo * fix yapf * fix deploy config * try to fix flake8 importlib-metadata * add mmyolo models ut * add deploy uts * add deploy uts * fix trt dynamic error * fix multi-batch for dynamic batch value * fix mode * fix lint * sync model.py * add ci for deploy test * fix ci * fix ci * fix ci * extract script to command for fixing CI * fix cmake for CI * sudo ln * move ort position * remove unused sdk compile * cd mmdeploy * simplify build * add missing make * change order * add -v * add setuptools * get locate * get locate * upgrade torch * change torchvision version * fix config * fix ci * fix ci * fix lint Co-authored-by: tripleMu Co-authored-by: RunningLeon --- .circleci/test.yml | 22 +- configs/deploy/base_dynamic.py | 17 ++ configs/deploy/base_static.py | 23 ++ .../deploy/detection_onnxruntime_dynamic.py | 15 + .../deploy/detection_onnxruntime_static.py | 15 + ...n_tensorrt-fp16_dynamic-320x320-640x640.py | 12 + .../detection_tensorrt-fp16_static-640x640.py | 13 + ...n_tensorrt-int8_dynamic-320x320-640x640.py | 14 + .../detection_tensorrt-int8_static-640x640.py | 15 + ...ection_tensorrt_dynamic-320x320-640x640.py | 12 + .../detection_tensorrt_static-640x640.py | 13 + mmyolo/deploy/__init__.py | 7 + mmyolo/deploy/models/__init__.py | 2 + mmyolo/deploy/models/dense_heads/__init__.py | 4 + .../deploy/models/dense_heads/yolov5_head.py | 108 +++++++ mmyolo/deploy/object_detection.py | 74 +++++ tests/test_deploy/conftest.py | 13 + tests/test_deploy/data/model.py | 263 ++++++++++++++++++ tests/test_deploy/test_mmyolo_models.py | 165 +++++++++++ tests/test_deploy/test_object_detection.py | 96 +++++++ 20 files changed, 901 insertions(+), 2 deletions(-) create mode 100644 configs/deploy/base_dynamic.py create mode 100644 configs/deploy/base_static.py create mode 100644 configs/deploy/detection_onnxruntime_dynamic.py create mode 100644 configs/deploy/detection_onnxruntime_static.py create mode 100644 configs/deploy/detection_tensorrt-fp16_dynamic-320x320-640x640.py create mode 100644 configs/deploy/detection_tensorrt-fp16_static-640x640.py create mode 100644 configs/deploy/detection_tensorrt-int8_dynamic-320x320-640x640.py create mode 100644 configs/deploy/detection_tensorrt-int8_static-640x640.py create mode 100644 configs/deploy/detection_tensorrt_dynamic-320x320-640x640.py create mode 100644 configs/deploy/detection_tensorrt_static-640x640.py create mode 100644 mmyolo/deploy/__init__.py create mode 100644 mmyolo/deploy/models/__init__.py create mode 100644 mmyolo/deploy/models/dense_heads/__init__.py create mode 100644 mmyolo/deploy/models/dense_heads/yolov5_head.py create mode 100644 mmyolo/deploy/object_detection.py create mode 100644 tests/test_deploy/conftest.py create mode 100644 tests/test_deploy/data/model.py create mode 100644 tests/test_deploy/test_mmyolo_models.py create mode 100644 tests/test_deploy/test_object_detection.py diff --git a/.circleci/test.yml b/.circleci/test.yml index 533f869627..448f71a21a 100644 --- a/.circleci/test.yml +++ b/.circleci/test.yml @@ -56,6 +56,12 @@ jobs: command: | python -V pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html + - run: + name: Install ONNXRuntime + command: | + pip install onnxruntime==1.8.1 + wget https://github.com/microsoft/onnxruntime/releases/download/v1.8.1/onnxruntime-linux-x64-1.8.1.tgz + tar xvf onnxruntime-linux-x64-1.8.1.tgz - run: name: Install mmyolo dependencies command: | @@ -65,6 +71,17 @@ jobs: pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x pip install -r requirements/albu.txt pip install -r requirements/tests.txt + - run: + name: Install mmdeploy + command: | + pip install setuptools + git clone -b dev-1.x --depth 1 https://github.com/open-mmlab/mmdeploy.git mmdeploy --recurse-submodules + wget https://github.com/Kitware/CMake/releases/download/v3.20.0/cmake-3.20.0-linux-x86_64.tar.gz + tar -xzvf cmake-3.20.0-linux-x86_64.tar.gz + sudo ln -sf $(pwd)/cmake-3.20.0-linux-x86_64/bin/* /usr/bin/ + cd mmdeploy && mkdir build && cd build && cmake .. -DMMDEPLOY_TARGET_BACKENDS=ort -DONNXRUNTIME_DIR=/home/circleci/project/onnxruntime-linux-x64-1.8.1 && make -j8 && make install + export LD_LIBRARY_PATH=/home/circleci/project/onnxruntime-linux-x64-1.8.1/lib:${LD_LIBRARY_PATH} + cd /home/circleci/project/mmdeploy && python -m pip install -v -e . - run: name: Build and install command: | @@ -72,6 +89,7 @@ jobs: - run: name: Run unittests command: | + export LD_LIBRARY_PATH=/home/circleci/project/onnxruntime-linux-x64-1.8.1/lib:${LD_LIBRARY_PATH} coverage run --branch --source mmyolo -m pytest tests/ coverage xml coverage report -m @@ -144,8 +162,8 @@ workflows: - main - build_cpu: name: minimum_version_cpu - torch: 1.7.0 - torchvision: 0.8.1 + torch: 1.8.0 + torchvision: 0.9.0 python: 3.8.0 # The lowest python 3.6.x version available on CircleCI images requires: - lint diff --git a/configs/deploy/base_dynamic.py b/configs/deploy/base_dynamic.py new file mode 100644 index 0000000000..747c21fd2b --- /dev/null +++ b/configs/deploy/base_dynamic.py @@ -0,0 +1,17 @@ +_base_ = ['./base_static.py'] +onnx_config = dict( + dynamic_axes={ + 'input': { + 0: 'batch', + 2: 'height', + 3: 'width' + }, + 'dets': { + 0: 'batch', + 1: 'num_dets' + }, + 'labels': { + 0: 'batch', + 1: 'num_dets' + } + }) diff --git a/configs/deploy/base_static.py b/configs/deploy/base_static.py new file mode 100644 index 0000000000..dee01dd5dd --- /dev/null +++ b/configs/deploy/base_static.py @@ -0,0 +1,23 @@ +onnx_config = dict( + type='onnx', + export_params=True, + keep_initializers_as_inputs=False, + opset_version=11, + save_file='end2end.onnx', + input_names=['input'], + output_names=['dets', 'labels'], + input_shape=None, + optimize=True) +codebase_config = dict( + type='mmyolo', + task='ObjectDetection', + model_type='end2end', + post_processing=dict( + score_threshold=0.05, + confidence_threshold=0.005, + iou_threshold=0.5, + max_output_boxes_per_class=200, + pre_top_k=5000, + keep_top_k=100, + background_label_id=-1), + module=['mmyolo.deploy']) diff --git a/configs/deploy/detection_onnxruntime_dynamic.py b/configs/deploy/detection_onnxruntime_dynamic.py new file mode 100644 index 0000000000..14f4a12115 --- /dev/null +++ b/configs/deploy/detection_onnxruntime_dynamic.py @@ -0,0 +1,15 @@ +_base_ = ['./base_dynamic.py'] +codebase_config = dict( + type='mmyolo', + task='ObjectDetection', + model_type='end2end', + post_processing=dict( + score_threshold=0.05, + confidence_threshold=0.005, + iou_threshold=0.5, + max_output_boxes_per_class=200, + pre_top_k=5000, + keep_top_k=100, + background_label_id=-1), + module=['mmyolo.deploy']) +backend_config = dict(type='onnxruntime') diff --git a/configs/deploy/detection_onnxruntime_static.py b/configs/deploy/detection_onnxruntime_static.py new file mode 100644 index 0000000000..3eac8ca757 --- /dev/null +++ b/configs/deploy/detection_onnxruntime_static.py @@ -0,0 +1,15 @@ +_base_ = ['./base_static.py'] +codebase_config = dict( + type='mmyolo', + task='ObjectDetection', + model_type='end2end', + post_processing=dict( + score_threshold=0.05, + confidence_threshold=0.005, + iou_threshold=0.5, + max_output_boxes_per_class=200, + pre_top_k=5000, + keep_top_k=100, + background_label_id=-1), + module=['mmyolo.deploy']) +backend_config = dict(type='onnxruntime') diff --git a/configs/deploy/detection_tensorrt-fp16_dynamic-320x320-640x640.py b/configs/deploy/detection_tensorrt-fp16_dynamic-320x320-640x640.py new file mode 100644 index 0000000000..92ca431e3f --- /dev/null +++ b/configs/deploy/detection_tensorrt-fp16_dynamic-320x320-640x640.py @@ -0,0 +1,12 @@ +_base_ = ['./base_dynamic.py'] +backend_config = dict( + type='tensorrt', + common_config=dict(fp16_mode=True, max_workspace_size=1 << 30), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 320, 320], + opt_shape=[1, 3, 640, 640], + max_shape=[1, 3, 640, 640]))) + ]) diff --git a/configs/deploy/detection_tensorrt-fp16_static-640x640.py b/configs/deploy/detection_tensorrt-fp16_static-640x640.py new file mode 100644 index 0000000000..06644ae822 --- /dev/null +++ b/configs/deploy/detection_tensorrt-fp16_static-640x640.py @@ -0,0 +1,13 @@ +_base_ = ['./base_static.py'] +onnx_config = dict(input_shape=(640, 640)) +backend_config = dict( + type='tensorrt', + common_config=dict(fp16_mode=True, max_workspace_size=1 << 30), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 640, 640], + opt_shape=[1, 3, 640, 640], + max_shape=[1, 3, 640, 640]))) + ]) diff --git a/configs/deploy/detection_tensorrt-int8_dynamic-320x320-640x640.py b/configs/deploy/detection_tensorrt-int8_dynamic-320x320-640x640.py new file mode 100644 index 0000000000..d01761f1f3 --- /dev/null +++ b/configs/deploy/detection_tensorrt-int8_dynamic-320x320-640x640.py @@ -0,0 +1,14 @@ +_base_ = ['./base_dynamic.py'] +backend_config = dict( + type='tensorrt', + common_config=dict( + fp16_mode=True, max_workspace_size=1 << 30, int8_mode=True), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 320, 320], + opt_shape=[1, 3, 640, 640], + max_shape=[1, 3, 640, 640]))) + ], + calib_config=dict(create_calib=True, calib_file='calib_data.h5')) diff --git a/configs/deploy/detection_tensorrt-int8_static-640x640.py b/configs/deploy/detection_tensorrt-int8_static-640x640.py new file mode 100644 index 0000000000..eeb330ea70 --- /dev/null +++ b/configs/deploy/detection_tensorrt-int8_static-640x640.py @@ -0,0 +1,15 @@ +_base_ = ['./base_static.py'] +onnx_config = dict(input_shape=(640, 640)) +backend_config = dict( + type='tensorrt', + common_config=dict( + fp16_mode=True, max_workspace_size=1 << 30, int8_mode=True), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 640, 640], + opt_shape=[1, 3, 640, 640], + max_shape=[1, 3, 640, 640]))) + ], + calib_config=dict(create_calib=True, calib_file='calib_data.h5')) diff --git a/configs/deploy/detection_tensorrt_dynamic-320x320-640x640.py b/configs/deploy/detection_tensorrt_dynamic-320x320-640x640.py new file mode 100644 index 0000000000..c02e397b24 --- /dev/null +++ b/configs/deploy/detection_tensorrt_dynamic-320x320-640x640.py @@ -0,0 +1,12 @@ +_base_ = ['./base_dynamic.py'] +backend_config = dict( + type='tensorrt', + common_config=dict(fp16_mode=False, max_workspace_size=1 << 30), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 320, 320], + opt_shape=[1, 3, 640, 640], + max_shape=[1, 3, 640, 640]))) + ]) diff --git a/configs/deploy/detection_tensorrt_static-640x640.py b/configs/deploy/detection_tensorrt_static-640x640.py new file mode 100644 index 0000000000..a0830dee59 --- /dev/null +++ b/configs/deploy/detection_tensorrt_static-640x640.py @@ -0,0 +1,13 @@ +_base_ = ['./base_static.py'] +onnx_config = dict(input_shape=(640, 640)) +backend_config = dict( + type='tensorrt', + common_config=dict(fp16_mode=False, max_workspace_size=1 << 30), + model_inputs=[ + dict( + input_shapes=dict( + input=dict( + min_shape=[1, 3, 640, 640], + opt_shape=[1, 3, 640, 640], + max_shape=[1, 3, 640, 640]))) + ]) diff --git a/mmyolo/deploy/__init__.py b/mmyolo/deploy/__init__.py new file mode 100644 index 0000000000..4904a9058b --- /dev/null +++ b/mmyolo/deploy/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from mmdeploy.codebase.base import MMCodebase + +from .models import * # noqa: F401,F403 +from .object_detection import MMYOLO, YOLOObjectDetection + +__all__ = ['MMCodebase', 'MMYOLO', 'YOLOObjectDetection'] diff --git a/mmyolo/deploy/models/__init__.py b/mmyolo/deploy/models/__init__.py new file mode 100644 index 0000000000..4b999a0161 --- /dev/null +++ b/mmyolo/deploy/models/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from . import dense_heads # noqa: F401,F403 diff --git a/mmyolo/deploy/models/dense_heads/__init__.py b/mmyolo/deploy/models/dense_heads/__init__.py new file mode 100644 index 0000000000..cc423af3ec --- /dev/null +++ b/mmyolo/deploy/models/dense_heads/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from . import yolov5_head # noqa: F401,F403 + +__all__ = ['yolov5_head'] diff --git a/mmyolo/deploy/models/dense_heads/yolov5_head.py b/mmyolo/deploy/models/dense_heads/yolov5_head.py new file mode 100644 index 0000000000..f8f2b5bf47 --- /dev/null +++ b/mmyolo/deploy/models/dense_heads/yolov5_head.py @@ -0,0 +1,108 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +from typing import List, Optional, Tuple + +import torch +from mmdeploy.codebase.mmdet import get_post_processing_params +from mmdeploy.codebase.mmdet.models.layers import multiclass_nms +from mmdeploy.core import FUNCTION_REWRITER +from mmengine.config import ConfigDict +from mmengine.structures import InstanceData +from torch import Tensor + + +@FUNCTION_REWRITER.register_rewriter( + func_name='mmyolo.models.dense_heads.yolov5_head.' + 'YOLOv5Head.predict_by_feat') +def yolov5_head__predict_by_feat(ctx, + self, + cls_scores: List[Tensor], + bbox_preds: List[Tensor], + objectnesses: Optional[List[Tensor]], + batch_img_metas: Optional[List[dict]] = None, + cfg: Optional[ConfigDict] = None, + rescale: bool = False, + with_nms: bool = True) -> Tuple[InstanceData]: + """Transform a batch of output features extracted by the head into + bbox results. + Args: + cls_scores (list[Tensor]): Classification scores for all + scale levels, each is a 4D-tensor, has shape + (batch_size, num_priors * num_classes, H, W). + bbox_preds (list[Tensor]): Box energies / deltas for all + scale levels, each is a 4D-tensor, has shape + (batch_size, num_priors * 4, H, W). + objectnesses (list[Tensor], Optional): Score factor for + all scale level, each is a 4D-tensor, has shape + (batch_size, 1, H, W). + batch_img_metas (list[dict], Optional): Batch image meta info. + Defaults to None. + cfg (ConfigDict, optional): Test / postprocessing + configuration, if None, test_cfg would be used. + Defaults to None. + rescale (bool): If True, return boxes in original image space. + Defaults to False. + with_nms (bool): If True, do nms before return boxes. + Defaults to True. + Returns: + tuple[Tensor, Tensor]: The first item is an (N, num_box, 5) tensor, + where 5 represent (tl_x, tl_y, br_x, br_y, score), N is batch + size and the score between 0 and 1. The shape of the second + tensor in the tuple is (N, num_box), and each element + represents the class label of the corresponding box. + """ + assert len(cls_scores) == len(bbox_preds) + cfg = self.test_cfg if cfg is None else cfg + cfg = copy.deepcopy(cfg) + + num_imgs = cls_scores[0].shape[0] + featmap_sizes = [cls_score.shape[2:] for cls_score in cls_scores] + + mlvl_priors = self.prior_generator.grid_priors( + featmap_sizes, dtype=cls_scores[0].dtype, device=cls_scores[0].device) + flatten_priors = torch.cat(mlvl_priors) + + mlvl_strides = [ + flatten_priors.new_full( + (featmap_size[0] * featmap_size[1] * self.num_base_priors, ), + stride) + for featmap_size, stride in zip(featmap_sizes, self.featmap_strides) + ] + flatten_stride = torch.cat(mlvl_strides) + # flatten cls_scores, bbox_preds and objectness + flatten_cls_scores = [ + cls_score.permute(0, 2, 3, 1).reshape(num_imgs, -1, self.num_classes) + for cls_score in cls_scores + ] + flatten_bbox_preds = [ + bbox_pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, 4) + for bbox_pred in bbox_preds + ] + + flatten_objectness = [ + objectness.permute(0, 2, 3, 1).reshape(num_imgs, -1) + for objectness in objectnesses + ] + + cls_scores = torch.cat(flatten_cls_scores, dim=1).sigmoid() + flatten_bbox_preds = torch.cat(flatten_bbox_preds, dim=1) + flatten_objectness = torch.cat(flatten_objectness, dim=1).sigmoid() + bboxes = self.bbox_coder.decode(flatten_priors[None], flatten_bbox_preds, + flatten_stride) + + # directly multiply score factor and feed to nms + scores = cls_scores * (flatten_objectness.unsqueeze(-1)) + + if not with_nms: + return bboxes, scores + deploy_cfg = ctx.cfg + post_params = get_post_processing_params(deploy_cfg) + max_output_boxes_per_class = post_params.max_output_boxes_per_class + iou_threshold = cfg.nms.get('iou_threshold', post_params.iou_threshold) + score_threshold = cfg.get('score_thr', post_params.score_threshold) + pre_top_k = post_params.pre_top_k + keep_top_k = cfg.get('max_per_img', post_params.keep_top_k) + + return multiclass_nms(bboxes, scores, max_output_boxes_per_class, + iou_threshold, score_threshold, pre_top_k, + keep_top_k) diff --git a/mmyolo/deploy/object_detection.py b/mmyolo/deploy/object_detection.py new file mode 100644 index 0000000000..2317ec9154 --- /dev/null +++ b/mmyolo/deploy/object_detection.py @@ -0,0 +1,74 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from typing import Callable + +from mmdeploy.codebase.base import CODEBASE, MMCodebase +from mmdeploy.codebase.mmdet.deploy import ObjectDetection +from mmdeploy.utils import Codebase, Task +from mmengine import Config +from mmengine.registry import Registry + +MMYOLO_TASK = Registry('mmyolo_tasks') + + +@CODEBASE.register_module(Codebase.MMYOLO.value) +class MMYOLO(MMCodebase): + """MMYOLO codebase class.""" + + task_registry = MMYOLO_TASK + + @classmethod + def register_all_modules(cls): + from mmdet.utils.setup_env import \ + register_all_modules as register_all_modules_mmdet + + from mmyolo.utils.setup_env import \ + register_all_modules as register_all_modules_mmyolo + register_all_modules_mmyolo(True) + register_all_modules_mmdet(False) + + +def _get_dataset_metainfo(model_cfg: Config): + """Get metainfo of dataset. + + Args: + model_cfg Config: Input model Config object. + + Returns: + list[str]: A list of string specifying names of different class. + """ + from mmyolo import datasets # noqa + from mmyolo.registry import DATASETS + + module_dict = DATASETS.module_dict + for dataloader_name in [ + 'test_dataloader', 'val_dataloader', 'train_dataloader' + ]: + if dataloader_name not in model_cfg: + continue + dataloader_cfg = model_cfg[dataloader_name] + dataset_cfg = dataloader_cfg.dataset + dataset_cls = module_dict.get(dataset_cfg.type, None) + if dataset_cls is None: + continue + if hasattr(dataset_cls, '_load_metainfo') and isinstance( + dataset_cls._load_metainfo, Callable): + meta = dataset_cls._load_metainfo( + dataset_cfg.get('metainfo', None)) + if meta is not None: + return meta + if hasattr(dataset_cls, 'METAINFO'): + return dataset_cls.METAINFO + + return None + + +@MMYOLO_TASK.register_module(Task.OBJECT_DETECTION.value) +class YOLOObjectDetection(ObjectDetection): + + def get_visualizer(self, name: str, save_dir: str): + from mmdet.visualization import DetLocalVisualizer # noqa: F401,F403 + metainfo = _get_dataset_metainfo(self.model_cfg) + visualizer = super().get_visualizer(name, save_dir) + if metainfo is not None: + visualizer.dataset_meta = metainfo + return visualizer diff --git a/tests/test_deploy/conftest.py b/tests/test_deploy/conftest.py new file mode 100644 index 0000000000..ed1bd3d889 --- /dev/null +++ b/tests/test_deploy/conftest.py @@ -0,0 +1,13 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import pytest + + +@pytest.fixture(autouse=True) +def init_test(): + # init default scope + from mmdet.utils import register_all_modules as register_det + + from mmyolo.utils import register_all_modules as register_yolo + + register_yolo(True) + register_det(False) diff --git a/tests/test_deploy/data/model.py b/tests/test_deploy/data/model.py new file mode 100644 index 0000000000..cf13167eb1 --- /dev/null +++ b/tests/test_deploy/data/model.py @@ -0,0 +1,263 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# model settings +default_scope = 'mmyolo' + +default_hooks = dict( + timer=dict(type='IterTimerHook'), + logger=dict(type='LoggerHook', interval=50), + param_scheduler=dict(type='ParamSchedulerHook'), + checkpoint=dict(type='CheckpointHook', interval=1), + sampler_seed=dict(type='DistSamplerSeedHook'), + visualization=dict(type='mmdet.DetVisualizationHook')) + +env_cfg = dict( + cudnn_benchmark=False, + mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0), + dist_cfg=dict(backend='nccl'), +) + +vis_backends = [dict(type='LocalVisBackend')] +visualizer = dict( + type='mmdet.DetLocalVisualizer', + vis_backends=vis_backends, + name='visualizer') +log_processor = dict(type='LogProcessor', window_size=50, by_epoch=True) + +log_level = 'INFO' +load_from = None +resume = False + +file_client_args = dict(backend='disk') + +# dataset settings +data_root = 'data/coco/' +dataset_type = 'YOLOv5CocoDataset' + +# parameters that often need to be modified +img_scale = (640, 640) # height, width +deepen_factor = 0.33 +widen_factor = 0.5 +max_epochs = 300 +save_epoch_intervals = 10 +train_batch_size_per_gpu = 16 +train_num_workers = 8 +val_batch_size_per_gpu = 1 +val_num_workers = 2 + +# persistent_workers must be False if num_workers is 0. +persistent_workers = True + +# only on Val +batch_shapes_cfg = dict( + type='BatchShapePolicy', + batch_size=val_batch_size_per_gpu, + img_size=img_scale[0], + size_divisor=32, + extra_pad_ratio=0.5) + +anchors = [[(10, 13), (16, 30), (33, 23)], [(30, 61), (62, 45), (59, 119)], + [(116, 90), (156, 198), (373, 326)]] +strides = [8, 16, 32] + +# single-scale training is recommended to +# be turned on, which can speed up training. +env_cfg = dict(cudnn_benchmark=True) + +model = dict( + type='YOLODetector', + data_preprocessor=dict( + type='mmdet.DetDataPreprocessor', + mean=[0., 0., 0.], + std=[255., 255., 255.], + bgr_to_rgb=True), + backbone=dict( + type='YOLOv5CSPDarknet', + deepen_factor=deepen_factor, + widen_factor=widen_factor, + norm_cfg=dict(type='BN', momentum=0.03, eps=0.001), + act_cfg=dict(type='SiLU', inplace=True)), + neck=dict( + type='YOLOv5PAFPN', + deepen_factor=deepen_factor, + widen_factor=widen_factor, + in_channels=[256, 512, 1024], + out_channels=[256, 512, 1024], + num_csp_blocks=3, + norm_cfg=dict(type='BN', momentum=0.03, eps=0.001), + act_cfg=dict(type='SiLU', inplace=True)), + bbox_head=dict( + type='YOLOv5Head', + head_module=dict( + type='YOLOv5HeadModule', + num_classes=80, + in_channels=[256, 512, 1024], + widen_factor=widen_factor, + featmap_strides=strides, + num_base_priors=3), + prior_generator=dict( + type='mmdet.YOLOAnchorGenerator', + base_sizes=anchors, + strides=strides), + loss_cls=dict( + type='mmdet.CrossEntropyLoss', + use_sigmoid=True, + reduction='mean', + loss_weight=0.5), + loss_bbox=dict( + type='IoULoss', + iou_mode='ciou', + bbox_format='xywh', + eps=1e-7, + reduction='mean', + loss_weight=0.05, + return_iou=True), + loss_obj=dict( + type='mmdet.CrossEntropyLoss', + use_sigmoid=True, + reduction='mean', + loss_weight=1.0), + prior_match_thr=4., + obj_level_weights=[4., 1., 0.4]), + test_cfg=dict( + multi_label=True, + nms_pre=30000, + score_thr=0.001, + nms=dict(type='nms', iou_threshold=0.65), + max_per_img=300)) + +albu_train_transforms = [ + dict(type='Blur', p=0.01), + dict(type='MedianBlur', p=0.01), + dict(type='ToGray', p=0.01), + dict(type='CLAHE', p=0.01) +] + +pre_transform = [ + dict(type='LoadImageFromFile', file_client_args=file_client_args), + dict(type='LoadAnnotations', with_bbox=True) +] + +train_pipeline = [ + *pre_transform, + dict( + type='Mosaic', + img_scale=img_scale, + pad_val=114.0, + pre_transform=pre_transform), + dict( + type='YOLOv5RandomAffine', + max_rotate_degree=0.0, + max_shear_degree=0.0, + scaling_ratio_range=(0.5, 1.5), + border=(-img_scale[0] // 2, -img_scale[1] // 2), + border_val=(114, 114, 114)), + dict( + type='mmdet.Albu', + transforms=albu_train_transforms, + bbox_params=dict( + type='BboxParams', + format='pascal_voc', + label_fields=['gt_bboxes_labels', 'gt_ignore_flags']), + keymap={ + 'img': 'image', + 'gt_bboxes': 'bboxes' + }), + dict(type='YOLOv5HSVRandomAug'), + dict(type='mmdet.RandomFlip', prob=0.5), + dict( + type='mmdet.PackDetInputs', + meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip', + 'flip_direction')) +] + +train_dataloader = dict( + batch_size=train_batch_size_per_gpu, + num_workers=train_num_workers, + persistent_workers=persistent_workers, + pin_memory=True, + sampler=dict(type='DefaultSampler', shuffle=True), + dataset=dict( + type=dataset_type, + data_root=data_root, + ann_file='annotations/instances_train2017.json', + data_prefix=dict(img='train2017/'), + filter_cfg=dict(filter_empty_gt=False, min_size=32), + pipeline=train_pipeline)) + +test_pipeline = [ + dict(type='LoadImageFromFile', file_client_args=file_client_args), + dict(type='YOLOv5KeepRatioResize', scale=img_scale), + dict( + type='LetterResize', + scale=img_scale, + allow_scale_up=False, + pad_val=dict(img=114)), + dict(type='LoadAnnotations', with_bbox=True), + dict( + type='mmdet.PackDetInputs', + meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', + 'scale_factor', 'pad_param')) +] + +val_dataloader = dict( + batch_size=val_batch_size_per_gpu, + num_workers=val_num_workers, + persistent_workers=persistent_workers, + pin_memory=True, + drop_last=False, + sampler=dict(type='DefaultSampler', shuffle=False), + dataset=dict( + type=dataset_type, + data_root=data_root, + test_mode=True, + data_prefix=dict(img='val2017/'), + ann_file='annotations/instances_val2017.json', + pipeline=test_pipeline, + batch_shapes_cfg=batch_shapes_cfg)) + +test_dataloader = val_dataloader + +param_scheduler = None +optim_wrapper = dict( + type='OptimWrapper', + optimizer=dict( + type='SGD', + lr=0.01, + momentum=0.937, + weight_decay=0.0005, + nesterov=True, + batch_size_per_gpu=train_batch_size_per_gpu), + constructor='YOLOv5OptimizerConstructor') + +default_hooks = dict( + param_scheduler=dict( + type='YOLOv5ParamSchedulerHook', + scheduler_type='linear', + lr_factor=0.01, + max_epochs=max_epochs), + checkpoint=dict( + type='CheckpointHook', interval=save_epoch_intervals, + max_keep_ckpts=3)) + +custom_hooks = [ + dict( + type='EMAHook', + ema_type='ExpMomentumEMA', + momentum=0.0001, + update_buffers=True, + priority=49) +] + +val_evaluator = dict( + type='mmdet.CocoMetric', + proposal_nums=(100, 1, 10), + ann_file=data_root + 'annotations/instances_val2017.json', + metric='bbox') +test_evaluator = val_evaluator + +train_cfg = dict( + type='EpochBasedTrainLoop', + max_epochs=max_epochs, + val_interval=save_epoch_intervals) +val_cfg = dict(type='ValLoop') +test_cfg = dict(type='TestLoop') diff --git a/tests/test_deploy/test_mmyolo_models.py b/tests/test_deploy/test_mmyolo_models.py new file mode 100644 index 0000000000..a709957f01 --- /dev/null +++ b/tests/test_deploy/test_mmyolo_models.py @@ -0,0 +1,165 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +import random + +import numpy as np +import pytest +import torch +from mmengine import Config + +try: + import importlib + importlib.import_module('mmdeploy') +except ImportError: + pytest.skip('mmdeploy is not installed.', allow_module_level=True) + +from mmdeploy.codebase import import_codebase +from mmdeploy.utils import Backend +from mmdeploy.utils.config_utils import register_codebase +from mmdeploy.utils.test import (WrapModel, check_backend, get_model_outputs, + get_rewrite_outputs) + +try: + codebase = register_codebase('mmyolo') + import_codebase(codebase, ['mmyolo.deploy']) +except ImportError: + pytest.skip('mmyolo is not installed.', allow_module_level=True) + + +def seed_everything(seed=1029): + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) # if you are using multi-GPU. + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.enabled = False + + +def get_yolov5_head_model(): + """YOLOv5 Head Config.""" + test_cfg = Config( + dict( + multi_label=True, + nms_pre=30000, + score_thr=0.001, + nms=dict(type='nms', iou_threshold=0.65), + max_per_img=300)) + + from mmyolo.models.dense_heads import YOLOv5Head + head_module = dict( + type='YOLOv5HeadModule', + num_classes=4, + in_channels=[2, 4, 8], + featmap_strides=[8, 16, 32], + num_base_priors=1) + + model = YOLOv5Head(head_module, test_cfg=test_cfg) + + model.requires_grad_(False) + return model + + +@pytest.mark.parametrize('backend_type', [Backend.ONNXRUNTIME]) +def test_yolov5_head_predict_by_feat(backend_type: Backend): + """Test predict_by_feat rewrite of YOLOXHead.""" + check_backend(backend_type) + yolov5_head = get_yolov5_head_model() + yolov5_head.cpu().eval() + s = 256 + batch_img_metas = [{ + 'scale_factor': (1.0, 1.0), + 'pad_shape': (s, s, 3), + 'img_shape': (s, s, 3), + 'ori_shape': (s, s, 3) + }] + output_names = ['dets', 'labels'] + deploy_cfg = Config( + dict( + backend_config=dict(type=backend_type.value), + onnx_config=dict(output_names=output_names, input_shape=None), + codebase_config=dict( + type='mmdet', + task='ObjectDetection', + post_processing=dict( + score_threshold=0.05, + iou_threshold=0.5, + max_output_boxes_per_class=20, + pre_top_k=-1, + keep_top_k=10, + background_label_id=-1, + ), + module=['mmyolo.deploy']))) + seed_everything(1234) + cls_scores = [ + torch.rand(1, yolov5_head.num_classes * yolov5_head.num_base_priors, + 4 * pow(2, i), 4 * pow(2, i)) for i in range(3, 0, -1) + ] + seed_everything(5678) + bbox_preds = [ + torch.rand(1, 4 * yolov5_head.num_base_priors, 4 * pow(2, i), + 4 * pow(2, i)) for i in range(3, 0, -1) + ] + seed_everything(9101) + objectnesses = [ + torch.rand(1, 1 * yolov5_head.num_base_priors, 4 * pow(2, i), + 4 * pow(2, i)) for i in range(3, 0, -1) + ] + + # to get outputs of pytorch model + model_inputs = { + 'cls_scores': cls_scores, + 'bbox_preds': bbox_preds, + 'objectnesses': objectnesses, + 'batch_img_metas': batch_img_metas, + 'with_nms': True + } + model_outputs = get_model_outputs(yolov5_head, 'predict_by_feat', + model_inputs) + + # to get outputs of onnx model after rewrite + wrapped_model = WrapModel( + yolov5_head, + 'predict_by_feat', + batch_img_metas=batch_img_metas, + with_nms=True) + rewrite_inputs = { + 'cls_scores': cls_scores, + 'bbox_preds': bbox_preds, + 'objectnesses': objectnesses, + } + rewrite_outputs, is_backend_output = get_rewrite_outputs( + wrapped_model=wrapped_model, + model_inputs=rewrite_inputs, + deploy_cfg=deploy_cfg) + + if is_backend_output: + # hard code to make two tensors with the same shape + # rewrite and original codes applied different nms strategy + min_shape = min(model_outputs[0].bboxes.shape[0], + rewrite_outputs[0].shape[1], 5) + for i in range(len(model_outputs)): + rewrite_outputs[0][i, :min_shape, 0::2] = \ + rewrite_outputs[0][i, :min_shape, 0::2].clamp_(0, s) + rewrite_outputs[0][i, :min_shape, 1::2] = \ + rewrite_outputs[0][i, :min_shape, 1::2].clamp_(0, s) + assert np.allclose( + model_outputs[i].bboxes[:min_shape], + rewrite_outputs[0][i, :min_shape, :4], + rtol=1e-03, + atol=1e-05) + assert np.allclose( + model_outputs[i].scores[:min_shape], + rewrite_outputs[0][i, :min_shape, 4], + rtol=1e-03, + atol=1e-05) + assert np.allclose( + model_outputs[i].labels[:min_shape], + rewrite_outputs[1][i, :min_shape], + rtol=1e-03, + atol=1e-05) + else: + assert rewrite_outputs is not None diff --git a/tests/test_deploy/test_object_detection.py b/tests/test_deploy/test_object_detection.py new file mode 100644 index 0000000000..69fe9fd019 --- /dev/null +++ b/tests/test_deploy/test_object_detection.py @@ -0,0 +1,96 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +from tempfile import NamedTemporaryFile, TemporaryDirectory + +import numpy as np +import pytest +import torch +from mmengine import Config + +try: + import importlib + importlib.import_module('mmdeploy') +except ImportError: + pytest.skip('mmdeploy is not installed.', allow_module_level=True) + +import mmdeploy.backend.onnxruntime as ort_apis +from mmdeploy.apis import build_task_processor +from mmdeploy.codebase import import_codebase +from mmdeploy.utils import load_config +from mmdeploy.utils.config_utils import register_codebase +from mmdeploy.utils.test import SwitchBackendWrapper + +try: + codebase = register_codebase('mmyolo') + import_codebase(codebase, ['mmyolo.deploy']) +except ImportError: + pytest.skip('mmyolo is not installed.', allow_module_level=True) + +model_cfg_path = 'tests/test_deploy/data/model.py' +model_cfg = load_config(model_cfg_path)[0] +model_cfg.test_dataloader.dataset.data_root = \ + 'tests/data' +model_cfg.test_dataloader.dataset.ann_file = 'coco_sample.json' +model_cfg.test_evaluator.ann_file = \ + 'tests/coco_sample.json' +deploy_cfg = Config( + dict( + backend_config=dict(type='onnxruntime'), + codebase_config=dict( + type='mmdet', + task='ObjectDetection', + post_processing=dict( + score_threshold=0.05, + confidence_threshold=0.005, # for YOLOv3 + iou_threshold=0.5, + max_output_boxes_per_class=200, + pre_top_k=5000, + keep_top_k=100, + background_label_id=-1, + ), + module=['mmyolo.deploy']), + onnx_config=dict( + type='onnx', + export_params=True, + keep_initializers_as_inputs=False, + opset_version=11, + input_shape=None, + input_names=['input'], + output_names=['dets', 'labels']))) +onnx_file = NamedTemporaryFile(suffix='.onnx').name +task_processor = None +img_shape = (32, 32) +img = np.random.rand(*img_shape, 3) + + +@pytest.fixture(autouse=True) +def init_task_processor(): + global task_processor + task_processor = build_task_processor(model_cfg, deploy_cfg, 'cpu') + + +@pytest.fixture +def backend_model(): + from mmdeploy.backend.onnxruntime import ORTWrapper + ort_apis.__dict__.update({'ORTWrapper': ORTWrapper}) + wrapper = SwitchBackendWrapper(ORTWrapper) + wrapper.set( + outputs={ + 'dets': torch.rand(1, 10, 5).sort(2).values, + 'labels': torch.randint(0, 10, (1, 10)) + }) + + yield task_processor.build_backend_model(['']) + + wrapper.recover() + + +def test_visualize(backend_model): + img_path = 'tests/data/color.jpg' + input_dict, _ = task_processor.create_input( + img_path, input_shape=img_shape) + results = backend_model.test_step(input_dict)[0] + with TemporaryDirectory() as dir: + filename = dir + 'tmp.jpg' + task_processor.visualize(img, results, filename, 'window') + assert os.path.exists(filename)