[Feature] Support more MMOCR models: DBNet++, TextSnake, ABINet, MRCNN

configs/mmocr/text-detection/text-detection_mrcnn_onnxruntime_dynamic.py

@@ -0,0 +1,33 @@
+_base_ = ['./text-detection_static.py', '../../_base_/backends/onnxruntime.py']
+onnx_config = dict(
+    output_names=['dets', 'labels', 'masks'],
+    dynamic_axes=dict(
+        input=dict({
+            0: 'batch',
+            2: 'height',
+            3: 'width'
+        }),
+        dets=dict({
+            0: 'batch',
+            1: 'num_dets'
+        }),
+        labels=dict({
+            0: 'batch',
+            1: 'num_dets'
+        }),
+        masks=dict({
+            0: 'batch',
+            1: 'num_dets',
+            2: 'height',
+            3: 'width'
+        })))
+codebase_config = dict(
+    post_processing=dict(
+        score_threshold=0.05,
+        confidence_threshold=0.005,
+        iou_threshold=0.5,
+        max_output_boxes_per_class=200,
+        pre_top_k=5000,
+        keep_top_k=100,
+        background_label_id=-1,
+        export_postprocess_mask=False))

configs/mmocr/text-detection/text-detection_mrcnn_tensorrt-fp16_dynamic-320x320-2240x2240.py

@@ -0,0 +1,2 @@
+_base_ = ['./text-detection_mrcnn_tensorrt_dynamic-320x320-2240x2240.py']
+backend_config = dict(common_config=dict(fp16_mode=True))

configs/mmocr/text-detection/text-detection_mrcnn_tensorrt-int8_dynamic-320x320-2240x2240.py

@@ -0,0 +1,5 @@
+_base_ = ['./text-detection_mrcnn_tensorrt_dynamic-320x320-2240x2240.py']
+
+backend_config = dict(common_config=dict(fp16_mode=True, int8_mode=True))
+
+calib_config = dict(create_calib=True, calib_file='calib_data.h5')

configs/mmocr/text-detection/text-detection_mrcnn_tensorrt_dynamic-320x320-2240x2240.py

@@ -0,0 +1,45 @@
+_base_ = ['./text-detection_static.py', '../../_base_/backends/tensorrt.py']
+onnx_config = dict(
+    output_names=['dets', 'labels', 'masks'],
+    dynamic_axes=dict(
+        input=dict({
+            0: 'batch',
+            2: 'height',
+            3: 'width'
+        }),
+        dets=dict({
+            0: 'batch',
+            1: 'num_dets'
+        }),
+        labels=dict({
+            0: 'batch',
+            1: 'num_dets'
+        }),
+        masks=dict({
+            0: 'batch',
+            1: 'num_dets',
+            2: 'height',
+            3: 'width'
+        })))
+
+backend_config = dict(
+    common_config=dict(max_workspace_size=1 << 30),
+    model_inputs=[
+        dict(
+            input_shapes=dict(
+                input=dict(
+                    min_shape=[1, 3, 320, 320],
+                    opt_shape=[1, 3, 600, 800],
+                    max_shape=[1, 3, 2240, 2240])))
+    ])
+
+codebase_config = dict(
+    post_processing=dict(
+        score_threshold=0.05,
+        confidence_threshold=0.005,
+        iou_threshold=0.5,
+        max_output_boxes_per_class=200,
+        pre_top_k=5000,
+        keep_top_k=100,
+        background_label_id=-1,
+        export_postprocess_mask=False))

configs/mmocr/text-recognition/text-recognition_tensorrt-fp16_dynamic-48x64-48x640.py

@@ -0,0 +1,13 @@
+_base_ = [
+    './text-recognition_dynamic.py', '../../_base_/backends/tensorrt-fp16.py'
+]
+backend_config = dict(
+    common_config=dict(max_workspace_size=1 << 30),
+    model_inputs=[
+        dict(
+            input_shapes=dict(
+                input=dict(
+                    min_shape=[1, 3, 48, 64],
+                    opt_shape=[1, 3, 48, 64],
+                    max_shape=[1, 3, 48, 640])))
+    ])

configs/mmocr/text-recognition/text-recognition_tensorrt-fp16_static-32x128.py

@@ -0,0 +1,15 @@
+_base_ = [
+    './text-recognition_static.py', '../../_base_/backends/tensorrt-fp16.py'
+]
+
+onnx_config = dict(input_shape=[128, 32])
+backend_config = dict(
+    common_config=dict(max_workspace_size=1 << 30),
+    model_inputs=[
+        dict(
+            input_shapes=dict(
+                input=dict(
+                    min_shape=[1, 3, 32, 128],
+                    opt_shape=[1, 3, 32, 128],
+                    max_shape=[1, 3, 32, 128])))
+    ])

configs/mmocr/text-recognition/text-recognition_tensorrt-int8_dynamic-48x64-48x640.py

@@ -0,0 +1,13 @@
+_base_ = [
+    './text-recognition_dynamic.py', '../../_base_/backends/tensorrt-int8.py'
+]
+backend_config = dict(
+    common_config=dict(max_workspace_size=1 << 30),
+    model_inputs=[
+        dict(
+            input_shapes=dict(
+                input=dict(
+                    min_shape=[1, 3, 48, 64],
+                    opt_shape=[1, 3, 48, 64],
+                    max_shape=[1, 3, 48, 640])))
+    ])

configs/mmocr/text-recognition/text-recognition_tensorrt-int8_static-32x128.py

@@ -0,0 +1,15 @@
+_base_ = [
+    './text-recognition_static.py', '../../_base_/backends/tensorrt-int8.py'
+]
+
+onnx_config = dict(input_shape=[128, 32])
+backend_config = dict(
+    common_config=dict(max_workspace_size=1 << 30),
+    model_inputs=[
+        dict(
+            input_shapes=dict(
+                input=dict(
+                    min_shape=[1, 3, 32, 128],
+                    opt_shape=[1, 3, 32, 128],
+                    max_shape=[1, 3, 32, 128])))
+    ])

configs/mmocr/text-recognition/text-recognition_tensorrt_dynamic-48x64-48x640.py

@@ -0,0 +1,11 @@
+_base_ = ['./text-recognition_dynamic.py', '../../_base_/backends/tensorrt.py']
+backend_config = dict(
+    common_config=dict(max_workspace_size=1 << 30),
+    model_inputs=[
+        dict(
+            input_shapes=dict(
+                input=dict(
+                    min_shape=[1, 3, 48, 64],
+                    opt_shape=[1, 3, 48, 64],
+                    max_shape=[1, 3, 48, 640])))
+    ])

configs/mmocr/text-recognition/text-recognition_tensorrt_static-32x128.py

@@ -0,0 +1,13 @@
+_base_ = ['./text-recognition_static.py', '../../_base_/backends/tensorrt.py']
+
+onnx_config = dict(input_shape=[128, 32])
+backend_config = dict(
+    common_config=dict(max_workspace_size=1 << 30),
+    model_inputs=[
+        dict(
+            input_shapes=dict(
+                input=dict(
+                    min_shape=[1, 3, 32, 128],
+                    opt_shape=[1, 3, 32, 128],
+                    max_shape=[1, 3, 32, 128])))
+    ])

docs/en/03-benchmark/benchmark.md

@@ -1178,6 +1178,42 @@ Users can directly test the performance through [how_to_evaluate_a_model.md](../
     <td align="center">0.7949</td>
     <td align="center">0.7950</td>
   </tr>
+ <tr>
+    <td align="center" rowspan="3"><a href="https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/dbnetpp/dbnetpp_resnet50_fpnc_1200e_icdar2015.py">DBNetpp</a></td>
+    <td align="center" rowspan="3">TextDetection</td>
+    <td align="center" rowspan="3">ICDAR2015</td>
+    <td align="center">recall</td>
+    <td align="center">0.8209</td>
+    <td align="center">0.8209</td>
+    <td align="center">0.8209</td>
+    <td align="center">0.8199</td>
+    <td align="center">0.8204</td>
+    <td align="center">0.8204</td>
+    <td align="center">-</td>
+    <td align="center">0.8209</td>
+  </tr>
+  <tr>
+    <td align="center">precision</td>
+    <td align="center">0.9079</td>
+    <td align="center">0.9079</td>
+    <td align="center">0.9079</td>
+    <td align="center">0.9117</td>
+    <td align="center">0.9117</td>
+    <td align="center">0.9142</td>
+    <td align="center">-</td>
+    <td align="center">0.9079</td>
+  </tr>
+  <tr>
+    <td align="center">hmean</td>
+    <td align="center">0.8622</td>
+    <td align="center">0.8622</td>
+    <td align="center">0.8622</td>
+    <td align="center">0.8634</td>
+    <td align="center">0.8637</td>
+    <td align="center">0.8648</td>
+    <td align="center">-</td>
+    <td align="center">0.8622</td>
+  </tr>
   <tr>
     <td align="center" rowspan="3"><a href="https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/psenet/psenet_resnet50_fpnf_600e_icdar2015.py">PSENet</a></td>
     <td align="center" rowspan="3">TextDetection</td>
@@ -1250,6 +1286,78 @@ Users can directly test the performance through [how_to_evaluate_a_model.md](../
     <td align="center">-</td>
     <td align="center">0.7955</td>
   </tr>
+  <tr>
+    <td align="center" rowspan="3"><a href="https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/textsnake/textsnake_resnet50_fpn-unet_1200e_ctw1500.py">TextSnake</a></td>
+    <td align="center" rowspan="3">TextDetection</td>
+    <td align="center" rowspan="3">CTW1500</td>
+    <td align="center">recall</td>
+    <td align="center">0.8052</td>
+    <td align="center">0.8052</td>
+    <td align="center">0.8052</td>
+    <td align="center">0.8055</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+  </tr>
+  <tr>
+    <td align="center">precision</td>
+    <td align="center">0.8535</td>
+    <td align="center">0.8535</td>
+    <td align="center">0.8535</td>
+    <td align="center">0.8538</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+  </tr>
+  <tr>
+    <td align="center">hmean</td>
+    <td align="center">0.8286</td>
+    <td align="center">0.8286</td>
+    <td align="center">0.8286</td>
+    <td align="center">0.8290</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+  </tr>
+  <tr>
+    <td align="center" rowspan="3"><a href="https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/maskrcnn/mask-rcnn_resnet50_fpn_160e_icdar2015.py">MaskRCNN</a></td>
+    <td align="center" rowspan="3">TextDetection</td>
+    <td align="center" rowspan="3">ICDAR2015</td>
+    <td align="center">recall</td>
+    <td align="center">0.7766</td>
+    <td align="center">0.7766</td>
+    <td align="center">0.7766</td>
+    <td align="center">0.7766</td>
+    <td align="center">0.7761</td>
+    <td align="center">0.7670</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+  </tr>
+  <tr>
+    <td align="center">precision</td>
+    <td align="center">0.8644</td>
+    <td align="center">0.8644</td>
+    <td align="center">0.8644</td>
+    <td align="center">0.8644</td>
+    <td align="center">0.8630</td>
+    <td align="center">0.8705</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+  </tr>
+  <tr>
+    <td align="center">hmean</td>
+    <td align="center">0.8182</td>
+    <td align="center">0.8182</td>
+    <td align="center">0.8182</td>
+    <td align="center">0.8182</td>
+    <td align="center">0.8172</td>
+    <td align="center">0.8155</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+  </tr>
   <tr>
     <td align="center"><a href="https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/crnn/crnn_mini-vgg_5e_mj.py">CRNN</a></td>
     <td align="center">TextRecognition</td>
@@ -1292,6 +1400,20 @@ Users can directly test the performance through [how_to_evaluate_a_model.md](../
     <td align="center">-</td>
     <td align="center">-</td>
   </tr>
+  <tr>
+    <td align="center"><a href="https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/abinet/abinet_20e_st-an_mj.py">ABINet</a></td>
+    <td align="center">TextRecognition</td>
+    <td align="center">IIIT5K</td>
+    <td align="center">acc</td>
+    <td align="center">0.9603</td>
+    <td align="center">0.9563</td>
+    <td align="center">0.9563</td>
+    <td align="center">0.9573</td>
+    <td align="center">0.9507</td>
+    <td align="center">0.9510</td>
+    <td align="center">-</td>
+    <td align="center">-</td>
+  </tr>
 </tbody>
 </table>
 </div>

docs/en/03-benchmark/supported_models.md

@@ -68,11 +68,15 @@ The table below lists the models that are guaranteed to be exportable to other b
 | [EDSR](https://github.com/open-mmlab/mmediting/tree/1.x/configs/edsr)                                   | MMEditing        |      Y      |      Y      |    Y     |  Y   |   N   |    Y     |   N    |  N   |
 | [RDN](https://github.com/open-mmlab/mmediting/tree/1.x/configs/rdn)                                     | MMEditing        |      Y      |      Y      |    Y     |  Y   |   Y   |    Y     |   N    |  N   |
 | [DBNet](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/dbnet)                             | MMOCR            |      Y      |      Y      |    Y     |  Y   |   Y   |    Y     |   Y    |  N   |
+| [DBNetpp](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/dbnetpp)                         | MMOCR            |      Y      |      Y      |    Y     |  ?   |   ?   |    Y     |   ?    |  N   |
 | [PANet](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/panet)                             | MMOCR            |      Y      |      Y      |    Y     |  Y   |   ?   |    Y     |   Y    |  N   |
 | [PSENet](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/psenet)                           | MMOCR            |      Y      |      Y      |    Y     |  Y   |   ?   |    Y     |   Y    |  N   |
+| [TextSnake](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/textsnake)                     | MMOCR            |      Y      |      Y      |    Y     |  Y   |   ?   |    ?     |   ?    |  N   |
+| [MaskRCNN](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/maskrcnn)                       | MMOCR            |      Y      |      Y      |    Y     |  ?   |   ?   |    ?     |   ?    |  N   |
 | [CRNN](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/crnn)                             | MMOCR            |      Y      |      Y      |    Y     |  Y   |   Y   |    N     |   N    |  N   |
 | [SAR](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/sar)                               | MMOCR            |      N      |      Y      |    N     |  N   |   N   |    N     |   N    |  N   |
 | [SATRN](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/satrn)                           | MMOCR            |      Y      |      Y      |    Y     |  N   |   N   |    N     |   N    |  N   |
+| [ABINet](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/abinet)                         | MMOCR            |      Y      |      Y      |    Y     |  N   |   N   |    N     |   N    |  N   |
 | [HRNet](https://mmpose.readthedocs.io/en/1.x/model_zoo_papers/backbones.html#hrnet-cvpr-2019)           | MMPose           |      N      |      Y      |    Y     |  Y   |   N   |    Y     |   N    |  N   |
 | [MSPN](https://mmpose.readthedocs.io/en/1.x/model_zoo_papers/backbones.html#mspn-arxiv-2019)            | MMPose           |      N      |      Y      |    Y     |  Y   |   N   |    Y     |   N    |  N   |
 | [LiteHRNet](https://mmpose.readthedocs.io/en/1.x/model_zoo_papers/backbones.html#litehrnet-cvpr-2021)   | MMPose           |      N      |      Y      |    Y     |  N   |   N   |    Y     |   N    |  N   |

docs/en/04-supported-codebases/mmocr.md

@@ -1,16 +1,19 @@
 # MMOCR Deployment
 
-- [Installation](#installation)
-  - [Install mmocr](#install-mmocr)
-  - [Install mmdeploy](#install-mmdeploy)
-- [Convert model](#convert-model)
-  - [Convert text detection model](#convert-text-detection-model)
-  - [Convert text recognition model](#convert-text-recognition-model)
-- [Model specification](#model-specification)
-- [Model Inference](#model-inference)
-  - [Backend model inference](#backend-model-inference)
-  - [SDK model inference](#sdk-model-inference)
-- [Supported models](#supported-models)
+- [MMOCR Deployment](#mmocr-deployment)
+  - [Installation](#installation)
+    - [Install mmocr](#install-mmocr)
+    - [Install mmdeploy](#install-mmdeploy)
+  - [Convert model](#convert-model)
+    - [Convert text detection model](#convert-text-detection-model)
+    - [Convert text recognition model](#convert-text-recognition-model)
+  - [Model specification](#model-specification)
+  - [Model Inference](#model-inference)
+    - [Backend model inference](#backend-model-inference)
+    - [SDK model inference](#sdk-model-inference)
+      - [Text detection SDK model inference](#text-detection-sdk-model-inference)
+      - [Text Recognition SDK model inference](#text-recognition-sdk-model-inference)
+  - [Supported models](#supported-models)
 
 ______________________________________________________________________
 
@@ -230,11 +233,15 @@ Besides python API, mmdeploy SDK also provides other FFI (Foreign Function Inter
 
 ## Supported models
 
-| Model                                                                         | Task             | TorchScript | OnnxRuntime | TensorRT | ncnn | PPLNN | OpenVINO |
-| :---------------------------------------------------------------------------- | :--------------- | :---------: | :---------: | :------: | :--: | :---: | :------: |
-| [DBNet](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/dbnet)   | text-detection   |      Y      |      Y      |    Y     |  Y   |   Y   |    Y     |
-| [PSENet](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/psenet) | text-detection   |      Y      |      Y      |    Y     |  Y   |   N   |    Y     |
-| [PANet](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/panet)   | text-detection   |      Y      |      Y      |    Y     |  Y   |   N   |    Y     |
-| [CRNN](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/crnn)   | text-recognition |      Y      |      Y      |    Y     |  Y   |   Y   |    N     |
-| [SAR](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/sar)     | text-recognition |      N      |      Y      |    N     |  N   |   N   |    N     |
-| [SATRN](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/satrn) | text-recognition |      Y      |      Y      |    Y     |  N   |   N   |    N     |
+| Model                                                                               | Task             | TorchScript | OnnxRuntime | TensorRT | ncnn | PPLNN | OpenVINO |
+| :---------------------------------------------------------------------------------- | :--------------- | :---------: | :---------: | :------: | :--: | :---: | :------: |
+| [DBNet](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/dbnet)         | text-detection   |      Y      |      Y      |    Y     |  Y   |   Y   |    Y     |
+| [DBNetpp](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/dbnetpp)     | text-detection   |      N      |      Y      |    Y     |  ?   |   ?   |    Y     |
+| [PSENet](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/psenet)       | text-detection   |      Y      |      Y      |    Y     |  Y   |   N   |    Y     |
+| [PANet](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/panet)         | text-detection   |      Y      |      Y      |    Y     |  Y   |   N   |    Y     |
+| [TextSnake](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/textsnake) | text-detection   |      Y      |      Y      |    Y     |  ?   |   ?   |    ?     |
+| [MaskRCNN](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textdet/maskrcnn)   | text-detection   |      Y      |      Y      |    Y     |  ?   |   ?   |    ?     |
+| [CRNN](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/crnn)         | text-recognition |      Y      |      Y      |    Y     |  Y   |   Y   |    N     |
+| [SAR](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/sar)           | text-recognition |      N      |      Y      |    Y     |  N   |   N   |    N     |
+| [SATRN](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/satrn)       | text-recognition |      Y      |      Y      |    Y     |  N   |   N   |    N     |
+| [ABINet](https://github.com/open-mmlab/mmocr/blob/1.x/configs/textrecog/abinet)     | text-recognition |      Y      |      Y      |    Y     |  ?   |   ?   |    ?     |