diff --git a/configs/lvis/README.md b/configs/lvis/README.md
index 32768030d61..10fa6d5425b 100644
--- a/configs/lvis/README.md
+++ b/configs/lvis/README.md
@@ -19,14 +19,7 @@
 * Run following scripts to install our forked lvis-api.
 
     ```shell
-    # mmlvis is fully compatible with official lvis
-    pip install mmlvis
-    ```
-
-    or
-
-    ```shell
-    pip install -r requirements/optional.txt
+    pip install git+https://github.com/lvis-dataset/lvis-api.git
     ```
 
 * All experiments use oversample strategy [here](../../docs/tutorials/new_dataset.md#class-balanced-dataset) with oversample threshold `1e-3`.
diff --git a/docs/compatibility.md b/docs/compatibility.md
index 922fc532abf..f9dfb2e3a05 100644
--- a/docs/compatibility.md
+++ b/docs/compatibility.md
@@ -80,3 +80,11 @@ model-level compatibility but slightly improves the performance. The major ones
 To convert the models trained by MMDetection V1.x to MMDetection V2.0, the users can use the script `tools/model_converters/upgrade_model_version.py` to convert
 their models. The converted models can be run in MMDetection V2.0 with slightly dropped performance (less than 1% AP absolute).
 Details can be found in `configs/legacy`.
+
+## pycocotools compatibility
+
+`mmpycocotools` is the OpenMMlab's folk of official `pycocotools`, which works for both MMDetection and Detectron2.
+Before [PR 4939](https://github.com/open-mmlab/mmdetection/pull/4939), since `pycocotools` and `mmpycocotool` have the same package name, if users already installed `pyccocotools` (installed Detectron2 first under the same environment), then the setup of MMDetection will skip installing `mmpycocotool`. Thus MMDetection fails due to the missing `mmpycocotools`.
+If MMDetection is installed before Detectron2, they could work under the same environment.
+[PR 4939](https://github.com/open-mmlab/mmdetection/pull/4939) deprecates mmpycocotools in favor of official pycocotools.
+Users may install MMDetection and Detectron2 under the same environment after [PR 4939](https://github.com/open-mmlab/mmdetection/pull/4939), no matter what the installation order is.
diff --git a/mmdet/datasets/api_wrappers/__init__.py b/mmdet/datasets/api_wrappers/__init__.py
new file mode 100644
index 00000000000..05f95c94ae6
--- /dev/null
+++ b/mmdet/datasets/api_wrappers/__init__.py
@@ -0,0 +1,3 @@
+from .coco_api import COCO, COCOeval
+
+__all__ = ['COCO', 'COCOeval']
diff --git a/mmdet/datasets/api_wrappers/coco_api.py b/mmdet/datasets/api_wrappers/coco_api.py
new file mode 100644
index 00000000000..5c38383a6d6
--- /dev/null
+++ b/mmdet/datasets/api_wrappers/coco_api.py
@@ -0,0 +1,46 @@
+# This file add snake case alias for coco api
+
+import warnings
+
+import pycocotools
+from pycocotools.coco import COCO as _COCO
+from pycocotools.cocoeval import COCOeval as _COCOeval
+
+
+class COCO(_COCO):
+    """This class is almost the same as official pycocotools package.
+
+    It implements some snake case function aliases. So that the COCO class has
+    the same interface as LVIS class.
+    """
+
+    def __init__(self, annotation_file=None):
+        if not getattr(pycocotools, '__version__', '0') >= '12.0.2':
+            warnings.warn(
+                'mmpycocotools is deprecated. Please install official pycocotools by "pip install pycocotools"',  # noqa: E501
+                UserWarning)
+        super().__init__(annotation_file=annotation_file)
+        self.img_ann_map = self.imgToAnns
+        self.cat_img_map = self.catToImgs
+
+    def get_ann_ids(self, img_ids=[], cat_ids=[], area_rng=[], iscrowd=None):
+        return self.getAnnIds(img_ids, cat_ids, area_rng, iscrowd)
+
+    def get_cat_ids(self, cat_names=[], sup_names=[], cat_ids=[]):
+        return self.getCatIds(cat_names, sup_names, cat_ids)
+
+    def get_img_ids(self, img_ids=[], cat_ids=[]):
+        return self.getImgIds(img_ids, cat_ids)
+
+    def load_anns(self, ids):
+        return self.loadAnns(ids)
+
+    def load_cats(self, ids):
+        return self.loadCats(ids)
+
+    def load_imgs(self, ids):
+        return self.loadImgs(ids)
+
+
+# just for the ease of import
+COCOeval = _COCOeval
diff --git a/mmdet/datasets/coco.py b/mmdet/datasets/coco.py
index 3a8e1bcfdd7..152f5bde4d6 100644
--- a/mmdet/datasets/coco.py
+++ b/mmdet/datasets/coco.py
@@ -6,13 +6,11 @@
 
 import mmcv
 import numpy as np
-import pycocotools
 from mmcv.utils import print_log
-from pycocotools.coco import COCO
-from pycocotools.cocoeval import COCOeval
 from terminaltables import AsciiTable
 
 from mmdet.core import eval_recalls
+from .api_wrappers import COCO, COCOeval
 from .builder import DATASETS
 from .custom import CustomDataset
 
@@ -44,12 +42,6 @@ def load_annotations(self, ann_file):
         Returns:
             list[dict]: Annotation info from COCO api.
         """
-        if not getattr(pycocotools, '__version__', '0') >= '12.0.2':
-            raise AssertionError(
-                'Incompatible version of pycocotools is installed. '
-                'Run pip uninstall pycocotools first. Then run pip '
-                'install mmpycocotools to install open-mmlab forked '
-                'pycocotools.')
 
         self.coco = COCO(ann_file)
         self.cat_ids = self.coco.get_cat_ids(cat_names=self.CLASSES)
diff --git a/mmdet/datasets/lvis.py b/mmdet/datasets/lvis.py
index 122c64e79cf..8850e3bc045 100644
--- a/mmdet/datasets/lvis.py
+++ b/mmdet/datasets/lvis.py
@@ -2,6 +2,7 @@
 import logging
 import os.path as osp
 import tempfile
+import warnings
 from collections import OrderedDict
 
 import numpy as np
@@ -277,17 +278,15 @@ def load_annotations(self, ann_file):
 
         try:
             import lvis
-            assert lvis.__version__ >= '10.5.3'
+            if lvis.__version__ >= '10.5.3':
+                warnings.warn(
+                    'mmlvis is deprecated, please install official lvis-api by "pip install git+https://github.com/lvis-dataset/lvis-api.git"',  # noqa: E501
+                    UserWarning)
             from lvis import LVIS
-        except AssertionError:
-            raise AssertionError('Incompatible version of lvis is installed. '
-                                 'Run pip uninstall lvis first. Then run pip '
-                                 'install mmlvis to install open-mmlab forked '
-                                 'lvis. ')
         except ImportError:
-            raise ImportError('Package lvis is not installed. Please run pip '
-                              'install mmlvis to install open-mmlab forked '
-                              'lvis.')
+            raise ImportError(
+                'Package lvis is not installed. Please run "pip install git+https://github.com/lvis-dataset/lvis-api.git".'  # noqa: E501
+            )
         self.coco = LVIS(ann_file)
         self.cat_ids = self.coco.get_cat_ids()
         self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)}
@@ -337,17 +336,15 @@ def evaluate(self,
 
         try:
             import lvis
-            assert lvis.__version__ >= '10.5.3'
+            if lvis.__version__ >= '10.5.3':
+                warnings.warn(
+                    'mmlvis is deprecated, please install official lvis-api by "pip install git+https://github.com/lvis-dataset/lvis-api.git"',  # noqa: E501
+                    UserWarning)
             from lvis import LVISResults, LVISEval
-        except AssertionError:
-            raise AssertionError('Incompatible version of lvis is installed. '
-                                 'Run pip uninstall lvis first. Then run pip '
-                                 'install mmlvis to install open-mmlab forked '
-                                 'lvis. ')
         except ImportError:
-            raise ImportError('Package lvis is not installed. Please run pip '
-                              'install mmlvis to install open-mmlab forked '
-                              'lvis.')
+            raise ImportError(
+                'Package lvis is not installed. Please run "pip install git+https://github.com/lvis-dataset/lvis-api.git".'  # noqa: E501
+            )
         assert isinstance(results, list), 'results must be a list'
         assert len(results) == len(self), (
             'The length of results is not equal to the dataset len: {} != {}'.
@@ -714,18 +711,11 @@ class LVISV1Dataset(LVISDataset):
 
     def load_annotations(self, ann_file):
         try:
-            import lvis
-            assert lvis.__version__ >= '10.5.3'
             from lvis import LVIS
-        except AssertionError:
-            raise AssertionError('Incompatible version of lvis is installed. '
-                                 'Run pip uninstall lvis first. Then run pip '
-                                 'install mmlvis to install open-mmlab forked '
-                                 'lvis. ')
         except ImportError:
-            raise ImportError('Package lvis is not installed. Please run pip '
-                              'install mmlvis to install open-mmlab forked '
-                              'lvis.')
+            raise ImportError(
+                'Package lvis is not installed. Please run "pip install git+https://github.com/lvis-dataset/lvis-api.git".'  # noqa: E501
+            )
         self.coco = LVIS(ann_file)
         self.cat_ids = self.coco.get_cat_ids()
         self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)}
diff --git a/requirements/optional.txt b/requirements/optional.txt
index 35b5242b5c4..ac9688b010d 100644
--- a/requirements/optional.txt
+++ b/requirements/optional.txt
@@ -1,6 +1,5 @@
 albumentations>=0.3.2
 cityscapesscripts
 imagecorruptions
-mmlvis
 scipy
 sklearn
diff --git a/requirements/runtime.txt b/requirements/runtime.txt
index 8eb0c3db8c1..f7a2cc7eb8b 100644
--- a/requirements/runtime.txt
+++ b/requirements/runtime.txt
@@ -1,5 +1,5 @@
 matplotlib
-mmpycocotools
 numpy
+pycocotools
 six
 terminaltables