diff --git a/examples/comparison/mmdet/coco_metric.py b/examples/comparison/mmdet/coco_metric.py deleted file mode 100644 index fbb5ac4..0000000 --- a/examples/comparison/mmdet/coco_metric.py +++ /dev/null @@ -1,703 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import datetime -import itertools -import os.path as osp -import tempfile -from collections import OrderedDict -from typing import Dict, List, Optional, Sequence, Union - -import numpy as np -import torch -from mmdet.datasets.api_wrappers import COCO, COCOeval, COCOevalMP -from mmdet.evaluation.functional import eval_recalls -from mmdet.registry import METRICS -from mmdet.structures.mask import encode_mask_results -from mmengine.evaluator import BaseMetric -from mmengine.fileio import dump, get_local_path, load -from mmengine.logging import MMLogger -from terminaltables import AsciiTable - -from faster_coco_eval import COCO as COCO_faster -from faster_coco_eval import COCOeval_faster - -if "CocoMetric" in METRICS.module_dict: - del METRICS.module_dict["CocoMetric"] - - -@METRICS.register_module() -class CocoMetric(BaseMetric): - """COCO evaluation metric. - - Evaluate AR, AP, and mAP for detection tasks including proposal/box - detection and instance segmentation. Please refer to - https://cocodataset.org/#detection-eval for more details. - - Args: - ann_file (str, optional): Path to the coco format annotation file. - If not specified, ground truth annotations from the dataset will - be converted to coco format. Defaults to None. - metric (str | List[str]): Metrics to be evaluated. Valid metrics - include 'bbox', 'segm', 'proposal', and 'proposal_fast'. - Defaults to 'bbox'. - classwise (bool): Whether to evaluate the metric class-wise. - Defaults to False. - proposal_nums (Sequence[int]): Numbers of proposals to be evaluated. - Defaults to (100, 300, 1000). - iou_thrs (float | List[float], optional): IoU threshold to compute AP - and AR. If not specified, IoUs from 0.5 to 0.95 will be used. - Defaults to None. - metric_items (List[str], optional): Metric result names to be - recorded in the evaluation result. Defaults to None. - format_only (bool): Format the output results without perform - evaluation. It is useful when you want to format the result - to a specific format and submit it to the test server. - Defaults to False. - outfile_prefix (str, optional): The prefix of json files. It includes - the file path and the prefix of filename, e.g., "a/b/prefix". - If not specified, a temp file will be created. Defaults to None. - file_client_args (dict, optional): Arguments to instantiate the - corresponding backend in mmdet <= 3.0.0rc6. Defaults to None. - backend_args (dict, optional): Arguments to instantiate the - corresponding backend. Defaults to None. - collect_device (str): Device name used for collecting results from - different ranks during distributed training. Must be 'cpu' or - 'gpu'. Defaults to 'cpu'. - prefix (str, optional): The prefix that will be added in the metric - names to disambiguate homonymous metrics of different evaluators. - If prefix is not provided in the argument, self.default_prefix - will be used instead. Defaults to None. - sort_categories (bool): Whether sort categories in annotations. Only - used for `Objects365V1Dataset`. Defaults to False. - use_mp_eval (bool): Whether to use mul-processing evaluation - - """ - - default_prefix: Optional[str] = "coco" - - def __init__( - self, - ann_file: Optional[str] = None, - metric: Union[str, List[str]] = "bbox", - classwise: bool = False, - proposal_nums: Sequence[int] = (100, 300, 1000), - iou_thrs: Optional[Union[float, Sequence[float]]] = None, - metric_items: Optional[Sequence[str]] = None, - format_only: bool = False, - outfile_prefix: Optional[str] = None, - file_client_args: dict = None, - backend_args: dict = None, - collect_device: str = "cpu", - prefix: Optional[str] = None, - sort_categories: bool = False, - use_mp_eval: bool = False, - use_faster: bool = False, - ) -> None: - super().__init__(collect_device=collect_device, prefix=prefix) - # coco evaluation metrics - self.metrics = metric if isinstance(metric, list) else [metric] - allowed_metrics = ["bbox", "segm", "proposal", "proposal_fast"] - for metric in self.metrics: - if metric not in allowed_metrics: - raise KeyError( - "metric should be one of 'bbox', 'segm', 'proposal', " - f"'proposal_fast', but got {metric}." - ) - - # do class wise evaluation, default False - self.classwise = classwise - # whether to use multi processing evaluation, default False - self.use_mp_eval = use_mp_eval - # faster_coco_eval - self.use_faster = use_faster - - # proposal_nums used to compute recall or precision. - self.proposal_nums = list(proposal_nums) - - # iou_thrs used to compute recall or precision. - if iou_thrs is None: - iou_thrs = np.linspace( - 0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True - ) - self.iou_thrs = iou_thrs - self.metric_items = metric_items - self.format_only = format_only - if self.format_only: - assert outfile_prefix is not None, "outfile_prefix must be not" - "None when format_only is True, otherwise the result files will" - "be saved to a temp directory which will be cleaned up at the end." - - self.outfile_prefix = outfile_prefix - - self.backend_args = backend_args - if file_client_args is not None: - raise RuntimeError( - "The `file_client_args` is deprecated, " - "please use `backend_args` instead, please refer to" - "https://github.com/open-mmlab/mmdetection/blob/main/configs/_base_/datasets/coco_detection.py" # noqa: E501 - ) - - # if ann_file is not specified, - # initialize coco api with the converted dataset - if ann_file is not None: - with get_local_path( - ann_file, backend_args=self.backend_args - ) as local_path: - if self.use_faster: - self._coco_api = COCO_faster(local_path) - else: - self._coco_api = COCO(local_path) - if sort_categories: - # 'categories' list in objects365_train.json and - # objects365_val.json is inconsistent, need sort - # list(or dict) before get cat_ids. - cats = self._coco_api.cats - sorted_cats = {i: cats[i] for i in sorted(cats)} - self._coco_api.cats = sorted_cats - categories = self._coco_api.dataset["categories"] - sorted_categories = sorted( - categories, key=lambda i: i["id"] - ) - self._coco_api.dataset["categories"] = sorted_categories - else: - self._coco_api = None - - # handle dataset lazy init - self.cat_ids = None - self.img_ids = None - - def fast_eval_recall( - self, - results: List[dict], - proposal_nums: Sequence[int], - iou_thrs: Sequence[float], - logger: Optional[MMLogger] = None, - ) -> np.ndarray: - """Evaluate proposal recall with COCO's fast_eval_recall. - - Args: - results (List[dict]): Results of the dataset. - proposal_nums (Sequence[int]): Proposal numbers used for - evaluation. - iou_thrs (Sequence[float]): IoU thresholds used for evaluation. - logger (MMLogger, optional): Logger used for logging the recall - summary. - Returns: - np.ndarray: Averaged recall results. - - """ - gt_bboxes = [] - pred_bboxes = [result["bboxes"] for result in results] - for i in range(len(self.img_ids)): - ann_ids = self._coco_api.get_ann_ids(img_ids=self.img_ids[i]) - ann_info = self._coco_api.load_anns(ann_ids) - if len(ann_info) == 0: - gt_bboxes.append(np.zeros((0, 4))) - continue - bboxes = [] - for ann in ann_info: - if ann.get("ignore", False) or ann["iscrowd"]: - continue - x1, y1, w, h = ann["bbox"] - bboxes.append([x1, y1, x1 + w, y1 + h]) - bboxes = np.array(bboxes, dtype=np.float32) - if bboxes.shape[0] == 0: - bboxes = np.zeros((0, 4)) - gt_bboxes.append(bboxes) - - recalls = eval_recalls( - gt_bboxes, pred_bboxes, proposal_nums, iou_thrs, logger=logger - ) - ar = recalls.mean(axis=1) - return ar - - def xyxy2xywh(self, bbox: np.ndarray) -> list: - """Convert ``xyxy`` style bounding boxes to ``xywh`` style for COCO - evaluation. - - Args: - bbox (numpy.ndarray): The bounding boxes, shape (4, ), in - ``xyxy`` order. - - Returns: - list[float]: The converted bounding boxes, in ``xywh`` order. - - """ - - _bbox: List = bbox.tolist() - return [ - _bbox[0], - _bbox[1], - _bbox[2] - _bbox[0], - _bbox[3] - _bbox[1], - ] - - def results2json( - self, results: Sequence[dict], outfile_prefix: str - ) -> dict: - """Dump the detection results to a COCO style json file. - - There are 3 types of results: proposals, bbox predictions, mask - predictions, and they have different data types. This method will - automatically recognize the type, and dump them to json files. - - Args: - results (Sequence[dict]): Testing results of the - dataset. - outfile_prefix (str): The filename prefix of the json files. If the - prefix is "somepath/xxx", the json files will be named - "somepath/xxx.bbox.json", "somepath/xxx.segm.json", - "somepath/xxx.proposal.json". - - Returns: - dict: Possible keys are "bbox", "segm", "proposal", and - values are corresponding filenames. - - """ - bbox_json_results = [] - segm_json_results = [] if "masks" in results[0] else None - for idx, result in enumerate(results): - image_id = result.get("img_id", idx) - labels = result["labels"] - bboxes = result["bboxes"] - scores = result["scores"] - # bbox results - for i, label in enumerate(labels): - data = dict() - data["image_id"] = image_id - data["bbox"] = self.xyxy2xywh(bboxes[i]) - data["score"] = float(scores[i]) - data["category_id"] = self.cat_ids[label] - bbox_json_results.append(data) - - if segm_json_results is None: - continue - - # segm results - masks = result["masks"] - mask_scores = result.get("mask_scores", scores) - for i, label in enumerate(labels): - data = dict() - data["image_id"] = image_id - data["bbox"] = self.xyxy2xywh(bboxes[i]) - data["score"] = float(mask_scores[i]) - data["category_id"] = self.cat_ids[label] - if isinstance(masks[i]["counts"], bytes): - masks[i]["counts"] = masks[i]["counts"].decode() - data["segmentation"] = masks[i] - segm_json_results.append(data) - - result_files = dict() - result_files["bbox"] = f"{outfile_prefix}.bbox.json" - result_files["proposal"] = f"{outfile_prefix}.bbox.json" - dump(bbox_json_results, result_files["bbox"]) - - if segm_json_results is not None: - result_files["segm"] = f"{outfile_prefix}.segm.json" - dump(segm_json_results, result_files["segm"]) - - return result_files - - def gt_to_coco_json( - self, gt_dicts: Sequence[dict], outfile_prefix: str - ) -> str: - """Convert ground truth to coco format json file. - - Args: - gt_dicts (Sequence[dict]): Ground truth of the dataset. - outfile_prefix (str): The filename prefix of the json files. If the - prefix is "somepath/xxx", the json file will be named - "somepath/xxx.gt.json". - Returns: - str: The filename of the json file. - - """ - categories = [ - dict(id=id, name=name) - for id, name in enumerate(self.dataset_meta["classes"]) - ] - image_infos = [] - annotations = [] - - for idx, gt_dict in enumerate(gt_dicts): - img_id = gt_dict.get("img_id", idx) - image_info = dict( - id=img_id, - width=gt_dict["width"], - height=gt_dict["height"], - file_name="", - ) - image_infos.append(image_info) - for ann in gt_dict["anns"]: - label = ann["bbox_label"] - bbox = ann["bbox"] - coco_bbox = [ - bbox[0], - bbox[1], - bbox[2] - bbox[0], - bbox[3] - bbox[1], - ] - - annotation = dict( - id=len(annotations) - + 1, # coco api requires id starts with 1 - image_id=img_id, - bbox=coco_bbox, - iscrowd=ann.get("ignore_flag", 0), - category_id=int(label), - area=coco_bbox[2] * coco_bbox[3], - ) - if ann.get("mask", None): - mask = ann["mask"] - # area = mask_util.area(mask) - if isinstance(mask, dict) and isinstance( - mask["counts"], bytes - ): - mask["counts"] = mask["counts"].decode() - annotation["segmentation"] = mask - # annotation['area'] = float(area) - annotations.append(annotation) - - info = dict( - date_created=str(datetime.datetime.now()), - description="Coco json file converted by mmdet CocoMetric.", - ) - coco_json = dict( - info=info, - images=image_infos, - categories=categories, - licenses=None, - ) - if len(annotations) > 0: - coco_json["annotations"] = annotations - converted_json_path = f"{outfile_prefix}.gt.json" - dump(coco_json, converted_json_path) - return converted_json_path - - # TODO: data_batch is no longer needed, consider adjusting the - # parameter position - def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: - """Process one batch of data samples and predictions. The processed - results should be stored in ``self.results``, which will be used to - compute the metrics when all batches have been processed. - - Args: - data_batch (dict): A batch of data from the dataloader. - data_samples (Sequence[dict]): A batch of data samples that - contain annotations and predictions. - - """ - for data_sample in data_samples: - result = dict() - pred = data_sample["pred_instances"] - result["img_id"] = data_sample["img_id"] - result["bboxes"] = pred["bboxes"].cpu().numpy() - result["scores"] = pred["scores"].cpu().numpy() - result["labels"] = pred["labels"].cpu().numpy() - # encode mask to RLE - if "masks" in pred: - result["masks"] = ( - encode_mask_results(pred["masks"].detach().cpu().numpy()) - if isinstance(pred["masks"], torch.Tensor) - else pred["masks"] - ) - # some detectors use different scores for bbox and mask - if "mask_scores" in pred: - result["mask_scores"] = pred["mask_scores"].cpu().numpy() - - # parse gt - gt = dict() - gt["width"] = data_sample["ori_shape"][1] - gt["height"] = data_sample["ori_shape"][0] - gt["img_id"] = data_sample["img_id"] - if self._coco_api is None: - # TODO: Need to refactor to support LoadAnnotations - assert "instances" in data_sample, ( - "ground truth is required for evaluation when " - "`ann_file` is not provided" - ) - gt["anns"] = data_sample["instances"] - # add converted result to the results list - self.results.append((gt, result)) - - def compute_metrics(self, results: list) -> Dict[str, float]: - """Compute the metrics from processed results. - - Args: - results (list): The processed results of each batch. - - Returns: - Dict[str, float]: The computed metrics. The keys are the names of - the metrics, and the values are corresponding results. - - """ - logger: MMLogger = MMLogger.get_current_instance() - - # split gt and prediction list - gts, preds = zip(*results) - - tmp_dir = None - if self.outfile_prefix is None: - tmp_dir = tempfile.TemporaryDirectory() - outfile_prefix = osp.join(tmp_dir.name, "results") - else: - outfile_prefix = self.outfile_prefix - - if self._coco_api is None: - # use converted gt json file to initialize coco api - logger.info("Converting ground truth to coco format...") - coco_json_path = self.gt_to_coco_json( - gt_dicts=gts, outfile_prefix=outfile_prefix - ) - - if self.use_faster: - self._coco_api = COCO_faster(coco_json_path) - else: - self._coco_api = COCO(coco_json_path) - - # handle lazy init - if self.cat_ids is None: - self.cat_ids = self._coco_api.get_cat_ids( - cat_names=self.dataset_meta["classes"] - ) - if self.img_ids is None: - self.img_ids = self._coco_api.get_img_ids() - - # convert predictions to coco format and dump to json file - result_files = self.results2json(preds, outfile_prefix) - - eval_results = OrderedDict() - if self.format_only: - logger.info(f"results are saved in {osp.dirname(outfile_prefix)}") - return eval_results - - for metric in self.metrics: - logger.info(f"Evaluating {metric}...") - - # TODO: May refactor fast_eval_recall to an independent metric? - # fast eval recall - if metric == "proposal_fast": - ar = self.fast_eval_recall( - preds, self.proposal_nums, self.iou_thrs, logger=logger - ) - log_msg = [] - for i, num in enumerate(self.proposal_nums): - eval_results[f"AR@{num}"] = ar[i] - log_msg.append(f"\nAR@{num}\t{ar[i]:.4f}") - log_msg = "".join(log_msg) - logger.info(log_msg) - continue - - # evaluate proposal, bbox and segm - iou_type = "bbox" if metric == "proposal" else metric - if metric not in result_files: - raise KeyError(f"{metric} is not in results") - try: - predictions = load(result_files[metric]) - if iou_type == "segm": - # Refer to https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/coco.py#L331 # noqa - # When evaluating mask AP, if the results contain bbox, - # cocoapi will use the box area instead of the mask area - # for calculating the instance area. Though the overall AP - # is not affected, this leads to different - # small/medium/large mask AP results. - for x in predictions: - x.pop("bbox") - coco_dt = self._coco_api.loadRes(predictions) - - except IndexError: - logger.error( - "The testing results of the whole dataset is empty." - ) - break - - if self.use_mp_eval: - coco_eval = COCOevalMP(self._coco_api, coco_dt, iou_type) - else: - if self.use_faster: - coco_eval = COCOeval_faster( - self._coco_api, coco_dt, iou_type, print_function=print - ) - else: - coco_eval = COCOeval(self._coco_api, coco_dt, iou_type) - - coco_eval.params.catIds = self.cat_ids - coco_eval.params.imgIds = self.img_ids - coco_eval.params.maxDets = list(self.proposal_nums) - coco_eval.params.iouThrs = self.iou_thrs - - # mapping of cocoEval.stats - coco_metric_names = { - "mAP": 0, - "mAP_50": 1, - "mAP_75": 2, - "mAP_s": 3, - "mAP_m": 4, - "mAP_l": 5, - "AR@100": 6, - "AR@300": 7, - "AR@1000": 8, - "AR_s@1000": 9, - "AR_m@1000": 10, - "AR_l@1000": 11, - } - metric_items = self.metric_items - if metric_items is not None: - for metric_item in metric_items: - if metric_item not in coco_metric_names: - raise KeyError( - f'metric item "{metric_item}" is not supported' - ) - - if metric == "proposal": - coco_eval.params.useCats = 0 - coco_eval.evaluate() - coco_eval.accumulate() - coco_eval.summarize() - if metric_items is None: - metric_items = [ - "AR@100", - "AR@300", - "AR@1000", - "AR_s@1000", - "AR_m@1000", - "AR_l@1000", - ] - - for item in metric_items: - val = float( - f"{coco_eval.stats[coco_metric_names[item]]:.3f}" - ) - eval_results[item] = val - else: - coco_eval.evaluate() - coco_eval.accumulate() - coco_eval.summarize() - if self.classwise: # Compute per-category AP - # Compute per-category AP - # from https://github.com/facebookresearch/detectron2/ - precisions = coco_eval.eval["precision"] - # precision: (iou, recall, cls, area range, max dets) - assert len(self.cat_ids) == precisions.shape[2] - - results_per_category = [] - for idx, cat_id in enumerate(self.cat_ids): - t = [] - # area range index 0: all area ranges - # max dets index -1: typically 100 per image - nm = self._coco_api.loadCats(cat_id)[0] - precision = precisions[:, :, idx, 0, -1] - precision = precision[precision > -1] - if precision.size: - ap = np.mean(precision) - else: - ap = float("nan") - t.append(f'{nm["name"]}') - t.append(f"{round(ap, 3)}") - eval_results[f'{nm["name"]}_precision'] = round(ap, 3) - - # indexes of IoU @50 and @75 - for iou in [0, 5]: - precision = precisions[iou, :, idx, 0, -1] - precision = precision[precision > -1] - if precision.size: - ap = np.mean(precision) - else: - ap = float("nan") - t.append(f"{round(ap, 3)}") - - # indexes of area of small, median and large - for area in [1, 2, 3]: - precision = precisions[:, :, idx, area, -1] - precision = precision[precision > -1] - if precision.size: - ap = np.mean(precision) - else: - ap = float("nan") - t.append(f"{round(ap, 3)}") - results_per_category.append(tuple(t)) - - num_columns = len(results_per_category[0]) - results_flatten = list( - itertools.chain(*results_per_category) - ) - headers = [ - "category", - "mAP", - "mAP_50", - "mAP_75", - "mAP_s", - "mAP_m", - "mAP_l", - ] - results_2d = itertools.zip_longest( - *[ - results_flatten[i::num_columns] - for i in range(num_columns) - ] - ) - table_data = [headers] - table_data += [result for result in results_2d] - table = AsciiTable(table_data) - logger.info("\n" + table.table) - - if metric_items is None: - metric_items = [ - "mAP", - "mAP_50", - "mAP_75", - "mAP_s", - "mAP_m", - "mAP_l", - ] - - for metric_item in metric_items: - key = f"{metric}_{metric_item}" - val = coco_eval.stats[coco_metric_names[metric_item]] - eval_results[key] = float(f"{round(val, 3)}") - - ap = coco_eval.stats[:6] - logger.info( - f"{metric}_mAP_copypaste: {ap[0]:.3f} " - f"{ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} " - f"{ap[4]:.3f} {ap[5]:.3f}" - ) - - if tmp_dir is not None: - tmp_dir.cleanup() - return eval_results - - -if __name__ == "__main__": - import argparse - import pickle - import time - - parser = argparse.ArgumentParser() - parser.add_argument("tmp_results", type=str, help="path to tmp_results.pkl") - parser.add_argument( - "dataset_meta", type=str, help="path to dataset_meta.pkl" - ) - parser.add_argument("--metric", type=str, help="metric", default="bbox") - parser.add_argument( - "--use_faster", action=argparse.BooleanOptionalAction, default=False - ) - - args = parser.parse_args() - - with open(args.tmp_results, "rb") as fd: - tmp_results = pickle.load(fd) - - with open(args.dataset_meta, "rb") as fd: - dataset_meta = pickle.load(fd) - - print(args) - - metric = CocoMetric(metric=args.metric, use_faster=args.use_faster) - metric.dataset_meta = dataset_meta - metric.results = tmp_results - - ts = time.time() - metric.evaluate(len(tmp_results)) - te = time.time() - - print(f"Total evaluate: {te - ts:.2f}s") diff --git a/examples/comparison/mmdet/comparison.ipynb b/examples/comparison/mmdet/comparison.ipynb deleted file mode 100644 index 54120c8..0000000 --- a/examples/comparison/mmdet/comparison.ipynb +++ /dev/null @@ -1,589 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Install MMDetection" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Download COCO VAL" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "bc2a4389", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--2024-04-13 21:58:23-- http://images.cocodataset.org/annotations/annotations_trainval2017.zip\n", - "Resolving images.cocodataset.org (images.cocodataset.org)... 3.5.28.198, 3.5.25.88, 52.217.14.124, ...\n", - "Connecting to images.cocodataset.org (images.cocodataset.org)|3.5.28.198|:80... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 252907541 (241M) [application/zip]\n", - "Saving to: ‘COCO/DIR/annotations_trainval2017.zip’\n", - "\n", - "annotations_trainva 100%[===================>] 241.19M 14.9MB/s in 26s \n", - "\n", - "2024-04-13 21:58:50 (9.27 MB/s) - ‘COCO/DIR/annotations_trainval2017.zip’ saved [252907541/252907541]\n", - "\n", - "--2024-04-13 21:58:50-- http://images.cocodataset.org/zips/val2017.zip\n", - "Resolving images.cocodataset.org (images.cocodataset.org)... 3.5.29.24, 52.216.52.249, 3.5.29.27, ...\n", - "Connecting to images.cocodataset.org (images.cocodataset.org)|3.5.29.24|:80... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 815585330 (778M) [application/zip]\n", - "Saving to: ‘COCO/DIR/val2017.zip’\n", - "\n", - "val2017.zip 100%[===================>] 777.80M 13.2MB/s in 67s \n", - "\n", - "2024-04-13 21:59:58 (11.6 MB/s) - ‘COCO/DIR/val2017.zip’ saved [815585330/815585330]\n", - "\n" - ] - } - ], - "source": [ - "!wget -P COCO/DIR/ http://images.cocodataset.org/annotations/annotations_trainval2017.zip\n", - "\n", - "!wget -P COCO/DIR/ http://images.cocodataset.org/zips/val2017.zip" - ] - }, - { - "cell_type": "markdown", - "id": "1b94cb3d", - "metadata": {}, - "source": [ - "## Unzip COCO VAL" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "94d9a6c0", - "metadata": {}, - "outputs": [], - "source": [ - "!unzip -qq COCO/DIR/annotations_trainval2017.zip -d COCO/DIR/" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "c266f572", - "metadata": {}, - "outputs": [], - "source": [ - "!unzip -qq COCO/DIR/val2017.zip -d COCO/DIR/" - ] - }, - { - "cell_type": "markdown", - "id": "ee83ac7b", - "metadata": {}, - "source": [ - "## Download model" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "b6f6860f", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "config_file='/home/mixaill76/.local/lib/python3.10/site-packages/mmdet/.mim/configs/rtmdet/rtmdet-ins_tiny_8xb32-300e_coco.py'\n", - "model_file='https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet-ins_tiny_8xb32-300e_coco/rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth'\n", - "--2024-04-13 22:13:51-- https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet-ins_tiny_8xb32-300e_coco/rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth\n", - "Resolving download.openmmlab.com (download.openmmlab.com)... 104.166.182.22, 104.166.182.24, 104.166.182.27, ...\n", - "Connecting to download.openmmlab.com (download.openmmlab.com)|104.166.182.22|:443... connected.\n", - "HTTP request sent, awaiting response... 200 OK\n", - "Length: 22757492 (22M) [application/octet-stream]\n", - "Saving to: ‘model/rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth’\n", - "\n", - "rtmdet-ins_tiny_8xb 100%[===================>] 21.70M 6.14MB/s in 4.2s \n", - "\n", - "2024-04-13 22:13:57 (5.21 MB/s) - ‘model/rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth’ saved [22757492/22757492]\n", - "\n", - "total 22M\n", - "drwxrwxrwx 2 mixaill76 mixaill76 4.0K Apr 13 22:13 .\n", - "drwxr-xr-x 4 mixaill76 mixaill76 4.0K Apr 13 22:13 ..\n", - "-rw-r--r-- 1 mixaill76 mixaill76 16K Apr 13 22:13 rtmdet-ins_tiny_8xb32-300e_coco.py\n", - "-rw-r--r-- 1 mixaill76 mixaill76 22M Dec 19 2022 rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth\n" - ] - } - ], - "source": [ - "import mmdet\n", - "import mmengine\n", - "import os\n", - "import os.path as osp\n", - "\n", - "config_dir = osp.dirname(mmdet.__file__)\n", - "sub_config = \"configs/rtmdet/rtmdet-ins_tiny_8xb32-300e_coco.py\"\n", - "config_file = osp.join(config_dir, \".mim\", sub_config)\n", - "cfg = mmengine.Config.fromfile(config_file)\n", - "\n", - "model_file = \"https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet-ins_tiny_8xb32-300e_coco/rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth\"\n", - "\n", - "print(f\"{config_file=}\")\n", - "print(f\"{model_file=}\")\n", - "\n", - "!mkdir -p -m 777 model\n", - "\n", - "cfg.dump(osp.join(\"model\", osp.basename(config_file)))\n", - "!wget -P model/ {model_file}\n", - "\n", - "!ls -lah model" - ] - }, - { - "cell_type": "markdown", - "id": "313f1978", - "metadata": {}, - "source": [ - "## Validate" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "eebcff84", - "metadata": {}, - "outputs": [], - "source": [ - "from mmdet.apis import inference_detector, init_detector\n", - "from mmengine.registry import init_default_scope\n", - "from mmdet.datasets import CocoDataset\n", - "import tqdm\n", - "import copy\n", - "import pickle\n", - "from coco_metric import CocoMetric" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "c2f5fedd", - "metadata": {}, - "outputs": [], - "source": [ - "init_default_scope(\"mmdet\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "1318e70c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loads checkpoint by local backend from path: ./model/rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth\n" - ] - } - ], - "source": [ - "model = init_detector(\n", - " \"./model/rtmdet-ins_tiny_8xb32-300e_coco.py\",\n", - " \"./model/rtmdet-ins_tiny_8xb32-300e_coco_20221130_151727-ec670f7e.pth\",\n", - " device=\"cuda\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "1f7878d1", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "loading annotations into memory...\n", - "Done (t=0.45s)\n", - "creating index...\n", - "index created!\n" - ] - }, - { - "data": { - "text/plain": [ - "5000" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pipeline = [\n", - " dict(type=\"LoadImageFromFile\"),\n", - " dict(type=\"mmdet.LoadAnnotations\", with_bbox=True),\n", - " dict(\n", - " type=\"mmdet.PackDetInputs\",\n", - " meta_keys=(\n", - " \"img\",\n", - " \"img_id\",\n", - " \"img_path\",\n", - " \"ori_shape\",\n", - " \"img_shape\",\n", - " \"instances\",\n", - " ),\n", - " ),\n", - "]\n", - "\n", - "dataset = CocoDataset(\n", - " data_root=\"./COCO/DIR/\",\n", - " ann_file=\"annotations/instances_val2017.json\",\n", - " data_prefix=dict(img=\"val2017/\"),\n", - " pipeline=pipeline,\n", - ")\n", - "len(dataset)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "24d1832a", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 0%| | 0/500 [00:00