evaluation.py

import numpy as np
import progressbar
import config
import logging
import json
import os

from tabulate import tabulate
from config import args
from pycocotools.cocoeval import COCOeval

log = logging.getLogger()


class Evaluation(object):
    def __init__(self, detector, loader, iou_thresh=0.5):
        self.detector = detector
        self.loader = loader
        self.gt = {}
        self.dets = {}
        self.iou_thresh = iou_thresh

    def evaluate_network(self, ckpt):
        filenames = self.loader.get_filenames()

        self.gt = {cid: {} for cid in range(1, self.loader.num_classes)}
        self.dets = {cid: [] for cid in range(1, self.loader.num_classes)}

        bar = progressbar.ProgressBar()
        for i in bar(range(len(filenames))):
            self.process_image(filenames[i], i)

        table = self.make_detection_table() if args.detect else None
        iou = self.compute_mean_iou() if args.segment else None

        return self.compact_results(table, ckpt, iou)

    def compute_ap(self):
        """computes average precision for all categories"""
        aps = {}
        for cid in range(1, self.loader.num_classes):
            cat_name = self.loader.ids_to_cats[cid]
            rec, prec = self.eval_category(cid)
            ap = voc_ap(rec, prec, self.loader.year == '07')
            aps[self.loader.ids_to_cats[cid]] = ap
        return aps

    def make_detection_table(self):
        """creates a table with AP per category and mean AP"""
        aps = self.compute_ap()
        eval_cache = [aps]

        table = []
        for cid in range(1, self.loader.num_classes):
            cat_name = self.loader.ids_to_cats[cid]
            table.append((cat_name, ) + tuple(aps.get(cat_name, 'N/A') for aps in eval_cache))
        mean_ap = np.mean([a for a in list(aps.values()) if a >= 0])
        table.append(("AVERAGE", ) + tuple(np.mean(list(aps.values())) for aps in eval_cache))
        x = tabulate(table, headers=(["Category", "mAP (all)"]),
                     tablefmt='orgtbl', floatfmt=".3f")
        log.info("Eval results:\n%s", x)
        return table

    def compact_results(self, table, ckpt, iou=None):
        """compresses the table for concise metrics representation
        during batch evaluation"""
        out = [str(ckpt)]
        if table:
            maps = table[-1][1:]
            out += ['%.3f%%' % maps[0]]
        if iou:
            out += [' %.3f%%' % iou + ' mIoU']
        s = '\t'.join(out) + '\n'
        return s

    def compute_mean_iou(self):
        iou = self.detector.get_mean_iou()
        print(iou)
        log.info("\n Mean IoU is %f", iou)
        return iou

    def process_image(self, name, img_id):
        img = self.loader.load_image(name)
        gt_bboxes, seg_gt, gt_cats, w, h, difficulty = self.loader.read_annotations(name)

        for cid in np.unique(gt_cats):
            mask = (gt_cats == cid)
            bbox = gt_bboxes[mask]
            diff = difficulty[mask]
            det = np.zeros(len(diff), dtype=np.bool)
            self.gt[cid][img_id] = {'bbox': bbox, 'difficult': diff, 'det': det}

        output = self.detector.feed_forward(img, seg_gt, w, h, name,
                                            gt_bboxes, gt_cats,
                                            img_id < args.save_first_n)

        if args.detect:
            det_bboxes, det_probs, det_cats = output[:3]
            for i in range(len(det_cats)):
                self.dets[det_cats[i]].append((img_id, det_probs[i]) + tuple(det_bboxes[i]))

    def eval_category(self, cid):
        """Computes average precision for one category"""
        cgt = self.gt[cid]
        cdets = np.array(self.dets[cid])
        if (cdets.shape == (0, )):
            return None, None
        scores = cdets[:, 1]
        sorted_inds = np.argsort(-scores)
        image_ids = cdets[sorted_inds, 0].astype(int)
        BB = cdets[sorted_inds]

        npos = 0
        for img_gt in cgt.values():
            img_gt['ignored'] = np.array(img_gt['difficult'])
            img_gt['det'] = np.zeros(len(img_gt['difficult']), dtype=np.bool)
            npos += np.sum(~img_gt['ignored'])

        nd = len(image_ids)
        tp = np.zeros(nd)
        fp = np.zeros(nd)
        for d in range(nd):
            ovmax = -np.inf
            if image_ids[d] in cgt:
                R = cgt[image_ids[d]]
                bb = BB[d, 2:].astype(float)

                BBGT = R['bbox'].astype(float)

                # compute overlaps
                # intersection
                ixmin = np.maximum(BBGT[:, 0], bb[0])
                iymin = np.maximum(BBGT[:, 1], bb[1])
                ixmax = np.minimum(BBGT[:, 0] + BBGT[:, 2], bb[0] + bb[2])
                iymax = np.minimum(BBGT[:, 1] + BBGT[:, 3], bb[1] + bb[3])
                iw = np.maximum(ixmax - ixmin, 0.)
                ih = np.maximum(iymax - iymin, 0.)
                inters = iw * ih

                # union
                uni = (bb[2] * bb[3] + BBGT[:, 2] * BBGT[:, 3] - inters)

                overlaps = inters / uni
                ovmax = np.max(overlaps)
                jmax = np.argmax(overlaps)

            if ovmax > self.iou_thresh:
                if not R['ignored'][jmax]:
                    if not R['det'][jmax]:
                        tp[d] = 1.
                        R['det'][jmax] = True
                    else:
                        fp[d] = 1.
            else:
                fp[d] = 1.

        # compute precision recall
        fp = np.cumsum(fp)
        tp = np.cumsum(tp)
        rec = tp / float(npos)
        N = float(npos)
        # avoid divide by zero in case the first detection matches a difficult
        # ground truth
        prec = rec * N / np.maximum(rec * N + fp, np.finfo(np.float32).eps)
        return rec, prec

    def reset(self):
        self.gt = {}
        self.dets = {}


class COCOEval(Evaluation):
    def __init__(self, net, loader):
        super().__init__(net, loader)

    def process_image(self, img_id, number):
        img = self.loader.load_image(img_id)
        gt_bboxes, gt_cats, w, h, _ = self.loader.read_annotations(img_id)
        seg_gt = self.loader.get_semantic_segmentation(img_id) if args.segment else None
        out = self.detector.feed_forward(img, seg_gt, w, h, img_id,
                                         gt_bboxes, gt_cats, False)
        detections = []
        if args.detect:
            det_bboxes, det_probs, det_cats = out[:3]
            for j in range(len(det_cats)):
                obj = {}
                obj['bbox'] = list(map(float, det_bboxes[j]))
                obj['score'] = float(det_probs[j])
                obj['image_id'] = img_id
                obj['category_id'] = self.loader.ids_to_coco_ids[det_cats[j]]
                detections.append(obj)
        return detections

    def compute_ap(self):
        coco_res = self.loader.coco.loadRes(self.filename)

        cocoEval = COCOeval(self.loader.coco, coco_res)
        cocoEval.params.imgIds = self.loader.get_filenames()
        cocoEval.params.useSegm = False

        cocoEval.evaluate()
        cocoEval.accumulate()
        cocoEval.summarize()
        return cocoEval

    def compact_results(self, stats, ckpt):
        out = [str(ckpt)]
        metrics = ['@0.5:0.95', '@0.5', '@0.75', 'small', 'medium', 'large']
        out += ['%.4f%%(%s)' % (val, desc) for val, desc in zip(stats, metrics)]
        s = '\t'.join(out) + '\n'
        return s

    def evaluate_network(self, ckpt):
        path = config.EVAL_DIR + '/Data/'
        self.filename = path + 'coco_%s_%s_%i.json' % (self.loader.split, args.run_name, ckpt)
        detections = []
        filenames = self.loader.get_filenames()

        bar = progressbar.ProgressBar()
        for i in bar(range(len(filenames))):
            img_id = filenames[i]
            detections.extend(self.process_image(img_id, i))
        with open(self.filename, 'w') as f:
            json.dump(detections, f)
        if args.segment:
            iou = self.compute_mean_iou()
        cocoEval = self.compute_ap()
        return self.compact_results(cocoEval.stats, ckpt)


def voc_ap(rec, prec, use_07_metric=False):
    """ ap = voc_ap(rec, prec, [use_07_metric])
    Compute VOC AP given precision and recall.
    If use_07_metric is true, uses the
    VOC 07 11 point method (default:False).
    """
    if use_07_metric:
        # 11 point metric
        ap = 0.
        for t in np.arange(0., 1.1, 0.1):
            p = 0 if np.sum(rec >= t) == 0 else np.max(prec[rec >= t])
            ap = ap + p / 11.
    else:
        # correct AP calculation
        # first append sentinel values at the end
        mrec = np.concatenate(([0.], rec, [1.]))
        mpre = np.concatenate(([0.], prec, [0.]))

        # compute the precision envelope
        for i in range(mpre.size - 1, 0, -1):
            mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])

        # to calculate area under PR curve, look for points
        # where X axis (recall) changes value
        i = np.where(mrec[1:] != mrec[:-1])[0]

        # and sum (\Delta recall) * prec
        ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
    return ap