wash_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 瓶装酒瑕疵数据清洗"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-04T08:03:55.827262Z",
     "start_time": "2020-01-04T08:03:55.821949Z"
    }
   },
   "source": [
    "## 读入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-05T03:25:24.223888Z",
     "start_time": "2020-01-05T03:25:24.206121Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "所有图片的数量： 4516\n",
      "所有标注的数量： 6945\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "DATASET_PATH = './data/chongqing1_round1_train1_20191223'\n",
    "with open(os.path.join(DATASET_PATH, 'annotations.json')) as f:\n",
    "    json_file = json.load(f)\n",
    "    \n",
    "print('所有图片的数量：', len(json_file['images']))\n",
    "print('所有标注的数量：', len(json_file['annotations']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 找到和背景有关的图片"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-05T03:25:24.620425Z",
     "start_time": "2020-01-05T03:25:24.224974Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "所有标注中包含背景的图片数量： 1168\n",
      "只包含背景的图片数量： 1145\n"
     ]
    }
   ],
   "source": [
    "bg_imgs = set()  # 所有标注中包含背景的图片 id\n",
    "for c in json_file['annotations']:\n",
    "    if c['category_id'] == 0:\n",
    "        bg_imgs.add(c['image_id'])\n",
    "        \n",
    "print('所有标注中包含背景的图片数量：', len(bg_imgs))\n",
    "\n",
    "bg_only_imgs = set()  # 只有背景的图片的 id\n",
    "for img_id in bg_imgs:\n",
    "    co = 0\n",
    "    for c in json_file['annotations']:\n",
    "        if c['image_id'] == img_id:\n",
    "            co += 1\n",
    "    if co == 1:\n",
    "        bg_only_imgs.add(img_id)\n",
    "print('只包含背景的图片数量：', len(bg_only_imgs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 删除只有背景标注的图片"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-05T03:25:24.675543Z",
     "start_time": "2020-01-05T03:25:24.621480Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "待删除图片的数量： 1145\n",
      "处理之后图片的数量： 3371\n"
     ]
    }
   ],
   "source": [
    "images_to_be_deleted = []\n",
    "for img in json_file['images']:\n",
    "    if img['id'] in bg_only_imgs:\n",
    "        images_to_be_deleted.append(img)\n",
    "# 删除的是只有一个标注，且为 background 的的图片\n",
    "print('待删除图片的数量：', len(images_to_be_deleted))\n",
    "for img in images_to_be_deleted:\n",
    "    json_file['images'].remove(img)\n",
    "\n",
    "\n",
    "print('处理之后图片的数量：', len(json_file['images']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "注：这里有一张图片有两个 annotation，全是 background，所以删漏掉了。正确处理后处理之后数量为 3370。这个数据 coco api 会自动处理。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 删除所有关于背景的标注"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-05T03:25:24.754821Z",
     "start_time": "2020-01-05T03:25:24.676380Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "待删除标注的数量： 1170\n",
      "处理之后标注的数量： 5775\n"
     ]
    }
   ],
   "source": [
    "ann_to_be_deleted = []\n",
    "for c in json_file['annotations']:\n",
    "    if c['category_id'] == 0:\n",
    "        ann_to_be_deleted.append(c)\n",
    "        \n",
    "print('待删除标注的数量：', len(ann_to_be_deleted))\n",
    "for img in ann_to_be_deleted:\n",
    "    json_file['annotations'].remove(img)\n",
    "\n",
    "print('处理之后标注的数量：', len(json_file['annotations']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 删除 categories 中关于背景的部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-05T03:25:24.761006Z",
     "start_time": "2020-01-05T03:25:24.755626Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'supercategory': '瓶盖破损', 'id': 1, 'name': '瓶盖破损'},\n",
       " {'supercategory': '喷码正常', 'id': 9, 'name': '喷码正常'},\n",
       " {'supercategory': '瓶盖断点', 'id': 5, 'name': '瓶盖断点'},\n",
       " {'supercategory': '瓶盖坏边', 'id': 3, 'name': '瓶盖坏边'},\n",
       " {'supercategory': '瓶盖打旋', 'id': 4, 'name': '瓶盖打旋'},\n",
       " {'supercategory': '瓶盖变形', 'id': 2, 'name': '瓶盖变形'},\n",
       " {'supercategory': '标贴气泡', 'id': 8, 'name': '标贴气泡'},\n",
       " {'supercategory': '标贴歪斜', 'id': 6, 'name': '标贴歪斜'},\n",
       " {'supercategory': '喷码异常', 'id': 10, 'name': '喷码异常'},\n",
       " {'supercategory': '标贴起皱', 'id': 7, 'name': '标贴起皱'}]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bg_cate = {'supercategory': '背景', 'id': 0, 'name': '背景'}\n",
    "json_file['categories'].remove(bg_cate)\n",
    "json_file['categories']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 标注的 id 有重复的，这里重新标号"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-05T03:25:24.766281Z",
     "start_time": "2020-01-05T03:25:24.761793Z"
    }
   },
   "outputs": [],
   "source": [
    "for idx in range(len(json_file['annotations'])):\n",
    "    json_file['annotations'][idx]['id'] = idx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导出数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-05T03:25:24.882283Z",
     "start_time": "2020-01-05T03:25:24.767336Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(os.path.join(DATASET_PATH, 'annotations_washed.json'), 'w') as f:\n",
    "    json.dump(json_file, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "open-mmlab",
   "language": "python",
   "name": "open-mmlab"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}