VCC-Note-Combined.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('blamed_commit', 714L), ('fixing_commit', 1137L), ('other_commit', 349558L)]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import psycopg2\n",
    "conn = psycopg2.connect(\"dbname=postgres host=localhost port=55432 user=postgres\")\n",
    "cur = conn.cursor()\n",
    "cur.execute(\"SELECT type, COUNT(id) FROM export.commits GROUP BY type\")\n",
    "cur.fetchall()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Started loading data\n",
      "Data loaded #840\n",
      "Start extracting lines...\n",
      "Completed extracting lines including #0 invalid patches.\n",
      "{0: 0.3086857293644672}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEZCAYAAACNebLAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FfW5x/HPEzZlCRBwYQlBUUFFRVTqVSoRrrLUaq+t\nZReVWmzRutTWggvBtV7rUrdWkaItAhbtFVyqWGuktuKOC4gLsoMLRAmCYITn/vGbJIc4SQ5JTk4S\nvu/Xa16Z5XdmnplzMs/M7zeLuTsiIiJlZaQ7ABERqZuUIEREJJYShIiIxFKCEBGRWEoQIiISSwlC\nRERiKUHspszsHTM7oZIy2WZWaGZWW3GlmpktM7P+Uf8kM/tLumOqjsT1EalpShB1jJktN7Mt0Y55\nnZlNM7PmNb0cd+/p7vMrKbPK3TM9BTfLRDvnr6P1LDCzF8zs2JpeThJ0I1ACM+tqZtvN7K4y43PM\nbIeZZZQZP83Mrk4Y3tfM7jOztWa20cwWR9/1nrsYR46Z/dPMNkfzGFBB2YvMbGm0vNVmdnNinGZ2\ntZm9ZWZFZnbVrsSxu1OCqHsc+J67ZwK9gaOBK+IKNoAj+1nRerYH8oHZ6Q1HgDOBAmComTUpM63C\nZGpmbYEXgWbAd9y9NXAS0BrototxzAReA7IIv/+HzaxdOWXnAEdHy+sJ9AJ+kTD9A+BXwOO7GMNu\nTwmibjIAd18H/J3wo8fMnjOza6Oj7c3AfmaWaWZToyO2VWZ2TWLiMLNzoyOwwqhaqVc0PrGq5Rgz\neyU6AltnZr+Lxu901GhmHcxsjpltMLP3zewnCcuZZGYPmdkD0bLeNrPeyaysu+8AHgQ6Ju4EzOwU\nM3vDzD6P1vmwhGmdzewRM/vUzD4zs9uj8fub2bNmtj6aNt3MMqv0JZSz/GgZGxK2ZcdoWSdEw2cl\nbPMPzeynCfPsF31PvzKzT8xsjZmdZmaDzey9KO4JZbbrbDObFc3vVTM7vJx4zcx+Ey3zs+gzbXZx\ntc8k7JCLgO/v4md/CRS6+2h3XwXg7mvc/RJ3fyfZmZjZgcCRQJ67b3P3vwFvAT+MK+/uy9z982iw\nEbADOCBh+l/c/Wngy11cn92eEkQdZmbZwBDg9YTRo4CfAK2AlcADwDZgf8I/1UnRdMzsDOAqYFR0\npH4qsCFmUb8HbouOwLoBf02YlnjU+FC0zH2BM4DrzSw3Yfr3gRmEI8bHgJ2qKSpYz6bAmCi2z6Nx\nRwJTgXMJR5H3AHPNrEmUsB4HlgFdgE7ArOLZAddHMR4MdAbykomjTEzlLt/dPwJ+DUy3UHUyDZiW\nUGX3CTAk2uZnA7cWJ5PIvkBToCMwCZgCjCR8fycAV5pZTkL5Uwnbvi3hyPpRM2sUE/YvorLfjeb9\nOXB3wjq9aWbDKljn71K6LWcTvpNdMQD4W0UFohgKou7zMn/vjIodCnzk7psTPvpmNL68+Q43s43A\nZ8DhhO9Lqsvd1dWhjrDTKySc5i8D7gCaRdOeIxxVFZfdG9haPD0aNwx4Nup/CrigguX0j/rzCTuq\ndmXK5ADbCQcS2YSjyuYJ068H/hT1TwLmJUw7GNhcwXpOIiS2AuAbwj/2CQnT7wYml/nMEsLO71jC\nTjgjie15GvBaOes9CfhzOZ8rd/kJw48SjmwXAk0qiOH/ir8HoB+wGbBouCXhiPfohPKvAqcmxPif\nhGkGrAWOj1mfxcCJCWU7AF8ns52i8lOAR6L+Y6Pvp33Z30KZz0wDro763wd+WgP/A6MS1zkad23x\nb62Sz3YDJgN7x0z7C3BVdePbnTqdQdRNp7l7lrvv5+4XuPu2hGmrEvpzgCbAuuIjMeCPwF7R9Gxg\naRLLGwt0B5aY2Utm9r2YMh2AAnffkjBuBeGIs9jHCf1bgD3MLMPMRpjZpqiK5ImEMg+5exYh0b1D\naG9JXLdfJh5tEs4GOkbrtcJD1dROzGxvM5tpobHyC2A6oY1jV1W0/GL3EY5q73D3ooQYBpvZi1E1\n1OfA4DIxbPBojwV8Ff39NGH6V4TEUazkO48+t7pMHIkx/19xzISEUQTsU9nKmtkehLPCGdFyFkTL\nHREV+Sb6W7Zdokm0DAhngB0qW1YSvgTKVgu2BjZV9kF3X0pY7z/UQBy7PSWIuqmixufEKp9VhDOI\ndlFCaevubdz98ITplTYOuvtSdx/h7nsB/0toECx71claIMvMWiSM6wKsSWL+M9y9lYcror6VfNy9\nABgH5JlZ8c5sFXBdtF7F69bS3R+KpnWxMlfURK4nHJEf6u5tCEejVWnMr2j5RNvhNkI1VF5xXX9U\nXfYwYTvu5e5tCe1I1bmgILu4x8yMkKjitvtKYHCZmFt4aMuqzP8Qdsp3W2iHWkdIQsXVTOsIiaBr\nmc/tRzhQAPhHNJ9yWWgHKyzTFR88FFeHLQL2L/NbOyIan4wmhCpXqSYliHrM3T8G5hHquFtFjZT7\nW+n9DfcBl1rUWGxm3aJ2jZ2Y2UgzKz7C3UhIQsVH58UN5quB/wA3mFmzqKF0LOG0vTxJ7xTd/X1C\nldhl0agpwHlm1ieKsYWZDYl2Gi8Tdli/NbPmUTzHRZ9rRTgC3WRmnQhXr1RFRcsHuB142d1/CjxJ\naZ1306hb7+47zGwwcHIVYyh2lJn9IGp3uJhwUPBSTLl7CO1CXaKY9zKzU5NcxhhCsjuMsDM+AugL\n9DKzQ6OztUeA68wsy8wam9lwQlXi36N53AJkWrhQoTiGThYuO+0JJZdXZ5bpig8efh6V+YBQbTcp\n+m5PJ1yo8Uhc4GY21sz2ivoPAX5DSFbF0xtHZ0gZQJNontr3JUEbqe6p6FLCuGlnEnZIiwn1+bMJ\njaC4+8PAdcAMMysk1IVnxcxrELAoKnMrMDShWiux3HDCEeNawj/rle7+XBXXJc7vgHPNrL27v0Zo\nIL4zqi55n+hoNtpZfR84kHDUvAr4cTSPycBRwBeEhvKyO5WkYqpo+dFO92Tg51HxS4AjzWy4u39J\naCyeHX1uGOEyzAoXV8nwHGAoodF5JHC6u2+PKfv7qOy8qMH2P0Cf4onR0fvwsgs3s45Af+BWd/80\noXudsPMvPosYT/iNvUVoA/o5oTH+MwAPVxIdRzjTeCmK4RnCd/FhJdugrGHAMdE6Xwf80N03RPH2\njX6rxY4H3jazTYSLFx4HLk+YPoVQ5TkMmBj1j9rFeHZLxQ1lIlIHmdkkoJu7n5nuWGT3ozMIERGJ\npQQhIiKxVMUkIiKxdAYhIiKxGqc7gGSZmU51RESqwN2rdB9OvTqDSPdt53WlmzRpUtpjqCudtoW2\nhbZFxV111KsEISIitUcJQkREYilB1EO5ubnpDqHO0LYopW1RStuiZtSby1zNzOtLrCIidYWZ4XWx\nkdrCm84+MbO3Kihzu5l9YGYLbeeXqoiISBqluoppGjCwvInRky67ufuBhMc9/zHF8YiISJJSmiDc\n/QWiV0iW4zTgz1HZl4DWCe8DEBGRNEr3jXKd2PkNaWuicZ/EFX7mmdoIKTV69YK99qq8nIhIXZHu\nBLFLxo3LK+lv2zaXrKzctMWyK5Ytg6FD4brr0h2JiDR0+fn55Ofn18i8Un4Vk5nlAI956WswE6f9\nEXjOS1/juATo5+7fOoOoz1cx3XADFBaGvyIitanOXsUUMcp/9eRcwhvRMLNjgS/ikoOIiNS+lFYx\nmdkMIBdoZ2YrgUmE12O6u9/r7k9G7/n9ENgMnJ3KeEREJHkpTRDuPiKJMuenMgYREakaPWpDRERi\nKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYml\nBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYS\nhIiIxFKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQ\nIiISSwlCRERipTxBmNkgM1tiZu+b2WUx0zPNbK6ZLTSzt83srFTHJCIilUtpgjCzDOBOYCBwKDDc\nzHqUKTYeWOTuvYATgZvNrHEq4xIRkcql+gyiD/CBu69w9yJgFnBamTIOtIr6WwEb3P2bFMclIiKV\nSHWC6ASsShheHY1LdCdwiJmtBd4ELkxxTCIikoS60Eg9EHjD3TsCRwJ3mVnLNMckIrLbS3Vd/xqg\nS8Jw52hcorOBGwDcfamZLQN6AK+WnVleXl5Jf25uLrm5uTUbrYhIPZefn09+fn6NzMvcvUZmFDtz\ns0bAe8AAYB3wMjDc3d9NKHMX8Km7TzazfQiJ4Qh3LygzL09lrKl0ww1QWBj+iojUJjPD3a0qn03p\nGYS7bzez84F5hOqsqe7+rpmNC5P9XuBa4H4zeyv62K/LJgcREal9Kb+c1N2fArqXGXdPQv86QjuE\niIjUIXWhkVpEROogJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERiKUGIiEgs\nJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGU\nIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKC\nEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmVdIIws05mdpyZnVDcJfm5QWa2xMzeN7PLyimTa2Zv\nmNk7ZvZcsjGJiEjqNE6mkJndCAwFFgPbo9EOzK/kcxnAncAAYC3wipnNcfclCWVaA3cBJ7v7GjNr\nv8trISIiNS6pBAH8AOju7tt2cf59gA/cfQWAmc0CTgOWJJQZATzi7msA3H39Li5DRERSINkqpo+A\nJlWYfydgVcLw6mhcooOALDN7zsxeMbPRVViOiIjUsGTPILYAC83sWaDkLMLdf1FDMfQG+gMtgBfN\n7EV3/7Bswby8vJL+3NxccnNza2DxIiINR35+Pvn5+TUyr2QTxNyo21VrgC4Jw52jcYlWA+vdfSuw\n1czmA0cAFSYIERH5trIHz5MnT67yvJJKEO7+gJk1JVQHAbzn7kVJfPQV4AAzywHWAcOA4WXKzAHu\nMLNGQDPgO8AtycQlIiKpk+xVTLnAA8BywIBsMxvj7hVexeTu283sfGAeob1jqru/a2bjwmS/192X\nmNnTwFuEK6TudffFVV4jERGpEclWMd1MuAz1PQAzOwiYCRxV2Qfd/Smge5lx95QZ/h3wuyRjERGR\nWpDsVUxNipMDgLu/T9WuahIRkXoi2TOIV83sPmB6NDwSeDU1IYmISF2QbIL4GTAeKL6s9V/A3SmJ\nSERE6oRkr2LaRriySFcXiYjsJipMEGb2V3f/sZm9TXj20k7c/fCURSYiImlV2RnEhdHfU1IdiIiI\n1C0VXsXk7uui3vXAquihe80IdzqvTXFsIiKSRsle5jof2MPMOhFuehsN3J+qoEREJP2STRDm7luA\n04G73f0M4NDUhSUiIumWdIIws/8i3P/wRDSuUWpCEhGRuiDZBHERMAH4P3dfZGb7A3o1qIhIA5bs\nfRDPA88nDH9E6U1zIiLSAFV2H8Rt7n6RmT1G/H0Qp6YsMhERSavKziD+Ev3Vk1ZFRHYzFSYId38t\n6n0V+MrddwAkvNxHREQaqGQbqZ8FmicM7wn8o+bDERGRuiLZBLGHu39ZPBD1N6+gvNSSjRthyBCY\nOTM18//Pf6B3b5g3LzXzT6X334dLL4XsbFi+PN3RiNQ/ySaIzWbWu3jAzI4CvkpNSJKsNWvgu9+F\nl16Cjz+u2Xlv3Ajjx8OPfgRbt8Jnn9Xs/FPl66/hr3+F/v3DtmncGJo0gS++SHdkIvXPrtwHMdvM\n/mVmLwAPAeenLiypzKJFcNxxMHIkjB5ds/N+9FHo2ROKisJyjjyyZuefCh99BBMmQJcu8Mc/wrhx\nsGoV/Pa3kJmZ7uhE6qdk74N4xcx6UPpu6ffcvSh1YUlF5s+HM86Am2+GUaPg4otrZr5r1sAFF4Sk\nMH069OtXM/NNlaIiePzxkBBefx3OPBOefx66d6/8syJSuaTOIMysOXAZcKG7vwN0NTM9AjwNZs8O\n1T7Tp4fkUBN27IA//AF69QpnDm++WbeTw8qVcNVV0LUr3HJLOINatSokTCUHkZqT7CtHpwGvAf8V\nDa8BZgOPpyIoiff738NNN4UG4169amaeixfDueeG/vx8OLSOPoJx+3b4+9/hnntCw/nIkWE71NV4\nRRqCZBNEN3cfambDAdx9i5lZCuOSBDt2wGWXheqUf/8bcnKqP89t2+D66+Huu+Hqq0OdfUayLVK1\naO1amDoVpkyBjh1DnA89BM11DZ1IyiWbIL42sz2JHrdhZt2AbSmLSkps2wZnnRWqUP79b8jKqv48\n58+Hn/4UDj4YFi6ETp2qP8+atGMHPPNMOFvIz4ehQ2Hu3Jo7axKR5CSbICYBTwHZZvYgcDxwVqqC\nkmDjRvif/4G2bcMOc889qze/L76AX/8annwS7rgjzLsu+eQTmDYtnC20bg3nnQcPPACtWqU7MpHd\nU6WVClFV0hLCy4LOAmYCR7t7fkoj280V3+Nw6KHhuv7qJAf30Lh96KHhvoBFi+pOcnAPZzRDh0KP\nHvDhhzBrFrz2WjjLUXIQSZ9KzyDc3c3sSXc/jNKXBUkKLVoU7o4ePx5+9SuoTmvPqlVhPkuXhkRz\n/PE1F2d1uMPTT8O118Knn8KFF8K994YzBxGpG5JtlnzdzI5JaSQChKPp/v1DA/Kvf1315LB9e6hG\nOvJIOProcJ9AXUgOO3aEG/GOOSY8BuP88+Hdd0MSU3IQqVuSbYP4DjDKzJYDmwEjnFwcnqrAdkez\nZ4cd5YwZ8N//XfX5vP12uHS1aVN44YVQdZNu27fDww/DddeFR19ccQWcdlrdvHJKRIJkE8TAlEYh\nNXKPw1dfwTXXhEbe66+HsWPTvwMuKgoJ7/rroV07uPFGGDSoetVmIlI7Knuj3B7AecABwNvAVHf/\npjYC213U1D0OCxaEexp69YK33oIOHWo2zl21bRvcf394FtL++4fHYeTmKjGI1CeVnUE8ABQB/wIG\nA4cAF6Y6qN1FTd3j0LRpuLv4rrvg1DS/BHbLlnAGc9NNcMQR8OCD4aGCIlL/VJYgDomuXsLMpgIv\npz6k3UNN3uNw5ZUwaVJ67y7etCmcwdx6a0gIc+bAUUelLx4Rqb7KaqhLntha1aolMxtkZkvM7H0z\nu6yCcseYWZGZnV6V5dQnNXmPA0DLlulLDp9/DpMnh2qkN9+Ef/wD/vY3JQeRhqCyBHGEmRVG3Sbg\n8OJ+MyusbOZmlgHcSWjkPhQYHj02PK7cb4Gnd30V6pfi9ziMGgW33w6NGqU7oqr57DOYOBEOOCC8\nre3f/w6N0T17pjsyEakpFVYxuXt1d199gA/cfQWAmc0CTiPcmZ3oAuBhoEHfa1H8HodbbglPI62P\n1q6F3/0uNEAPHRrueO7aNd1RiUgqpPoiyE7AqoTh1dG4EmbWEfiBu/+BcH9Fg/SPf4T3ODz4YP1M\nDitWwM9/Hs4Q3MO9Fn/4g5KDSEOW7H0QqXQb4WVExRpckmjcOLQ71OR7HGpTo0Zwww3hruclS2Dv\nvdMdkYjUhlQniDVAl4ThztG4REcDs6KHArYHBptZkbvPLTuzvLy8kv7c3Fxyc3NrOt6UGDcOzjkn\n3ChWH914I9x2W808alxEUis/P5/8/PwamZe5e43MKHbmZo2A94ABwDrCZbLD3f3dcspPAx5z97/F\nTPNUxioNV69eoc2kPp69iVSXmeHuVaqZSekZhLtvN7PzgXmE9o6p7v6umY0Lk/3esh9JZTwiIpK8\nlLdBuPtTQPcy4+4pp+w5qY5HRESSo2dpiohILCUIERGJpQQhIiKxlCBERCSWEoSIiMRSghARkVhK\nECIiEksJQkREYilBiIhILCUIERGJpQQhIiKxlCBERCSWEoSIiMRSghARkVhKECIiEksJQkREYilB\niIhILCUIERGJpQQhIiKxlCBERCSWEoSIiMRSghARkVhKECIiEksJQkREYilBiIhILCUIERGJpQQh\nIiKxlCBERCSWEoSIiMRSghARkVhKECIiEksJQkREYilBiIhILCUIERGJlfIEYWaDzGyJmb1vZpfF\nTB9hZm9G3QtmdliqYxIRkcqlNEGYWQZwJzAQOBQYbmY9yhT7CDjB3Y8ArgWmpDImERFJTqrPIPoA\nH7j7CncvAmYBpyUWcPcF7r4xGlwAdEpxTCIikoRUJ4hOwKqE4dVUnAB+Avw9pRGJiEhSGqc7gGJm\ndiJwNtC3vDJ5eXkl/bm5ueTm5qY8LhGR+iQ/P5/8/PwamZe5e43MKHbmZscCee4+KBr+DeDufmOZ\ncocDjwCD3H1pOfPyVMYqDVevXnD//eGvyO7GzHB3q8pnU13F9ApwgJnlmFlTYBgwN7GAmXUhJIfR\n5SUHERGpfSmtYnL37WZ2PjCPkIymuvu7ZjYuTPZ7gSuBLOBuMzOgyN37pDIuERGpXEqrmGqSqpik\nqlTFJLuzulzFJCIi9ZQShIiIxFKCEBGRWEoQIiISSwlCRERiKUGIVNFXX8HChfD446AL7KQhqjOP\n2hCpqzZvhiVLYPHi0m7RIlizBrp1g/ffhxUroEOHdEcqUrOUIEQihYXw7rs7J4LFi+GTT+Cgg+CQ\nQ0I3Zkz4260bNGkCHTvqDEIaJiUI2e18/nl8ItiwAXr0KE0E48aFv/vtB40apTtqkdpX7++k7tq1\nKytWrEhDRCLVl5OTw/Lly9MdhjRg1bmTut4niGjl0xCRSPXp9yuppkdtiIhIjVOCEBGRWEoQIiIS\nSwlCRERiKUE0ED179mT+/PkVllm1ahWZmZkNqlF0v/3245///CcAkydPZvTo0WmOSKThUIJIsa5d\nu9K8eXMyMzPp0KEDZ599Nlu2bKnx5bzzzjuccMIJFZbJzs6msLCQ8OK+mjV58mSaNm1KZmYmWVlZ\n9O3blwULFtT4ciqTinUT2V0pQaSYmfHEE09QWFjI66+/zquvvsq1114bW7a+H9kPGzaMwsJC1q9f\nT25uLmeccUa6Q6px27dvT3cIIrVGCaIWFO/4O3TowODBg3nnnXcAOPHEE7niiivo27cvLVq0YNmy\nZRQWFjJ27Fg6duxIdnY2V1555U6JY8qUKRxyyCFkZmbSs2dPFi5cCOxc1fLKK69wzDHH0Lp1azp0\n6MCll14KwIoVK8jIyGDHjh0ArFu3jtNOO4127dpx0EEHcd9995UsZ/LkyQwdOpQxY8aQmZnJYYcd\nxuuvv57U+mZkZDBy5EjWrl3Lhg0bSsY//vjjHHnkkbRt25a+ffvy9ttvl0xbvXo1P/zhD9l7773Z\na6+9+MUvfgHARx99xIABA2jfvj177703o0aNorCwcNe+gMicOXM48sgjad26NQceeCDz5s371rYr\nXvfiqqribfanP/2JnJwcBgwYwJAhQ7j77rt3mnevXr149NFHAViyZAknn3wy7dq14+CDD2b27NlV\nilck3ZQgatGqVat48skn6d27d8m46dOnc99997Fp0ya6dOnCmDFjaNasGR999BFvvPEGzzzzTMmO\ne/bs2Vx99dVMnz6dwsJC5s6dS7t27b61nAsvvJCLLrqIjRs3snTpUn784x+XTEusghk6dChdunTh\n448/Zvbs2UycOJH8/PyS6Y899hgjRoxg48aNfP/732f8+PFJrefXX3/NAw88QLt27Wjbti0Ab7zx\nBmPHjmXKlCkUFBQwbtw4Tj31VIqKitixYwennHIK++23HytXrmTNmjUMGzYMCMl14sSJfPzxx7z7\n7rusXr2avLy8pLd5sZdffpkxY8Zw8803s3HjRubPn0/Xrl3LLV+2qmr+/Pm89957PP300wwfPpwZ\nM2aUTFu8eDErV67klFNOYcuWLZx88smMGjWK9evXM2vWLMaPH8+SJUt2OWaRtHP3etGFUL+tvPE7\nl6l+V1Vdu3b1Vq1aedu2bb1r165+/vnn+9atW93dPTc31ydNmlRS9pNPPvFmzZqVTHd3nzlzpvfv\n39/d3QcOHOi33357uct59tln3d29X79+npeX5+vXr9+pzPLlyz0jI8O3b9/uK1eu9MaNG/vmzZtL\npk+YMMHPPvtsd3fPy8vzk046qWTa4sWLvXnz5uWuZ15enjdt2tTbtm3rjRo18vbt2/vzzz9fMv1n\nP/uZX3XVVTt9pnv37j5//nx/8cUXfe+99/bt27eXO/9ijz76qPfu3Tt2vfPy8nz06NGxnxs3bpxf\ncsklsdMS51F2PsXbbPny5SXTN23a5C1btvSVK1e6u/vll1/uY8eOdXf3hx56yE844YRvLfvqq6+O\nXXYyv1+R6oh+Y1Xa7+4WZxA1kSKqY86cORQUFLBs2TLuuOMOmjVrVjItOzu7pH/FihUUFRXRoUMH\nsrKyaNu2Leeddx6fffYZEM5AunXrVunypk6dynvvvUePHj34zne+wxNPPPGtMuvWrSMrK4vmzZuX\njMvJyWHNmjUlw/vuu29Jf/Pmzdm6dSs7duxgxowZtGrViszMTL73ve+VlBk6dCgFBQV8+umn9OzZ\nk1dffXWndbv55pvJysoqWbfVq1ezdu1aVq1aRU5ODhkZ3/45fvrppwwfPpzOnTvTpk2bkiPzXZXs\ntitP586dS/pbtmzJkCFDmDVrFgAzZ85k1KhRQFjPBQsW7LSeM2bM4OOPP67ysqtj61ZYty48nvyF\nF2DuXLj/frj1VrjyShg/Hi6/PC2hST2gp7nWAq8gwyRWZWRnZ7PHHnuwYcOG2KtxsrOzWbp0aaXL\n69atW0kVyCOPPMKPfvQjCgoKdirTsWNHCgoK2Lx5My1atABg5cqVdOrUqdL5jxgxghEjRpQ7PSsr\ni3vuuYejjz6akSNHss8++5Cdnc3ll1/OhAkTvlV+wYIFrFy5kh07dnwrSUycOJGMjAwWLVpE69at\nmTNnDhdccEGlMZZV0bZr0aLFTleWxe3My34fw4cPZ/LkyXz3u99l27Zt5ObmliwnNzeXp59+epdj\nLMsdtmxOPTT7AAAKJUlEQVSBbdsgIyM8hbagIHSJ/WWHE/u/+QayskLXtu23+7Oz4X//F667rvwY\nvvwSNm4M3Rdf7Fr/xo1wxBHwr39Ve3NIGihB1CH77rsvJ598MhdffDHXXHMNLVu2ZNmyZaxevZoT\nTjiBn/zkJ/zyl7/k+OOPp3fv3ixdupSmTZvudBYC8OCDDzJw4EDat29P69atMbOSHW9xsurcuTPH\nHXccEyZM4KabbuK9995j6tSpzJw5s9z4Kkp0ZR100EEMGjSIG2+8kVtuuYVzzz2X008/nQEDBtCn\nTx82b97M888/T79+/ejTpw8dOnTgN7/5DXl5eTRq1IjXXnuN4447jk2bNtGmTRtatWrFmjVruOmm\nm6qwZWHs2LEMHDiQU045hdzcXNatW8emTZvo3r07vXr1YtasWQwaNIiFCxfy8MMPM3jw4ArXe8iQ\nIZxzzjlcddVVDB06tGT8KaecwoQJE5g+fTrDhg3D3XnzzTdp2bIlPXr0iI3t0kt33tkXFIRHjxcU\nhMeMf/UVtGoF7drF7+j32QcOPrh0fOL05s2hoit/CwpgwgQ488z4HXxhITRrBm3aQOvWoUvsLx7u\n3Pnb09q0gY8/DvOW+kkJIsUqui4/btqf//xnLrvsMg455BC+/PJL9t9/fy677DKAkjOBESNGsHbt\nWrp27cpf/vIXsrOzd5rXU089xSWXXMJXX31FTk4ODz30UEm1VmK5mTNnMm7cODp27EhWVhbXXHMN\nJ554YpXWJc6ll17KgAEDmDhxIkcddRRTpkzh/PPP58MPP2TPPfekb9++9OvXj4yMDB577DEuuOAC\nunTpQkZGBiNGjOC4445j0qRJnHnmmbRp04YDDjiA0aNHc+utt+5yTMcccwzTpk3joosuYtmyZey7\n777cdddddO/enWuuuYbhw4eTlZVFv379GDly5E5nXHHLaNq0KaeffjrTpk3jhhtuKBnfsmVL5s2b\nx8UXX8wll1yCu3PEEUdwyy23lBvbPvuE904U79QTd/577hmO4lN1e0fbtjB1ajhDidvBZ2aGlyJV\nVQpu+ZFapMd9i6RRQ//9LlkCP/hB+CvpUZ3HfesMQkTqlKKicOaRbHfwwTBkSLqjbph0BiGSRg39\n97tkSdiB/+xn396xb94cv8N3hxYtQvtJZd2GDaGdo5LHkO3WdAYhInXS/vvD7bdD48bJ7fCbN9+1\nNo/58+GKK0qH3cNVX1u3hi6uv7LpZfv794fovs3djs4gRNJIv9/qeeUV6NMnNKZv3Rqqp5o1C90e\ne4Susv6Kpr/zDixfDtOnw9dfh8Tx9ddV6y9v2g9/CKeeGr9+O3aEMkVFpZ/b1f4zz9QZhIjsho45\nBtavD2cdzZpB06Y1e8XX889Dbi707Bnm3bRp6XKq0t+69c7DL70Eo0eHq8nidvLbt5eWbdo0rOeu\n9leHziBE0ki/391bUREsW1b+Dr5Ro+onvN26DSInJ0fvAJB6KycnJ90hSBo1aQIHHZTuKMqX8jMI\nMxsE3EZ4cuxUd78xpsztwGBgM3CWuy+MKRN7BiEiIuWrzhlESh/WZ2YZwJ3AQOBQYLiZ9ShTZjDQ\nzd0PBMYBf0xlTA1B4iO5d3faFqW0LUppW9SMVD/NtQ/wgbuvcPciYBZwWpkypwF/BnD3l4DWZrZP\niuOq1/TjL6VtUUrbopS2Rc1IdYLoBKxKGF4djauozJqYMiIiUst2i/dBiIjIrktpI7WZHQvkufug\naPg3hLcb3ZhQ5o/Ac+7+UDS8BOjn7p+UmZdaqEVEqqCuXub6CnCAmeUA64BhwPAyZeYC44GHooTy\nRdnkAFVfQRERqZqUJgh3325m5wPzKL3M9V0zGxcm+73u/qSZDTGzDwmXuZ6dyphERCQ59eZOahER\nqV11rpHazAaZ2RIze9/MLiunzO1m9oGZLTSzXrUdY22pbFuY2QgzezPqXjCzw9IRZ21I5ncRlTvG\nzIrM7PTajK82Jfk/kmtmb5jZO2b2XG3HWFuS+B/JNLO50b7ibTM7Kw1hppyZTTWzT8zsrQrK7Pp+\n093rTEdIWB8COUATYCHQo0yZwcATUf93gAXpjjuN2+JYoHXUP2h33hYJ5Z4FHgdOT3fcafxdtAYW\nAZ2i4fbpjjuN22ICcEPxdgA2AI3THXsKtkVfoBfwVjnTq7TfrGtnELqxrlSl28LdF7j7xmhwAQ33\n/pFkfhcAFwAPA5/WZnC1LJltMQJ4xN3XALj7+lqOsbYksy0caBX1twI2uPs3tRhjrXD3F4DPKyhS\npf1mXUsQurGuVDLbItFPgL+nNKL0qXRbmFlH4Afu/gegIV/xlszv4iAgy8yeM7NXzGx0rUVXu5LZ\nFncCh5jZWuBN4MJaiq2uqdJ+s94/zVXAzE4kXP3VN92xpNFtQGIddENOEpVpDPQG+gMtgBfN7EV3\n/zC9YaXFQOANd+9vZt2AZ8zscHf/Mt2B1Qd1LUGsAbokDHeOxpUtk11JmYYgmW2BmR0O3AsMcveK\nTjHrs2S2xdHALAvPfm8PDDazInefW0sx1pZktsVqYL27bwW2mtl84AhCfX1Dksy2OBu4AcDdl5rZ\nMqAH8GqtRFh3VGm/WdeqmEpurDOzpoQb68r+g88FzoSSO7Vjb6xrACrdFmbWBXgEGO3uS9MQY22p\ndFu4+/5Rtx+hHeLnDTA5QHL/I3OAvmbWyMyaExol363lOGtDMttiBfDfAFGd+0HAR7UaZe0xyj9z\nrtJ+s06dQbhurCuRzLYArgSygLujI+cid++TvqhTI8ltsdNHaj3IWpLk/8gSM3saeAvYDtzr7ovT\nGHZKJPm7uBa4P+Hyz1+7e0GaQk4ZM5sB5ALtzGwlMAloSjX3m7pRTkREYtW1KiYREakjlCBERCSW\nEoSIiMRSghARkVhKECIiEksJQkREYilBiETMbLuZvR49FnqOmWXW8PzHmNntUf8kM7ukJucvUtOU\nIERKbXb33u5+GOHJmOPTHZBIOilBiMR7kYSnXZrZpWb2cvSylUkJ48+MXtj0hpk9EI07xcwWmNlr\nZjbPzPZKQ/wi1VanHrUhkmYGYGaNgAHAfdHwScCB7t4neqTJXDPrCxQAE4H/cvfPzaxNNJ9/ufux\n0WfHEp4ye2ntropI9SlBiJTa08xeJzzpcjHwTDT+ZOCkaJoRHqF9YPR3dvFTdN39i6h8tpn9FehA\neNPZstpbBZGaoyomkVJb3L034RHSRmkbhBFeW9nb3Y9094PcfVoF87kDuN3dDwfOA/ZIadQiKaIE\nIVLKAKL3KFwIXGpmGcDTwDlm1gLC2+uidoV/AmeYWVY0vm00n0xgbdQ/phbjF6lRqmISKVXyaGN3\nX2hmbwLD3f1BMzuY8GY2gE3AKHdfbGbXAc+b2TfAG8A5wGTgYTMrICSRrrW8HiI1Qo/7FhGRWKpi\nEhGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxPp/MSLbSqlIj20AAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10bc76710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn import cross_validation\n",
    "import re\n",
    "import math\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import enum\n",
    "from datetime import datetime\n",
    "from diff_extractor import extract_lines, extract_added_lines, extract_removed_lines\n",
    "from unidiff.errors import UnidiffParseError\n",
    "\n",
    "class Colum(enum.IntEnum):\n",
    "    id = 1\n",
    "    type = 3\n",
    "    additions = 13\n",
    "    deletions = 14\n",
    "    past_different_authors = 18\n",
    "    future_different_authors = 19\n",
    "    author_contributions_percent = 20\n",
    "    patch = 21\n",
    "    hunk_count = 23\n",
    "    files_changed = 25\n",
    "    \n",
    "def fetch(filename, key = None):\n",
    "    npz = np.load(filename)\n",
    "    key = npz.files[0] if key is None else key\n",
    "    data = npz[key]\n",
    "    npz.close()\n",
    "    return data\n",
    "\n",
    "def logarithm(matched):\n",
    "    num = float(matched.group())\n",
    "    return str(0 if num == 0 else math.log(math.fabs(num)))\n",
    "\n",
    "print 'Started loading data'\n",
    "data = fetch('var/vcc_sample_40x800.npz')\n",
    "# data = fetch('var/vcc_data.npz')\n",
    "print 'Data loaded #%d' % len(data)\n",
    "\n",
    "patches = data[:, Colum.patch]\n",
    "labels = data[:, Colum.type]\n",
    "\n",
    "# Note: Be sure that unicode(patch, 'utf-8')\n",
    "print 'Start extracting lines...'\n",
    "cleansed_paches = [u''] * len(patches)\n",
    "invalid_paches = []\n",
    "\n",
    "# @TODO Replace all numeric token N by log(N) in each patch\n",
    "# Only need C/C++ file\n",
    "# Do not need comment line\n",
    "matcher = r'(\\d+)'\n",
    "for index, patch in enumerate(patches):\n",
    "    try:\n",
    "#         patch = re.sub(matcher, logarithm, patch)\n",
    "        cleansed_paches[index] = u' '.join(extract_lines(patch.splitlines()))\n",
    "    except UnidiffParseError as e:\n",
    "        # @todo Recover 445 patches at total in vcc_data.npz\n",
    "        invalid_paches.append((index, patch, e))\n",
    "\n",
    "print 'Completed extracting lines including #%d invalid patches.' % len(invalid_paches)\n",
    "\n",
    "# @TODO: stop_words=[]\n",
    "vectorizer = CountVectorizer(min_df=1, stop_words=None)\n",
    "vectorized = vectorizer.fit_transform(cleansed_paches) # csr_matrix\n",
    "# feature_names = vectorizer.get_feature_names()\n",
    "X = vectorized.toarray()\n",
    "\n",
    "# Now X is sparse array looks like:\n",
    "# [[0 0 0 ..., 0 0 0]\n",
    "#  [0 0 0 ..., 0 0 0]\n",
    "#  [0 0 0 ..., 0 0 0]\n",
    "#  ..., \n",
    "#  [0 0 0 ..., 0 0 0]\n",
    "#  [0 0 0 ..., 0 0 0]\n",
    "#  [0 0 0 ..., 0 0 0]]\n",
    "\n",
    "# Bind metrics from Git metadata\n",
    "target_metrics = map(lambda n:n-1, [\n",
    "    Colum.additions,\n",
    "    Colum.deletions,\n",
    "    Colum.past_different_authors,\n",
    "    Colum.future_different_authors,\n",
    "    Colum.author_contributions_percent,\n",
    "    Colum.hunk_count,\n",
    "    Colum.files_changed,\n",
    "])\n",
    "\n",
    "X2 = np.hstack(([row[target_metrics] for row in data], X))\n",
    "# Now combined X2 looks like\n",
    "# [[3L 66L 7L ..., 0L 0L 0L]\n",
    "#  [54L 23L 6L ..., 0L 0L 0L]\n",
    "#  [3L 1L 12L ..., 0L 0L 0L]\n",
    "#  ..., \n",
    "#  [1L 2L 1L ..., 0L 0L 0L]\n",
    "#  [46L 21L 0L ..., 0L 0L 0L]\n",
    "#  [32L 12L 42L ..., 0L 0L 0L]]\n",
    "\n",
    "y = is_vcc = (labels == 'blamed_commit')\n",
    "\n",
    "# Split into training and test\n",
    "X_train, X_test, y_train, y_test = train_test_split(X2, y, test_size=.5, random_state=0)\n",
    "\n",
    "# Run classifier\n",
    "weight = {0:1, 1:.01}\n",
    "classifier = LinearSVC(C=1.0, class_weight=weight)\n",
    "y_score = classifier.fit(X_train, y_train).decision_function(X_test)\n",
    "\n",
    "# Compute Precision-Recall and plot curve\n",
    "precision = dict()\n",
    "recall = dict()\n",
    "average_precision = dict()\n",
    "precision[0], recall[0], _ = precision_recall_curve(y_test, y_score)\n",
    "average_precision[0] = average_precision_score(y_test, y_score)\n",
    "\n",
    "print average_precision\n",
    "\n",
    "# Plot Precision-Recall curve\n",
    "plt.clf()\n",
    "plt.plot(recall[0], precision[0], label='Precision-Recall curve')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.title('Precision-Recall example: AUC={0:0.2f}'.format(average_precision[0]))\n",
    "plt.legend(loc=\"lower left\")\n",
    "plt.savefig(\"figure_%s\" % datetime.now().strftime('%s'))\n",
    "\n",
    "!osascript -e 'display notification \"Done\" with title \"VCC-Note\" sound name \"Purr\"'\n",
    "# !open .\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}