notebooks/experiments/AIGenomics_Model_Quickstart.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## In this notebook:\n",
    "\n",
    "* Final Selected Model: RF \n",
    "* Feature set 5 \n",
    "* Data: chromosomes 2, 3, 4 \n",
    "* labels: Pathogenic, Benign\n",
    "* requires: /pickled/chr2-4_featureset5.pkl \n",
    "* note: this is the same notebook as 41.5.2_4.RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of entire loaded dataset (13820, 14)\n",
      "TOTAL SUM OF LABELS: \n",
      " 1    9863\n",
      "0    3957\n",
      "Name: Problematic, dtype: int64\n",
      "PERCENT OF LABELS: \n",
      " 1    0.713676\n",
      "0    0.286324\n",
      "Name: Problematic, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import plot_roc_curve\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "## pull in pickled dataframe:\n",
    "df = pd.read_pickle(\"./pickled/chr2-4_featureset5.pkl\")\n",
    "\n",
    "print(\"Shape of entire loaded dataset\", df.shape)\n",
    "\n",
    "## Print out distribution of labels\n",
    "print(\"TOTAL SUM OF LABELS: \\n\", df['Problematic'].value_counts())\n",
    "print(\"PERCENT OF LABELS: \\n\", df['Problematic'].value_counts(normalize=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of training dataset (10365, 14)\n",
      "Shape of testing dataset (3413, 14)\n",
      "TOTAL SUM OF LABELS: \n",
      " 1    7393\n",
      "0    2972\n",
      "Name: Problematic, dtype: int64\n",
      "PERCENT OF LABELS: \n",
      " 1    0.713266\n",
      "0    0.286734\n",
      "Name: Problematic, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "## Split into train/test\n",
    "data_copy = df.copy()\n",
    "df_train = data_copy.sample(frac=0.75, random_state=1)\n",
    "df_test = data_copy.drop(df_train.index)\n",
    "\n",
    "\n",
    "print(\"Shape of training dataset\", df_train.shape)\n",
    "#print ('Training data subset\\n')\n",
    "#print (df_train.head(3)) \n",
    "\n",
    "print(\"Shape of testing dataset\", df_test.shape)\n",
    "#print ('\\nTest data subset\\n')\n",
    "#print (df_test.head(3))\n",
    "\n",
    "\n",
    "## Get labels:\n",
    "train_labels = df_train.pop('Problematic')\n",
    "test_labels = df_test.pop('Problematic')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fit the Random Forest model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final Selected Model: Random forest\n",
    "\n",
    "forest = RandomForestClassifier(n_estimators=1000, random_state=42, class_weight='balanced')\n",
    "forest.fit(df_train, train_labels)\n",
    "ax = plt.gca()\n",
    "rfc_disp = plot_roc_curve(forest, df_test, test_labels, ax=ax, alpha=0.8)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate the models preformance on unseen test data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXxU9bn48c8zk5UQiCTsW5BNNgkQUbQitGjRCnitVrho3SrVurRu17Z6W3/UXrUuvRekl1pt3SqoVIVWeq0KiqIiQfYdASFhC0sCIevMPL8/zmScQJYJZGaSmef9euXFnHO+55znTMh5zvl+z/l+RVUxxhgTv1zRDsAYY0x0WSIwxpg4Z4nAGGPinCUCY4yJc5YIjDEmziVEO4DGysrK0uzs7GiHYYwxLcqKFSsOqmr72pa1uESQnZ1NXl5etMMwxpgWRUS+rmuZVQ0ZY0ycs0RgjDFxzhKBMcbEOUsExhgT5ywRGGNMnAtbIhCRP4vIARFZV8dyEZEZIrJNRNaIyPBwxWKMMaZu4bwjeAEYX8/yS4G+/p9pwP+GMRZjjDF1CNt7BKq6RESy6ykyCXhJnX6wPxeRDBHprKp7wxWTMcaoKsG972vwfOBoWRU+BeWbcj7/Ogr4fOov75TxKRSVVqL+bfjUKePToGlVKjw+Sis9lFV68fhOrfv/c3u1o2/H9FM99DpF84WyrsDuoOl8/7yTEoGITMO5a6BHjx4RCc4Y0zCfT9l/rJwqj1Lu8XKopJJEtwBQVFpFQVEZVV4fHp/i9SlVXh9lVV4qqnyUVXqp8Hip8jrzSyu9VHp935yZcU60QOAk7HzWoOWctJwTljfHIVdETm29zLSkmEsEIVPVZ4FnAXJzc5vhr9WY5qPS46OkwkOlx0dBURkVVd7AVSkEX906/1ZfuSrOle2uw6U15vuCrmrVP6+00svh45UcOFZRbyxul5DkduF2CQluIcElpCa5SUlwk5Lk5ozkJJLcLhLcQqukBJITTq6tFv9ZU6h5ApUTl8s38wIzTlhPgkp8M6/m8lbJbhLdLlz+mS6p3razA5dIYJ3qzx6fkp6SQGqSG5cIruplIrhFEIHkBBepSW5SE90kuJvXczrRTAQFQPeg6W7+ecbEhSqvj6NlVXhV8fnAq4rXq+wpLqOsyouq4vVVn4Cdk/GeojKSE1yBK2yPT6nyn/grPD6OV3jYtO/YacWV0SqRtKQEXC7n5OdyOSe74JNbcqKLQV3acGFaEm63i57tWpHodqGqZKUnA9AqyU2Xtqm4XKd4+WsiJpqJYAFwh4jMBc4Fiq19wLR01Sdsj8/HsXIPhccqAtUey7YfZsPeYo4cr6LC41yln6pEt5Dg+uZKu01KIskJLhLdLsYP7kTb1ES6ZKTQNjWRzLTkwNWtCIETe/AVbfWVrluEjFaJgatsEx/ClghEZA4wBsgSkXzg10AigKrOBhYClwHbgFLgxnDFYsypOHK8kiOlzk95lVPP7fH6yD9SRqLbhdfno8Lj46vCEtYVHA15uyN6nkHv9mkkJbhom5qI2+XC7cJfpeD8ZLZOIqNVYtC8b5anpyTYVbZpUuF8amhKA8sVuD1c+zemIR6vj91Hytiy/xh7i8rYvL8EgNJKDweOVlBS4al3fZdAgttFRmoiI3qeAUDfjq1JcAlul4tWSW46tkkmOcFNUoKLTm1TaJOSGPbjMqaxWkRjsTGNVeX1ceBYBYXHKlhXUEyS20WVz8cXOw6z63AprZMTKCqtqrFOaqKb5EQXfTukk52ZBkD/Tul0bJNMotvFGa2SSPQ3fGamJdlVuYkZlghMi7d6dxGr84vYfbiU/Ucr2HHweIPrdDujFRf1SyM5wUXv9q3p06E1HdqkRCBaY5ofSwSmRajweFmy5SBLtx3k8PFKSiu9gLL/6DePL3Zvl0qnNql0b5eK1wfZma3okdmK1EQ3g7q0JdEt1ghqTC0sEZhmw+tT1hYUs66gmAPHKjhaVkWV18fWAyWUVXoD5VwCHduk0KdDOmd3y6DS4+Oac7rTvV2rKEZvTMtlicBEnMfro7Ckgk+3HeKVZV/TNtV5OqbwhJeT+nZoTYJb6NuhNV6fMnFoF87p1Y7EZvYyjjEtnSUCExaVHh+LNu2ntNLLgWMV7C0qY//RCgqKyk4q2zo5gT7+k71LhIk5Xejhf0HJGBN+lghMkykqrWTu8t28t2E/lR7fSct7tGtFh/Rk+nRoTb+O6aQmuRk3oCNJtXQrYIyJHEsE5pQUlVayruAoh45XsDa/mC93HaHK+82rskO6teXcXu0YN6AjiW6XNdQa04xZIjD1Ol7hYW9xOV/uOsL2wuPsPlzKrsOlNcqkJrpp63+p6rIhnemVlWYnfWNaEEsE5iRllV72FJfx/Cc7WJtfXGNZ3w6t6dGuFQO7tGHcgI50PSOVVolue7nKmBbMEoEBnJP/exv3s3p3EV/sOFxj2Y8u7EXv9q3p3DaFzNbJUYrQGBMulgjiWHmVly+/PsKbKwvYHNR1cWqim3EDOzBuQEe629M7xsQ8SwRxpLTSw8FjlRSVVbLzUCl/WrK9xvJpo89k3ICOpCa5oxShMSYaLBHEOJ9PWVNQzFP/2nxSJ2sAE4Z25oejsklJtJO/MfHKEkGMqfL6WL7jMDsPlXK0vIp31tQc6+cnY3rTNjWRnllptGuVZFf/xhhLBLFi496jvL2ygJW7iiir+qZfnhE9z6BjmxSuzu1GZlqSPdZpjDmJJYIWzudTfr1gPat2FwEwfnAnzjuzHTndz8Btj3QaY0JgiaCFOnC0nL8u28WiTQcC8x4Yfxbf6psVxaiMMS2RJYIWqKi0kgf+toaDJZUkuoVLB3fmxguySbDHPI0xp8ASQQuzencRD729DnCqgW4f2yfKERljWjpLBC1IeZU3kASuO68nE3O6RDkiY0wssETQAlR6fCz96iDPfey8ANY1I5UfnNM9ylEZY2KFJYJm7FBJBR9tKeTd9fvYU1QOQEqii/+enBPlyIwxscQSQTO0fk8xr3z+NesKjgbmjenfnlsv6k1asv3KjDFNy84qzcj2whKeWbSNrQdKAKcd4Jxe7cjObGUvghljwsYSQTOwbPshnvtkB/uKneqfQV3aMG30mZzZvnWUIzPGxANLBFF0vMLDbxduDAz+4nIJj105hAGd20Q5MmNMPLFEECWffnWQRxduwiVw8cCOTBnZg/bpNuiLMSbyLBFEwYGj5Ty6cBMAj1wxhCHd2kY5ImNMPLM+CaJg8Wanf6AfjuppScAYE3V2RxBBPp9y7fPLOFbuAeDcXplRjsgYY8J8RyAi40Vks4hsE5Gf17K8h4gsFpGVIrJGRC4LZzzRUlrpYe4Xu5g0a2kgCfz35Bx6ZLaKcmTGGBPGOwIRcQOzgIuBfGC5iCxQ1Q1BxR4CXlfV/xWRgcBCIDtcMUVShcfLQ2+tY1PQoPAA2VlpPHX1UJISrFbOGNM8hLNqaCSwTVW3A4jIXGASEJwIFKh+VrItsCeM8UTUwwvWB5LAWZ3SmTyyOwM7t7WhIY0xzU44E0FXYHfQdD5w7gllHgb+JSJ3AmnAuNo2JCLTgGkAPXr0aPJAm9qry3YFuoeYf/sFuGykMGNMMxbt+okpwAuq2g24DHhZRE6KSVWfVdVcVc1t3759xINsjGXbDzHni10AvHzzSEsCxphmL5yJoAAI7iu5m39esJuB1wFU9TMgBWixYy1u2X+MR97ZCMBTPxhKRqukKEdkjDENC2ciWA70FZFeIpIETAYWnFBmF/AdABEZgJMICsMYU9gs3nSAe19fDcBD3xtAv47pUY7IGGNCE7Y2AlX1iMgdwLuAG/izqq4XkelAnqouAO4F/iQid+M0HN+gqhqumMLpXxv2AzBzyjCys9KiHI0xxoQurC+UqepCnEdCg+f9KujzBuCCcMYQCTsOHmddQTFDurW1JGCMaXHszeLT9OaX+fxl6U4ALurXvBuyjTGmNpYITsMHG/czd7nzhOz/TM6hl90NGGNaIEsEp2jTvqP89/tbAbj74r42iIwxpsWyRHAKSis93P/GGgBmXzeCrhmpUY7IGGNOXbRfKGuR3vM/IdS5bYolAWNMi2eJ4BRs2e/0IfT0NTlRjsQYY06fJYJG8vmUtQVHyWydRJp1IGeMiQGWCBpp+8ESjhyv5OKBHRGxfoSMMS2fNRY3wuJNB3j6vS0AdEhPiXI0xhjTNEK6IxCRJBHpE+5gmrvqbiT+Y3x/vnNWhyhHY4wxTaPBRCAi3wPWAu/5p3NE5K1wB9bcrCsoZl1BMSN6nsGFfdtb99LGmJgRyh3BdJwBZYoAVHUVEFd3B+VVXh5esB6Am7/VK8rRGGNM0wolEVSpatEJ81pkD6GnasYHW6nw+Lj2vB50b2cDzhtjYksojcUbReQHgEtEegF3AZ+HN6zmo7isio+3HgTg6hHdGyhtjDEtTyh3BHcAIwAf8CZQAfw0nEE1J0u3OUngRxf2snYBY0xMCuWO4Luq+gDwQPUMEbkSJynEvAqPF4AL+rTYETSNMaZeodwRPFTLvAebOpDm6vOvDpOS6KJNSmK0QzHGmLCo845ARL4LjAe6isjTQYva4FQTxbzyKi8b9h7lsiGdSUqwl7CNMbGpvqqhA8A6oBxYHzT/GPDzcAbVXGze53Qu1yvLnhQyxsSuOhOBqq4EVorIX1W1PIIxNRsHjlUAMKhL2yhHYowx4RNKY3FXEfktMBAIdLCjqv3CFlUzkbfzMG6X0C4tKdqhGGNM2IRS8f0C8BdAgEuB14HXwhhTs6CqfPrVIUb3zSIt2frmM8bErlASQStVfRdAVb9S1YdwEkJMeyMvH8DGIjbGxLxQLnUrRMQFfCUitwIFQHp4w4qu/COlvPz51wBc2NfeHzDGxLZQEsHdQBpO1xK/BdoCN4UzqGg7WuYB4IHxZ5HZOjnK0RhjTHg1mAhUdZn/4zHgOgAR6RrOoKKtsMR5Wigt2YaiNMbEvnrbCETkHBG5QkSy/NODROQlYFl967V0ry3fBUBmmt0NGGNiX52JQEQeBf4KTAX+T0QeBhYDq4GYfXR016FSdh8uY1TvTHpk2otkxpjYV1/V0CRgqKqWiUg7YDcwRFW3Rya06Hj6vc0AjOnXPsqRGGNMZNRXNVSuqmUAqnoY2BLrSQDAJUKvrDTOt95GjTFxor5EcKaIvOn/eQvoFTQdUhfUIjJeRDaLyDYRqbV/IhH5gYhsEJH1IvLqqRxEU9p6oMTeJDbGxJX6qoa+f8L0M43ZsIi4gVnAxUA+sFxEFqjqhqAyfYFfABeo6hER6dCYfTSlgyUVPP7PTcA3YxAYY0w8qK/TuQ9Oc9sjgW3V1UkiMhen3WFDUJlbgFmqesS/zwOnuc9T9uqyXWzad4zxgzvZAPXGmLgSzk72u+I0MFfL988L1g/oJyJLReRzERlf24ZEZJqI5IlIXmFhYViCLa300jUjldvH9iEl0d4fMMbEj2iPtpIA9AXGAFOAP4lIxomFVPVZVc1V1dz27cPzNM/GvUdplWQJwBgTf0JOBCLS2LerCoDuQdPd/POC5QMLVLVKVXcAW3ASQ0Qt3nSAw8cr+Zb1K2SMiUMNJgIRGSkia4Gt/umhIjIzhG0vB/qKSC8RSQImAwtOKPM2zt0A/reX+wERf0R1xddHSE1yc0VOTPecYYwxtQrljmAGcDlwCEBVVwNjG1pJVT3AHcC7wEbgdVVdLyLTRWSiv9i7wCER2YDz1vL9qnqo8Ydx6o6VV/HRlkIGdm6DyyWR3LUxxjQLofQ+6lLVr0VqnCRDer5SVRcCC0+Y96ugzwrc4/+JioKiMgB6d7BxB4wx8SmURLBbREYC6n834E6cuvyYUOVRAAZ2jukhFowxpk6hVA3dhnPF3gPYD5znnxcT3lrptF+3b53SQEljjIlNodwReFR1ctgjiZI9RWWM6HmG9TRqjIlbodwRLBeRhSJyvYjEZP2JvT9gjIlnDSYCVe0NPAKMANaKyNsiErN3CMYYE29CeqFMVT9V1buA4cBRnAFrjDHGxIBQXihrLSJTReTvwBdAIXB+2CMzxhgTEaE0Fq8D/g78TlU/DnM8xhhjIiyURHCmqvrCHkmUKBrtEIwxJqrqTAQi8pSq3gv8TUROOluq6pVhjSwCikor2VtczoV9bXxiY0z8qu+O4DX/v40amawlOVbuQRV6tLN3CIwx8au+Ecq+8H8coKo1koGI3AGc7ghmxhhjmoFQHh+9qZZ5Nzd1IMYYY6KjvjaCa3DGEOglIm8GLUoHisIdWCTsPlwa7RCMMSbq6msj+AJnDIJuwKyg+ceAleEMKlJe/WIXAAO7tIlyJMYYEz31tRHsAHYA70cunMhKTnDTKyuNrNaNHYXTGGNiR31VQx+p6kUicgRqPGwvOGPKtAt7dGEmAhmtEqMdhjHGRFV9VUPVw1HaiO7GGBPD6nxqKOht4u6AW1W9wCjgx0BaBGILq/1Hyyk4UkZacigvVxtjTOwK5fHRt3GGqewN/AXoC7wa1qgi4C9Ld1JS4eH7w7tFOxRjjImqUBKBT1WrgCuBmap6N9A1vGGF3+rdRfTt0Jo+Nmi9MSbOhZIIPCJyNXAd8A//vBbdwurx+iip8FBUVhXtUIwxJupCfbN4LE431NtFpBcwJ7xhhZfP/wzUJQM7RjcQY4xpBhpsKVXVdSJyF9BHRM4Ctqnqb8MfWvgcLKkAoG1qi76xMcaYJtFgIhCRC4GXgQKcdwg6ich1qro03MGFy/bC4wD075Qe5UiMMSb6Qnl28vfAZaq6AUBEBuAkhtxwBhZOPnXqhhLdIQ3ZbIwxMS2UM2FSdRIAUNWNQFL4QjLGGBNJodwRfCkis4FX/NNTiZFO54wxxoSWCG4F7gL+wz/9MTAzbBFFwJHSSgAS3BLlSIwxJvrqTQQiMgToDbylqr+LTEjht3V/CZmtk+jUJiXaoRhjTNTV2UYgIr/E6V5iKvCeiNQ2UlmL5FUlOcGFiN0RGGNMfY3FU4GzVfVq4BzgtsZuXETGi8hmEdkmIj+vp9z3RURFJCJPIm07UEK3M2zAemOMgfoTQYWqHgdQ1cIGyp5ERNw4I5tdCgwEpojIwFrKpQM/BZY1Zvun4/DxSrpmpEZqd8YY06zV10ZwZtBYxQL0Dh67WFWvbGDbI3HeQt4OICJzgUnAhhPK/QZ4HLi/MYGfLqsVMsYYR32J4PsnTD/TyG13BXYHTecD5wYXEJHhQHdVfUdE6kwEIjINmAbQo0ePRoZhjDGmPvWNWfxBOHcsIi7gaeCGhsqq6rPAswC5ubnaQPEGqZ72JowxJmaEs4+FApzRzap188+rlg4MBj4UkZ3AecCCcDcYHzleSZVXaZdmL0cbYwyENxEsB/qKSC8RSQImAwuqF6pqsapmqWq2qmYDnwMTVTUvjDGx+0gpAD0zW/xom8YY0yRCTgQiktyYDauqB7gDeBfYCLyuqutFZLqITGxcmE3H4x+MIDnBOpwzxhgIrRvqkcDzQFugh4gMBX6kqnc2tK6qLgQWnjDvV3WUHRNKwKfreIUHsERgjDHVQjkbzgAuBw4BqOpqnBHLWqSdB4/jEuyFMmOM8QslEbhU9esT5nnDEUwkHKvw0DolgSS7IzDGGCC03kd3+6uH1P+28J3AlvCGFT6qINjbZMYYUy2Uy+LbgHuAHsB+nMc8G93vUHNReKzCHh01xpggoQxefwDn0c+YUFBURt8OraMdhjHGNBuhPDX0J+CkV3FVdVpYIgqzCo+PVknuaIdhjDHNRihtBO8HfU4B/o2afQgZY4xpwUKpGnoteFpEXgY+CVtExhhjIupUnqHsBXRs6kAixTqcM8aYmkJpIzjCN20ELuAwUOdoY83ZgaPlFJVW0dHGKjbGmICGBq8XYCjf9Brq0xZ8Sf3x1oMAXNi3fZQjMcaY5qPeqiH/SX+hqnr9Py02CQB8su0gfTu0plNbuyMwxphqobQRrBKRYWGPJMwOlVSw7UAJF/TJinYoxhjTrNRZNSQiCf6upIcBy0XkK+A4zvjFqqrDIxRjkyj3+ABo19reKjbGmGD1tRF8AQwHojZ2gDHGmPCrLxEIgKp+FaFYjDHGREF9iaC9iNxT10JVfToM8YRNeZXTc3ZKgnUvYYwxwepLBG6gNcRGn80l5c7IZOkpofSqYYwx8aO+s+JeVZ0esUjCrHqISutwzhhjaqrv8dGYuBOoVuJPBK3tjsAYY2qoLxF8J2JRREBppdNGkJZkicAYY4LVmQhU9XAkAwk3r895KdrtiqkbHWOMOW02grsxxsQ5SwTGGBPnLBEYY0ycs0RgjDFxzhKBMcbEOUsExhgT5ywRGGNMnLNEYIwxcS6siUBExovIZhHZJiInDXgvIveIyAYRWSMiH4hIz3DGY4wx5mRhSwQi4gZmAZcCA4EpIjLwhGIrgVxVPRuYB/wuXPEYY4ypXTjvCEYC21R1u6pWAnOBScEFVHWxqpb6Jz8HuoUxHmOMMbUIZyLoCuwOms73z6vLzcA/a1sgItNEJE9E8goLC5swRGOMMc2isVhErgVygSdqW66qz6pqrqrmtm/fPrLBGWNMjAtnn8wFQPeg6W7+eTWIyDjgQeAiVa0IYzzGGGNqEc47guVAXxHpJSJJwGRgQXABERkG/BGYqKoHwhiLMcaYOoQtEaiqB7gDeBfYCLyuqutFZLqITPQXewJnXOQ3RGSViCyoY3PGGGPCJKzDdanqQmDhCfN+FfR5XDj3b4wxpmHNorHYGGNM9FgiMMaYOGeJwBhj4pwlAmOMiXOWCIwxJs5ZIjDGmDhnicAYY+KcJQJjjIlzlgiMMSbOWSIwxpg4Z4nAGGPinCUCY4yJc5YIjDEmzlkiMMaYOGeJwBhj4pwlAmOMiXOWCIwxJs5ZIjDGmDhnicAYY+KcJQJjjIlzlgiMMSbOJUQ7ABO7qqqqyM/Pp7y8PNqhGBM3UlJS6NatG4mJiSGvY4nAhE1+fj7p6elkZ2cjItEOx5iYp6ocOnSI/Px8evXqFfJ6VjVkwqa8vJzMzExLAsZEiIiQmZnZ6LtwSwQmrCwJGBNZp/I3Z4nAGGPinCUCE9Pcbjc5OTkMHjyYCRMmUFRU1CTb3blzJ4MHD26Sbd1www306tWLnJwccnJymDFjRpNstzYffvghn376aY15L730EoMHD2bIkCEMGzaMJ598MhDXvHnzmmS/e/bs4aqrrgpMT5kyhbPPPpvf//73/OpXv+L9998/re2//fbbTJ8+vca8nJwcJk+eXGPemDFjyMvLC0yf+Hv84osvGD16NP3792fYsGH86Ec/orS09LRi27FjB+eeey59+vThmmuuobKy8qQylZWV3HjjjQwZMoShQ4fy4YcfBpaNHz+eoUOHMmjQIG699Va8Xi8A9913H4sWLTqt2AJUtUX9jBgxQk/FvLzdevmMj7Ws0nNK65vG27BhQ7RD0LS0tMDnH/7wh/rII480yXZ37NihgwYNapJtXX/99frGG2+c0roeT+P+P//617/WJ554IjC9cOFCHTZsmBYUFKiqanl5uT777LOnHVd99u7dq7179z7l9auqqk6aN2rUKC0sLAxMb9iwQQcPHqxdunTRkpKSwPyLLrpIly9fHpgO/j3u27dPe/TooZ9++mlg+RtvvKH79u075VhVVa+++mqdM2eOqqr++Mc/1j/84Q8nlXnmmWf0hhtuUFXV/fv36/Dhw9Xr9aqqanFxsaqq+nw+vfLKKwPb2rlzp1588cW17rO2vz0gT+s4r9pTQyYi/rRkO9sPljTpNs/Mas0to88MufyoUaNYs2YNACUlJUyaNIkjR45QVVXFI488wqRJk9i5cyeXXnop3/rWt/j000/p2rUr8+fPJzU1lRUrVnDTTTcBcMkllwS2W15ezm233UZeXh4JCQk8/fTTjB07lhdeeIG3336b48ePs3XrVu677z4qKyt5+eWXSU5OZuHChbRr167OeOfMmcN//dd/oap873vf4/HHHwegdevW/PjHP+b9999n1qxZpKamcs8991BSUkJWVhYvvPACnTt3ZsaMGcyePZuEhAQGDhzIY489xuzZs3G73bzyyivMnDmTRx99lCeffJIuXboAkJyczC233HJSLNOnT+fvf/87ZWVlnH/++fzxj39ERE7ax9y5c/noo4/46U9/Cjj11UuWLOHQoUNcfvnlrFu3jksuuYSCggJycnKYOXMmzz//PJdffjlXXXUVK1asqPVYxowZQ05ODp988glTpkzh3nvvDcS2ZcsWkpOTycrKqvHdXXfddWzcuJH58+fz7//+7w3+/5g1axbXX389o0aNCswLvos5FarKokWLePXVVwG4/vrrefjhh7nttttqlNuwYQPf/va3AejQoQMZGRnk5eUxcuRI2rRpA4DH46GysjLQBtCzZ08OHTrEvn376NSp02nFaVVDJi54vV4++OADJk6cCDjPWr/11lt8+eWXLF68mHvvvRfnogm2bt3K7bffzvr168nIyOBvf/sbADfeeCMzZ85k9erVNbY9a9YsRIS1a9cyZ84crr/++sBTG+vWrePNN99k+fLlPPjgg7Rq1YqVK1cyatQoXnrppcA27r///kDV0Nq1a9mzZw8PPPAAixYtYtWqVSxfvpy3334bgOPHj3PuueeyevVqzj33XO68807mzZsXSFQPPvggAI899hgrV65kzZo1zJ49m+zsbG699VbuvvtuVq1axYUXXsi6desYMWJEg9/fHXfcwfLly1m3bh1lZWX84x//qHUfAE8++SSzZs1i1apVfPzxx6SmptbY1oIFC+jdu3cghmpVVVV1Hgs41Sd5eXk1kgDA0qVLGT58eI15r732GpMnT2bKlCnMmTOnweMDQv4uNm/eHPhdnfhzYtXjoUOHyMjIICHBuebu1q0bBQUFJ21z6NChLFiwAI/Hw44dO1ixYgW7d+8OLP/ud79Lhw4dSE9Pr5Gchg8fztKlS0M6vvrYHYGJiMZcuTelsrIycnJyKCgoYMCAAVx88cWAc6X2y1/+kiVLluByuSgoKGD//v0Agfp6gBEjRrBz506KioooKipi9OjRAFx33XX885//BOCTTz7hzjvvBOCss86iZ8+ebNmyBZZL/q0AAA44SURBVICxY8eSnp5Oeno6bdu2ZcKECQAMGTIkcHcC8MQTT9T4A58/fz5jxoyhffv2AEydOpUlS5ZwxRVX4Ha7+f73vw84J6V169YFjsvr9dK5c2cAzj77bKZOncoVV1zBFVdccVrf4+LFi/nd735HaWkphw8fZtCgQUyYMKHWfVxwwQXcc889TJ06lSuvvJJu3bqFtI/6jgXgmmuuqXW9vXv3Br4ngLy8PLKysujRowddu3blpptu4vDhw7Rr167WJ2oa+5RN//79WbVqVaPWachNN93Exo0byc3NpWfPnpx//vm43e7A8nfffZfy8nKmTp3KokWLAt9Rhw4d2LNnz2nvP6x3BCIyXkQ2i8g2Efl5LcuTReQ1//JlIpIdznhM/ElNTWXVqlV8/fXXqCqzZs0C4K9//SuFhYWsWLGCVatW0bFjx8BVfHJycmB9t9uNx+M55f0Hb8vlcgWmXS7XKW83JSUlcJJQVQYNGsSqVatYtWoVa9eu5V//+hcA77zzDrfffjtffvkl55xzTq37GzRoECtWrKh3f+Xl5fzkJz9h3rx5rF27lltuuSXwXdW2j5///Oc899xzlJWVccEFF7Bp06aQjqu+YwFIS0urdb3U1NQaz83PmTOHTZs2kZ2dTe/evTl69Gjgri4zM5MjR44Eyh4+fDhQpRTKdwGNuyPIzMykqKgo8N3n5+fTtWvXk7aZkJDA73//e1atWsX8+fMpKiqiX79+NcqkpKQwadIk5s+fH5hXXl5+0h3XqQhbIhARNzALuBQYCEwRkYEnFLsZOKKqfYDfA4+HKx4T31q1asWMGTN46qmn8Hg8FBcX06FDBxITE1m8eDFff/11vetnZGSQkZHBJ598AjiJpNqFF14YmN6yZQu7du2if//+pxXvyJEj+eijjzh48CBer5c5c+Zw0UUXnVSuf//+FBYW8tlnnwFO9cr69evx+Xzs3r2bsWPH8vjjj1NcXExJSQnp6ekcO3YssP4vfvEL7r//fvbt2wc41S/PPfdcjX1Un2SzsrIoKSkJPElU1z6++uorhgwZwgMPPMA555wTciKo61gaMmDAALZt2xaI6fXXX2ft2rXs3LmTnTt3Mn/+/ED10JgxY3jllVcC1YAvvvgiY8eOBZzqrxdffJFly5YFtv3mm28G7hSD46xOVif+ZGRk1CgrIowdOzbwnb344otMmjTppGMoLS3l+PHjALz33nuBNpeSkhL27t0LOG0E77zzDmeddVZgvS1btjTJ02vhrBoaCWxT1e0AIjIXmARsCCozCXjY/3ke8IyIiFb/loxpQsOGDePss89mzpw5TJ06lQkTJjBkyBByc3Nr/HHV5S9/+Qs33XQTIlKjsfgnP/kJt912G0OGDCEhIYEXXnihxp3AqejcuTOPPfYYY8eODTQW13YCSUpKYt68edx1110UFxfj8Xj42c9+Rr9+/bj22mspLi5GVbnrrrvIyMhgwoQJXHXVVcyfP5+ZM2dy2WWXsX//fsaNG4eqIiKBBvFqGRkZ3HLLLQwePJhOnTpxzjnnAE7VTW37+M///E8WL16My+Vi0KBBXHrppYGTWX3qOpZBgwbVu97o0aMDbTwff/wxXbt2DTR+Vy/fsGEDe/fuZdq0aWzatImhQ4ciIuTm5vLoo48C0LFjR+bOnct9993HgQMHcLlcjB49mvHjxzcYe30ef/xxJk+ezEMPPcSwYcO4+eabAaetJC8vj+nTp3PgwAG++93v4nK56Nq1Ky+//DLgtAdNnDiRiooKfD4fY8eO5dZbbwWcRLlt2zZyc3NPKz4ACdc5V0SuAsar6o/809cB56rqHUFl1vnL5Punv/KXOXjCtqYB0wB69OgxoqGrt9p8vv0Qizcf4N6L+5OUYG3kkbBx40YGDBgQ7TBMHPjpT3/KhAkTGDduXLRDiZjqhx1+85vfnLSstr89EVmhqrVmjRZxRlTVZ1U1V1VzgxuFGuO8MzP5xaUDLAkYE4N++ctfnvaLXy2Nx+M56QmqUxXOqqECoHvQdDf/vNrK5ItIAtAWOBTGmIwxMahjx46BR4PjxdVXX91k2wrn5fFyoK+I9BKRJGAysOCEMguA6/2frwIWWftAbLFfpzGRdSp/c2FLBKrqAe4A3gU2Aq+r6noRmS4i1an7eSBTRLYB9wAnPWJqWq6UlBQOHTpkycCYCFH/eAQpKSmNWi9sjcXhkpubq8GdRpnmy0YoMyby6hqhrL7GYnuz2IRNYmJio0ZJMsZEhz1CY4wxcc4SgTHGxDlLBMYYE+daXGOxiBQCjX+12JEFHGywVGyxY44Pdszx4XSOuaeq1vpGbotLBKdDRPLqajWPVXbM8cGOOT6E65itasgYY+KcJQJjjIlz8ZYIno12AFFgxxwf7JjjQ1iOOa7aCIwxxpws3u4IjDHGnMASgTHGxLmYTAQiMl5ENovINhE5qUdTEUkWkdf8y5eJSHbko2xaIRzzPSKyQUTWiMgHItIzGnE2pYaOOajc90VERaTFP2oYyjGLyA/8v+v1IvJqpGNsaiH83+4hIotFZKX///dl0YizqYjIn0XkgH8Ex9qWi4jM8H8fa0Rk+GnvVFVj6gdwA18BZwJJwGpg4AllfgLM9n+eDLwW7bgjcMxjgVb+z7fFwzH7y6UDS4DPgdxoxx2B33NfYCVwhn+6Q7TjjsAxPwvc5v88ENgZ7bhP85hHA8OBdXUsvwz4JyDAecCy091nLN4RjAS2qep2Va0E5gInjvo9CXjR/3ke8B0RkQjG2NQaPGZVXayq1WP5fY4zYlxLFsrvGeA3wONALPSFHcox3wLMUtUjAKp6IMIxNrVQjlmBNv7PbYE9EYyvyanqEuBwPUUmAS+p43MgQ0Q6n84+YzERdAV2B03n++fVWkadAXSKgcyIRBceoRxzsJtxrihasgaP2X/L3F1V34lkYGEUyu+5H9BPRJaKyOciMj5i0YVHKMf8MHCtiOQDC4E7IxNa1DT2771BNh5BnBGRa4Fc4KJoxxJOIuICngZuiHIokZaAUz00Bueub4mIDFHVoqhGFV5TgBdU9SkRGQW8LCKDVdUX7cBaili8IygAugdNd/PPq7WMiCTg3E4eikh04RHKMSMi44AHgYmqWhGh2MKloWNOBwYDH4rITpy61AUtvME4lN9zPrBAVatUdQewBScxtFShHPPNwOsAqvoZkILTOVusCunvvTFiMREsB/qKSC8RScJpDF5wQpkFwPX+z1cBi9TfCtNCNXjMIjIM+CNOEmjp9cbQwDGrarGqZqlqtqpm47SLTFTVljzOaSj/t9/GuRtARLJwqoq2RzLIJhbKMe8CvgMgIgNwEkFhRKOMrAXAD/1PD50HFKvq3tPZYMxVDamqR0TuAN7FeeLgz6q6XkSmA3mqugB4Huf2cRtOo8zk6EV8+kI85ieA1sAb/nbxXao6MWpBn6YQjzmmhHjM7wKXiMgGwAvcr6ot9m43xGO+F/iTiNyN03B8Q0u+sBOROTjJPMvf7vFrIBFAVWfjtINcBmwDSoEbT3ufLfj7MsYY0wRisWrIGGNMI1giMMaYOGeJwBhj4pwlAmOMiXOWCIwxJs5ZIjDNjoh4RWRV0E92PWWz6+qlsZH7/NDfw+Vqf/cM/U9hG7eKyA/9n28QkS5By54TkYFNHOdyEckJYZ2fiUir0923iV2WCExzVKaqOUE/OyO036mqOhSnQ8InGruyqs5W1Zf8kzcAXYKW/UhVNzRJlN/E+QdCi/NngCUCUydLBKZF8F/5fywiX/p/zq+lzCAR+cJ/F7FGRPr6518bNP+PIuJuYHdLgD7+db/j7+d+rb+f+GT//Mfkm/EdnvTPe1hE7hORq3D6c/qrf5+p/iv5XP9dQ+Dk7b9zeOYU4/yMoM7GROR/RSRPnHEI/p9/3l04CWmxiCz2z7tERD7zf49viEjrBvZjYpwlAtMcpQZVC73ln3cAuFhVhwPXADNqWe9W4H9UNQfnRJzv73LgGuAC/3wvMLWB/U8A1opICvACcI2qDsF5E/82EckE/g0YpKpnA48Er6yq84A8nCv3HFUtC1r8N/+61a4B5p5inONxupSo9qCq5gJnAxeJyNmqOgOnW+axqjrW3+3EQ8A4/3eZB9zTwH5MjIu5LiZMTCjznwyDJQLP+OvEvTh96JzoM+BBEekGvKmqW0XkO8AIYLm/a41UnKRSm7+KSBmwE6cr4/7ADlXd4l/+InA78AzO+AbPi8g/gH+EemCqWigi2/19xGwFzgKW+rfbmDiTcLoMCf6efiAi03D+rjvjDNKy5oR1z/PPX+rfTxLO92bimCUC01LcDewHhuLcyZ400Iyqvioiy4DvAQtF5Mc4ozi9qKq/CGEfU4M7pRORdrUV8vd/MxKno7OrgDuAbzfiWOYCPwA2AW+pqopzVg45TmAFTvvATOBKEekF3Aeco6pHROQFnM7XTiTAe6o6pRHxmhhnVUOmpWgL7PX3MX8dTgdkNYjImcB2f3XIfJwqkg+Aq0Skg79MOwl9vObNQLaI9PFPXwd85K9Tb6uqC3ES1NBa1j2G0xV2bd7CGWVqCk5SoLFx+jtV+0/gPBE5C2eEruNAsYh0BC6tI5bPgQuqj0lE0kSktrsrE0csEZiW4g/A9SKyGqc65XgtZX4ArBORVThjEbzkf1LnIeBfIrIGeA+n2qRBqlqO07PjGyKyFvABs3FOqv/wb+8Taq9jfwGYXd1YfMJ2jwAbgZ6q+oV/XqPj9Lc9PIXTw+hqnLGKNwGv4lQ3VXsW+D8RWayqhThPNM3x7+cznO/TxDHrfdQYY+Kc3REYY0ycs0RgjDFxzhKBMcbEOUsExhgT5ywRGGNMnLNEYIwxcc4SgTHGxLn/Dz3OJ/s/nSPSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix: \n",
      " [[ 710  266]\n",
      " [ 179 2258]]\n",
      "Accuracy: 0.8696161734544389\n",
      "Recall TP/(TP+FN): [0.72745902 0.92654904]\n",
      "Precision TP/(TP+FP): [0.79865017 0.89461173]\n",
      "F1: [0.7613941  0.91030034]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "## ROC\n",
    "ax = plt.gca()\n",
    "rfc_disp = plot_roc_curve(forest, df_test, test_labels, ax=ax, alpha=0.8)\n",
    "plt.show()\n",
    "\n",
    "## metrics:\n",
    "y_true = test_labels.to_numpy()\n",
    "y_pred = forest.predict(df_test)\n",
    "\n",
    "# Confusion Matrix\n",
    "from sklearn.metrics import confusion_matrix\n",
    "print(\"Confusion matrix: \\n\", confusion_matrix(y_true, y_pred))\n",
    "\n",
    "# Accuracy\n",
    "from sklearn.metrics import accuracy_score\n",
    "print(\"Accuracy:\", accuracy_score(y_true, y_pred))\n",
    "\n",
    "# Recall\n",
    "from sklearn.metrics import recall_score\n",
    "recall = recall_score(y_true, y_pred, average=None)\n",
    "print(\"Recall TP/(TP+FN):\", recall)\n",
    "\n",
    "# Precision\n",
    "from sklearn.metrics import precision_score\n",
    "precision = precision_score(y_true, y_pred, average=None)\n",
    "print(\"Precision TP/(TP+FP):\",precision)\n",
    "\n",
    "\n",
    "# F1 score\n",
    "F1 = 2 * (precision * recall) / (precision + recall)\n",
    "print(\"F1:\", F1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inspecting the feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature ranking:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAGpCAYAAACDCv0RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5hkZZn+8e8NSBDJjIGc/YkIKAPG1VEkKkEFCaKgKIY17aqYUVndBXfXXXVBAUExooDCKKOI4qBEGXIQdBiQoJJBVEAH7t8f7ymmpunu6Zk6p6vr1P25rrqm6pyq57zdU/3UW2+UbSIior2W6ncBIiKiWUn0EREtl0QfEdFySfQRES2XRB8R0XJJ9BERLZdEH0NN0pclfbzf5YhokjKOPpaEpJuApwCPdB3ezPYfeog5A/im7XV6K91gkvQ14FbbH+t3WaJdUqOPXuxm+0ldtyVO8nWQtEw/r98LSUv3uwzRXkn0UTtJz5N0vqT7JF1R1dQ7594o6TeSHpA0T9Jbq+MrAj8G1pL0l+q2lqSvSfp01+tnSLq16/FNkj4o6Urgr5KWqV53qqQ7Jd0o6d3jlPWx+J3Ykg6VdIekP0raU9Kukn4r6R5JH+l67SclnSLpu9XPc6mkrbrOP0PS7Or3cI2k3Udc90uSZkn6K3Aw8Drg0Opn/2H1vA9JuqGKf62kV3XFOEjSuZL+S9K91c+6S9f51SV9VdIfqvOndZ17paTLq7KdL2nLrnMflHRbdc3rJW0/gf/2mMps55bbYt+Am4CXj3J8beBuYFdKRWKH6vG06vwrgI0BAS8B/gY8pzo3g9J00R3va8Cnux4v9JyqHJcD6wIrVNe8BDgMWBbYCJgH7DTGz/FY/Cr2/Oq1TwDeAtwJfBtYCXgm8CCwYfX8TwL/APaqnv9+4Mbq/hOAucBHqnK8DHgAeHrXde8HXliVefmRP2v1vL2Btarn7AP8FXhade6g6vpvAZYG3g78gQVNsmcA3wVWq8rzkur4s4E7gOdWrzuw+j0uBzwduAVYq3ruBsDG/X6/5dbbLTX66MVpVY3wvq7a4gHALNuzbD9q+yxgDiXxY/sM2ze4OAf4KfBPPZbjC7Zvsf0gsC3lQ+Vw23+3PQ84Dth3grH+AXzG9j+Ak4A1gc/bfsD2NcC1wFZdz7/E9inV8z9HSdjPq25PAo6oynE28CNgv67Xnm77vOr39NBohbF9su0/VM/5LvA7YLuup/ze9nG2HwFOBJ4GPEXS04BdgLfZvtf2P6rfN8AhwDG2L7L9iO0TgYerMj9CSfibS3qC7Zts3zDB311MUUn00Ys9ba9a3fasjq0P7N31AXAf8CJKAkLSLpIurJpB7qN8AKzZYzlu6bq/PqX5p/v6H6F0HE/E3VXShFJ7B7i96/yDlAT+uGvbfhS4lVIDXwu4pTrW8XvKN57Ryj0qSW/oamK5D9iChX9ff+q6/t+qu0+ifMO5x/a9o4RdH3jfiN/RupRa/FzgvZRvK3dIOknSWosqZ0xtSfRRt1uAb3R9AKxqe0XbR0haDjgV+C/gKbZXBWZRmnEARhsC9lfgiV2PnzrKc7pfdwtw44jrr2R7155/stGt27kjaSlgHUrzyR+AdatjHesBt41R7sc9lrQ+5dvIO4E1qt/X1Sz4fY3nFmB1SauOce4zI35HT7T9HQDb37b9IsoHgoEjJ3C9mMKS6KNu3wR2k7STpKUlLV91cq5DaatejtLuPb/qONyx67W3A2tIWqXr2OXArlXH4lMptc3x/Bp4oOpQXKEqwxaStq3tJ1zYNpJerTLi572UJpALgYso/Q+HSnpC1SG9G6U5aCy3U/oUOlakJNo7oXRkU2r0i2T7j5TO7aMlrVaV4cXV6eOAt0l6rooVJb1C0kqSni7pZdWH8kOUbzCPjnGZGBBJ9FEr27cAe1CaS+6k1B4/ACxl+wHg3cD3gHuB/YGZXa+9DvgOMK9qUlgL+AZwBaWz8KeUzsXxrv8I8Epga0rH6F3AV4BVxntdD06ndJLeC7weeHXVHv53SmLfpSrD0cAbqp9xLMdT2sbvk3Sa7WuB/wYuoHwIPAs4bzHK9npKn8N1lM7X9wLYnkPpwP2/qtxzKR27UD6Ij6jK/CfgycCHF+OaMQVlwlTEEpL0SWAT2wf0uywR40mNPiKi5ZLoIyJaLk03EREtlxp9RETLJdFHRLTclFvtb8011/QGG2zQ72JERAyUSy655C7b00Y7N+US/QYbbMCcOXP6XYyIiIEi6fdjnUvTTUREyyXRR0S0XBJ9RETLJdFHRLRcEn1ERMsl0UdEtFwSfUREyw1Vop8xYwYzZszodzEiIibVUCX6iIhhlEQfEdFySfQRES03oUQvaWdJ10uaK+lDo5z/V0nXSrpS0s+r3es75w6U9LvqdmCdhY+IiEVbZKKXtDRwFGWT482B/SRtPuJplwHTbW8JnAJ8tnrt6sAngOcC2wGfkLRafcWPiIhFmUiNfjtgru151c72JwF7dD/B9i9s/616eCGwTnV/J+As2/fYvhc4C9i5nqJHRMRETCTRrw3c0vX41urYWA4GfryEr42IiJrVuh69pAOA6cBLFvN1hwCHAKy33np1FmlSdcboz549u6/liIjoNpEa/W3Aul2P16mOLUTSy4GPArvbfnhxXmv7WNvTbU+fNm3UDVIiImIJTSTRXwxsKmlDScsC+wIzu58g6dnAMZQkf0fXqTOBHSWtVnXC7lgdi4iISbLIphvb8yW9k5KglwZOsH2NpMOBObZnAv8JPAk4WRLAzbZ3t32PpH+jfFgAHG77nkZ+koiIGNWE2uhtzwJmjTh2WNf9l4/z2hOAE5a0gBER0ZvMjI2IaLkk+oiIlkuij4houST6iIiWS6KPiGi5JPqIiJZLoo+IaLkk+oiIlkuij4houST6iIiWS6KPiGi5JPqIiJZLoo+IaLkk+oiIlkuij4houST6iIiWS6KPiGi5JPqIiJZLoo+IaLkk+oiIlkuij4houST6iIiWW6bfBaiVVN/z7N7KEhExRaRGHxHRckn0EREtl0QfEdFySfQRES2XRB8R0XJJ9BERLZdEHxHRckn0EREtl0QfEdFySfQRES2XRB8R0XLtWuumaVlLJyIGUGr0EREtl0QfEdFySfQRES03oUQvaWdJ10uaK+lDo5x/saRLJc2XtNeIc49Iury6zayr4BERMTGL7IyVtDRwFLADcCtwsaSZtq/tetrNwEHA+0cJ8aDtrWsoa0RELIGJjLrZDphrex6ApJOAPYDHEr3tm6pzjzZQxoiI6MFEmm7WBm7penxrdWyilpc0R9KFkvYc7QmSDqmeM+fOO+9cjNAREbEok9EZu77t6cD+wP9K2njkE2wfa3u67enTpk2bhCJFRAyPiST624B1ux6vUx2bENu3Vf/OA2YDz16M8kVERI8mkugvBjaVtKGkZYF9gQmNnpG0mqTlqvtrAi+kq20/IiKat8hEb3s+8E7gTOA3wPdsXyPpcEm7A0jaVtKtwN7AMZKuqV7+DGCOpCuAXwBHjBitExERDZvQWje2ZwGzRhw7rOv+xZQmnZGvOx94Vo9ljIiIHmRmbEREyyXRR0S0XBJ9RETLJdFHRLRcEn1ERMsl0UdEtFwSfUREyyXRR0S0XBJ9RETLTWhmbFvM7ncBIiL6IDX6ATJjxgxmzJjR72JExIBJoo+IaLkk+oiIlkuij4houST6iIiWS6KPiGi5JPqIiJZLoo+IaLkk+oiIlkuij4houST6iIiWS6KPiGi5JPp4TNbSiWinJPqIiJZLoo+IaLkk+oiIlhuqjUeaNrvfBYiIGEVq9BERLZdEHxHRckn0EREtl0QfEdFySfQRES2XRB8R0XJJ9BERLZdEHxHRckn0EREtl0QfEdFyE0r0knaWdL2kuZI+NMr5F0u6VNJ8SXuNOHegpN9VtwPrKnhEREzMIhO9pKWBo4BdgM2B/SRtPuJpNwMHAd8e8drVgU8AzwW2Az4habXeix0RERM1kUXNtgPm2p4HIOkkYA/g2s4TbN9UnXt0xGt3As6yfU91/ixgZ+A7PZe8jaT6nmf3VpaIaI2JNN2sDdzS9fjW6thETOi1kg6RNEfSnDvvvHOCoSMiYiKmRGes7WNtT7c9fdq0af0uTkREq0wk0d8GrNv1eJ3q2ET08tqIiKjBRBL9xcCmkjaUtCywLzBzgvHPBHaUtFrVCbtjdSwiIibJIhO97fnAOykJ+jfA92xfI+lwSbsDSNpW0q3A3sAxkq6pXnsP8G+UD4uLgcM7HbMRETE5JrSVoO1ZwKwRxw7run8xpVlmtNeeAJzQQxkjIqIH2TN2mGT4ZsRQmhKjbiIiojlJ9BERLZdEHxHRckn0EREtl0QfEdFySfQRES2XRB8R0XJJ9BERLZdEHxEDYcaMGcyYMaPfxRhISfQRES2XRB+TJjWyiP5Ioo+IaLkk+oiIlkuij4houST6aI1B7wMY9PLH1JVEHxHRckn0EROUGncMqiT6iIiWS6KPGBL5RjK8kugjIloum4MPkNn9LkBEDKTU6CMiWi6JPiKi5dJ0E4+Z3e8CREQjUqOPiFpkVM/UlUQfEdFyabqJekj1Ptde8rJExEJSo4+IaLnU6GMwNP2Noc74+TYSU0xq9BERLZcafcRkyDeG6KPU6CMiWi6JPiKGXtvnAKTpJqIN0jQU40iNPiKi5VKjj4hFyzeGgTahGr2knSVdL2mupA+Ncn45Sd+tzl8kaYPq+AaSHpR0eXX7cr3Fj4iIRVlkjV7S0sBRwA7ArcDFkmbavrbraQcD99reRNK+wJHAPtW5G2xvXXO5I6JN8o2hUROp0W8HzLU9z/bfgZOAPUY8Zw/gxOr+KcD20uL8z0VERFMmkujXBm7penxrdWzU59ieD9wPrFGd21DSZZLOkfRPPZY3IiIWU9OdsX8E1rN9t6RtgNMkPdP2n7ufJOkQ4BCA9dZbr+EiRSyZ2f0uQMQSmkiN/jZg3a7H61THRn2OpGWAVYC7bT9s+24A25cANwCbjbyA7WNtT7c9fdq0aYv/U0RETGH9npA1kRr9xcCmkjakJPR9gf1HPGcmcCBwAbAXcLZtS5oG3GP7EUkbAZsC82orfQyU2QMeP2JQLTLR254v6Z3AmcDSwAm2r5F0ODDH9kzgeOAbkuYC91A+DABeDBwu6R/Ao8DbbN/TxA8SERGjm1Abve1ZwKwRxw7ruv8QsPcorzsVOLXHMkZERA8yMzYiajG73wWIMSXRR0wRs/tdgGitLGoWEdFyqdFHDInZ/S5A9E1q9BERLZcafUS030SX3prI8wZw0bTU6CMiWi6JPiKi5ZLoIyJaLok+IqLlkugjIlouiT4iouWS6CMiWi6JPiKi5ZLoIyJaLok+IqLlkugjIlouiT4iouWS6CMiWi6JPiKi5ZLoIyJaLuvRR8RAmN3vAoxniq93nxp9RETLpUYfEUNvdr8L0LDU6CMiWi6JPiKi5ZLoIyJaLok+IqLlkugjIlouiT4iouWS6CMiWi6JPiKi5ZLoIyJaLok+IqLlkugjIlouiT4iouWS6CMiWi6JPiKi5SaU6CXtLOl6SXMlfWiU88tJ+m51/iJJG3Sd+3B1/HpJO9VX9IiImIhFJnpJSwNHAbsAmwP7Sdp8xNMOBu61vQnwP8CR1Ws3B/YFngnsDBxdxYuIiEkykRr9dsBc2/Ns/x04CdhjxHP2AE6s7p8CbC9J1fGTbD9s+0ZgbhUvIiImyUR2mFobuKXr8a3Ac8d6ju35ku4H1qiOXzjitWuPvICkQ4BDANZbb72Jlv3xGthrMfGnQOzET/xBjz9jRvl39uxmyzGGKdEZa/tY29NtT582bVq/ixMR0SoTSfS3Aet2PV6nOjbqcyQtA6wC3D3B10ZERIMmkugvBjaVtKGkZSmdqzNHPGcmcGB1fy/gbNuuju9bjcrZENgU+HU9RY+IiIlYZBt91eb+TuBMYGngBNvXSDocmGN7JnA88A1Jc4F7KB8GVM/7HnAtMB/4Z9uPNPSzRETEKOSmOykW0/Tp0z1nzpx+FyMiYqBIusT29NHOTYnO2IiIaE4SfUREyyXRR0S0XBJ9RETLJdFHRLRcEn1ERMsl0UdEtFwSfUREyyXRR0S03JSbGSvpTuD3DV5iTeCuxE/8xE/8SYw9GfHXtz3q8r9TLtE3TdKcsaYJJ37iJ/7wxh/ksi9Kmm4iIlouiT4iouWGMdEfm/iJn/iJP8mxJyP+mIaujT4iYtgMY40+ImKoJNFHRLRcEn1ERMsl0ddM0mqStqwx3jRJm49yfHNJo06OiMEjaVlJW0p6lqRlBy3+ZKj7b6tJkp7X7zJ0W+Tm4INO0qvHO2/7+zVcYzawO+X3eQlwh6TzbP9rr7GBLwJHj3J8DeBjwP41XANJ6wOb2v6ZpBWAZWw/UEfsKv5zgBcBBs6zfWldsav4TwW2q+JfbPtPNcZeHngHC8p/LvAl2w/VFP8VwJeBGwABG0p6q+0fD0L86hqNvH8a/ttC0iuBfwPWr64hwLZX7jH00cBzeoxRH9utvgFfrW5nAPcCp1a3e4Af1XSNy6p/3wx8qrp/ZU2x54xz7uqarvEW4GLghurxpsDPa/w/OAy4CvhUdbsC+FiN8d8M3Ax8DTgRuAl4U43xvwccD7y0uh0HnFxj/OuATboebwxcN0DxG3v/NPm3VcWaC2xJNQKxxriX1hmv11vra/S23wgg6afA5rb/WD1+GiUx1GGZKt5rgY/WFLNjpXHOPaGma/wzpTZ8EYDt30l6ck2xAV4HbOWqBizpCOBy4NM1xf8A8Gzbd1fx1wDOB06oKf4Wtrubz34h6dqaYgM8YHtu1+N5QG3fpiYhfpPvnyb/tgBuoVSY6h5nvpGkmWOdtL17zdcbV+sTfZd1O0m+cjuwXk2xDwfOpDRJXCxpI+B3NcWeK2lX27O6D0rahfIHW4eHbf9dUif2MpQmirr8AVge6DR1LAfcVmP8u1k4cT1QHavLpZKeZ/tCAEnPBeb0GrSrWXGOpFmUbw4G9qbUkKd0/C5Nvn+a/NsCOBSYJekc4OHOQduf6zHuncB/9xijNsOU6H8u6UzgO9XjfYCf1RHY9snAyV2P5wGvqSM28F7gDEmvpbRRAkwHng+8sqZrnCPpI8AKknagtEf/sNegkr5I+YO/H7hG0lnV4x2AX9cQv9NOOxe4SNLpVfw9gCtriH9VFe8JwPmSbq4er09pDunVbl33bwdeUt2/k/LBONXjdzTy/oHG/7YAPgP8hfL7qLOT+i+2z6kxXk+GamaspFcBL64e/tL2D2qKuw6l0/SF1aFfAe+xfWtN8ZejdLpuUR26Bvi26+sMXAo4GNiR0hl1JvCVXr/OSjpwvPO2T+wx/ifGD+/De4y//njnbdeynLak1W3fM+LYhrZvHJD4jbx/qthN/21dbXuLRT9zseN+3/a4A0EmVb87CSbzRqmJvby6/0RgpZringW8kfINaRngIOCshn+WFwFH1RBnaeBbDZd1m1GOvbLG+BuOcmzbGuMfPMqxI2qMfx6wctfjZ1BTR3vT8Zt+/zT9twV8FtixgXK/Bnj1WLemfl9j3YZmHL2ktwCnAMdUh9YGTqsp/DTbX7U9v7p9Dah9jLukZ0v6rKSbKEPCem4+sP0IsH7DY6uPk/RYrUnSfsDHa4x/iqS1u+K/mPo6YgFeI+l1XfGPot7/338HfijpSZK2obxPDxiE+JPw/mn6b+vtwE8kPSTpger25xrivpLSdLYbZTGz3bpudTW5TtgwtdE3OTLgbkkHsKD9fz9q6gyUtFkVbz/K7jTfpTS5vbSO+JV5wHnVKIG/dg669w6pjr0oyXh/4J+AN1C+5tflbcBpknajjF3+D2DXGuO/Bpgp6VFgZ+A+2wfXFdz2GZKeAPyUMsrqVbZ/Oyjxafb909jfFoDt8Ua19RL3jZ37ki7rftwPw5TomxwZ8CZKO+L/VI/Po3zdrMN1lHbJV7oaIifpX2qK3XFDdVuK8YdzLhHb8yTtS/kGdTPlq/KDNca/WNK7KYnsIUrz3J29xpW0etfDN1PKfx7wqdHavZcgfqezumMVyv/DOyVh+91TOX6XJt8/Tf5tASBpdxb03c22/aM641PvCLYlMjSdsZI+C9xHqU2+izIy4FrbTYzNrY2kPYF9KZ1RPwFOonR0bdjAtZ4EYPsvNcXrjFrpeDJlBM7D1XV6ms4u6Ycj4m8O/JEyMQ73OFZZ0o1VfI1y2rY36jF+053VjcYf5Xq1vn8mQzWnY1vgW9Wh/SiTFD9c4zUutd3XWbLDlOibHBmwEfB54HmUxHAB8C8uQ8FqIWlFyrDB/YCXAV8HfmD7pzXE3gL4BtCpwd4FvMH2NT3GbXTUiqSXjHfeU2h4W5s19f6pYjf6tyXpSmBr249Wj5emzMatsxLyYuCX3ed7rYQsdnmGIdFX/3lft/26RT55yeJfCBzFgnbEfYF32X5uQ9dbjTLpZR/b29cQ73zgo7Z/UT2eAfy77RfUEFuUvpFOZ+ltwK/r+IDtusZTuuPbvr3G2P+P8gHbXf7Tbdcxjn686/7Y9i49xtjZ9k+q+6sAn6PUXq+mJMtafk8Nv38a/duqEv2MTjNc1Vw3u4ZEP6UqIUOR6AEknQu8zPbfG4h95cg3hqQrbG9V4zVebvtnI44dVI1C6DX248paR/kl7UhZ3Ol3LJgJuw6wCfCOXr+NSNqasljXKiPi3we83fZlPcb/IOUb1ElAZ9z2OpRkc5LtI3qMP9bXeVHWYXpaj/EfazKQ9BXgT5R1el4NvMT2nr3E77pOI++fKk6jf1vVCLAjgF9Qfu8vBj5k+7t1xJ8qhinRf50yfrj2kQGSjqS0C59E+bq2D7Aa8J/VNXrqtKuu8UvKRKn3A08CvkLpYN6rhtg/AC6lfP2GMvRuG9uv6jHub4BdbN804viGwCzbz+gx/uXAW21fNOL484Bjavig+i3wTNv/GHF8WeAa25v2GP8R4BxG7wN4nu0Veozfnegvt71117mFHvd4nUbeP1Xsyfjbehrlmw6Ub5s9r3wqaQ9gHdtHVY8vYsGw0ENtn9LrNRbHMI26aXJkwGurf9864vi+lDdnT512lZcA76MsBgZwmO3vjPP8xfEmyqqS36eU91fVsV4tw4KacLfbqGdBthVHJnkA2xdWfRq9ehRYCxjZl/C06lyvfkP5oHrc2i2Sbqkh/pNVlokQsLIkdTWZ1TmHpqn3DzT0tzXKt6nO+3QtSWu592W0D6WUsWM5yofJipTVdJPom2D7UwCSVi4P61trvYkRMKNYjdLWfQOl+WD9EX+4S8z2vUBdQ+26nQBcLOkkyiqBAOtS/gCOryH+jyWdQemY7o7/BsoIpV69l7JG0u+64q9HaXp6Zw3xP8nYCfddNcQ/jgWVmhOBNYE7Vdbuv3zMVy2mBt8/Tf5tdS84tg1lkbrONytTBjz0Ylnb3R/W57qsrnp3TZWQxTJMTTfTKZ+knTf+/ZQ1yy8Z+1WLFX8LyvC+xxaLsv31OmJX8X9LmXZ/gsrGDkcC02vq8DoL2Nv2fdXj1Sht0DvVEPsZPL4zc6btWpb5lbQrZWOKkfFnjf2qxYq/FI/vTL7YZUZo0Oz7p4r3AmADuiqmNf9tXWb72XXFq2LOtb3JGOdusL1xnddblKGp0VNql++w/SsASS+iJP6etyZTWVxrBiXRzwJ2oexCVNubkTIJ6GaAarLRu1Wm+tdhzc4faRX/XtU0a9j2byhNFI2oEnotSX2sS3TdOo/raLYBHvuG+UJ37fZUdTI/aPv6qR6/0tj7R9I3KBulXA50PlxNvX9bTdR2L5L0FtvHdR+U9FZqWLl1cQ1Ton+kk+QBbJ8raX5NsfcCtqKMv31jNdzvm3UElnSA7W/avlnSC22f13V6S0aMz11Cj0par/NBUo1/b/SrXk3DB58KfIKSeA+jNHe8mjKb+D1eeP+BJYk/5qghST2PGqo8APyvpIu6OhaPpr7Zn03Hh2bfP9MpGwYNWtPDv1CW5dif0lENpYloOaCW0U6LY5gS/TmSjqGMx+303s/udMr02PnyoO1HJc2valB3UNqK6/CvLPjQ+CIL70P5JuD/arjGR4FzVTZfEGU9mkN6DbqI4YN1jPj4GmWLyBUpw+O+BbyC8of0ZUqTUS8+T/kmdVP3wc6oIcoorp7YtqRvU0aqfKEat++6attNx6808v6pXA08lTLjuTZasDyEgHUkfaH7vHtcHsL2HcALJL0MeGZ1+AzbZ/cSd0kNUxv9L8Y5bdtL3Pki6WjgI5ROxvdRNjK4zHbPIw+62w9HtiXW2bYoaU3K7EOAC23fVUPMpocPdv9ubra9Xte5nocPVp2wz7A9f8TxZSnLZ4zaBrsE11mPMst5G5Up+b+1Xdvqm03Hr65R+/univsLSqXg1yy8A1Svy1s0vfzE6uOdr2NY6OIYmhq9F7Hao6QDl/Q/1/Y7qg6oSyk71qxE2QC7Dh7j/miPl4ikFwKX2/6RykqBH5H0efe+sUbTwwe7R6yMbLOtY/hg06OGAKia5e6UtC3wKqDWjsGm4zf4/oEyMql2tk9UmTF/pO33N3CJS1jwjaHzd9o9qqeOIdcTNjQ1+kVRDwsPSXoz8B5K++3llJrNBb18S+iK/TfKVnmidEp1NnkWsJHtnodqqUwD34rS5v9VShJ7re1xp3FPIO5ewFWjNRNI2tN2T/sBSDoc+KxHLKIlaRPKCKU6JpM1Omqo6zp7UZLaBbbfUmfspuM39f7piv8UFp7QdEcdcavYF9h+fl3xpqok+kovzSAqqzRuS/nKunXVDvrvrmErMU3CdnadDzlJh1HWijm+lw++WHwq68XPAd5su86NuxuP3+T7R2Wv5P8EZrOg/f8DrmlmqaQvUT7ET2bhGfPf7zHu+pR9C+6vHr+U0nd0E2VnuNqXYhnP0DTdTEAvn3gP2X5IEpKWs32dpKfXUqgJJvIeayYPSPowpcPuxdXY8TpmrrZl+OBo1+151FA3l2UWalsbaZLjN/b+oXT0btupxUuaBvyM+maWLk/ZyKT727cps3x78T1KM9n91fvxZMqGOFtTRj29ucf4iyWJfoHROgwn6lZJq1I2pjhL0r08fjnPmZ4AABxWSURBVNp805Zf9FPGtA9l8/GDbf+p6rz7z3qKNbjDBydh1FD3tdam7GncPSmojqGzkxG/yffPUiOaau6mxuUb3NzOTyvY/kN1/wDgBNv/XX0I1jYreaKS6Bc4b9FPGZ0XLN70yWqUwCrUMwV/sYqxxC8sizh9ruvxzXR1bvbybWHAhw9ezNijhlatIT7w2MJd+wDXsvCkoFoScdPxF/X+6dFPJJ3JgmWK96GmCXKSdgE+TJnoCGXRwCNdz6zq7vfMy6rrUA3DriH8YvIk70bejxtlp/o1ux4vSxnn+5t+l63Gn/HSBmNf1uPr1wMuqe4fQVl6os7yNRKfMoZ70zHO3VJj+a8Hlmvw/6/R+GNc89gaY72G8kHyOcp+t3XEfAulz+JlwMrV7WWUYZyH1BD/85Tmm88DNwJPqI4/jbKD1aT9X9huf41eZa/SY4C/VuOiP0M1bA5oZCOSPmmymtBTj70Hd/jgJ2l20bGOeZQ27YcX9cQpGn80x9QVyPapwKl1xav8C/AiLzye/eyqln8ucGyP8d9L+fbxtOo6naWun0rpd5hUrU/0wMcoa2PPrdpcLwD2sv3DPpdriUhag7I5ws1eeEG21/epSBP1FcrQu1/a/tsgxPc4Izvc49DQEf4GXC7p5yw8KaiuFSEbjS/pOR4xs9w9LhYo6QHGqWDYXrmX+JQRh4+btGT77jqaVlyq7yeNcrynzXCW1DAk+r/bngtlmQNJvxukJC/pR5Qdb65W2SDhUspXzo0lHWv7fwFsX91kMWqIcTrwcXqvKU1q/Eka1TOzujWl6fj/rbLu0CnAd+t4L9peCUDSv1GWP/gG5X34OkotuVd/lrSV7YUmNkraitLB3xMt2Fj+sUNdj+1JXr2y9ePoJd1KV0cRZe2Y7o6jnneYapKka2w/s7r/EeD/2X6DpJWA89zj3pYTLMMWDX+QTFkq1bvrgOd7wb6i5wNvrDHRd5ZV2Kx6eL1H7Go1APGfStkkZB9Ke/d3bX+6hrhNbXP5IsraSF+lzGKFsoDagcABts/tMf4aIw4tRfn9vJ/Sn/aaXuIvrmGo0XdvvjDy8SB8ynX/QW5PKT+2H5BUy3K5kl5NWd/+yZSah8olytfjOpL8oA4ftJtfFExlM+0TKZNpBKyrsiRHXaNuGo0Pj428+UI16uxQymqiPSd6St/a61iwleB+dE1sWlIuq9duB/wzcFB1+FrKGkw9byXosslIZz+D1wMfoAyrfIVrnlU90QIN7Y0yEaPv5VhEGX9I6fh7FWXvzFWr4ytQ9i2t4xpzKYt3NfUzHElJMrOqn+eHlGUEBiV+06OGLgGe3vV4s871BiT+Mygd11dRZrC+HXhyTbE3oDTL3VXdTgM2qLHs75nIsSWI+wTK9ofXUfqPNqnzPbO4t9Y33YwkaXNKrWA/yhTl6X0u0rhUNnA4nNIueZSrNdCrKdXb2P6vGq5xnu0X9hpnnPjXA1vabmTUR9Pxq2v8hNIH8E3g2a6xQ1nSlR7RBDfasSkc/wJKjftkL5gkNBBGW6qhl+VQumLcCswH/he4eeR597jEwuIahqYbJG3AguT+D8pX/Okesc74VOQyK/Btoxz/BWUN9jrMkfRdSm2pe1RGXW/GNgwfbHLU0BxJX2HBvgOvo3S4D0R8N7gomKR1KPswdCoiv6LUuEfbdH5x4u5Hmc27oaTujuqVgDqWEP4ZpalpKx6/9EQdSywsltbX6KvaxsqUGsdJtn8n6UZPzobePZP0Q8YfZtbTutzVNb46euje19Ov4p9KebM3Nbyv0fjVNZpcFGw5Slvxi6pDvwKOrusbyiTEfyGl6abTR9Lp4+l5KV6V/Wi/TRl1A6Wv5HW2d+gx7vrAhpT1Zz7UdeoB4EqP2INg0A1Doj+NsivTTODbts+XNK+ON+FkkDTuUq+2z5mssiwpjbHJg3vc3GGy4sf4JF1HmYB0CQuWWMBVh2SPsR+3gcxox3q8xvqUGdA/k7QCsIztnoZYSvrX8c57kkf7tb7pxvaeklah7CX6SUmbAqtK2s72pG/SuwS2o3wTqWOjjlE19fW4w2WTh8aG9zUdH5oZ1SPpe7Zfq7LM9eNqXL22oTcdv8v97ppnULO7VTYz6ax1sx9lYbNaSHoLZTmU1Sn7PaxD2YZy+x5DrzTOuUmvXbe+Rj9S1bn5WsobZj3bde3t2ghJ/0PZfPwmypv9ZNt31nyNRr4ed8WfwYjhfcCBvSbKSYw/6qJgvTabSXqa7T9qjD0H3ONeA03H77rOEZT1pL7Pwk1nvezD3Im9PqUS8nxKgjwfeFddFR9Jl1MqUxd5wbaUV9l+Vh3xx7jme11NdJw0/RzyMxk3YG9g+THOrd/v8k3wZxDwEuBLlFmCP6FM7FippviXT+RYD/EHffhg04uOHTmRY1M4/i9GuZ1dU+wTgdW6Hq9OWfK3rrJfVP17WfXvMpQ2+kb+r6tr3Nxk/NFuta3rPIXtD9ws6RuSdlXZJxKor0bTNBfn2H475avl/1AWTbq9pkvcLekASUtXtwOo8esxZeW+xyYY2f4t9W1MMRnxO6N6mjLaN6faNjVpOr7tl45y63kbzcqWtu/tutY91Lso3jnVjPMVJO1A2SCk6SVSJn2d4mFoo39VtV7JqygTj46XdDrwHQ9AR2Y3Sc+ibEy9D2XyyIdrCv0mytfj/2HB1+M6N2QY6OGDNLQomKS3A+8ANlLZd7VjJXrYH2Gy4ndd598pe/feVz1eDXif7Y/VEH4pSat1kr2k1ak3b30IOJgy2eutlEl3X6kx/mjSRt+0ag2KvSh/AKt76rfRb0pJ7vtS2oc7w0Tn1XiNNVzDCIlx4g/68MFGRvVUgwRWY5Qhfh5lZcWpFr/rOo+bYDTaRKQljP0G4COUmjaUptjP2P7G2K/qP429+qYou09NaiV7qBJ9VdPYi9IRuylwiu1/6W+pxifpBkon7EluaGExlXX6L6es0/8TD9ObYoKaHtVTXePJdG0J6bJT05SPX31b2LbzwVoNUZzjajG+GuJvzoI9Xc92DWvFjDUSqcOTsFjgZGp9opf0JEqzzX6Utr2ZlFrx7EFOaNViSfvZ/lYNsQS8nNKEsy1lZ5yvVW3dvcRtxfDBSRjVsxtlRdW1gDsowzh/U2OibDr+B4HdKDOHoTT7zbT92TriN6FrJJKAM4Bdu88PSv/dRA1Dor+LMkrlJODMJmpiTar6F/4ZWJvyIXUW8E7gfcAVtveo+XovpbR1rwhcQVkL/4IljNWW4YOXAPt3OnwlbUbp49mmpvhXUGqsP7P97Or/4ADbBw9C/OoaO1MqCwBn2T6zrthNq6uZaSprfWcssK7tB/tdiB58g7Jq5QXAmyntlQL2tF3LbvJVv8UBlOVUb6d0Ws8Etqa0jS7RchG2/1jdfYftD4645pHABx//qqkTv8vjRvVUSyLU5R8uOxstJWkp27+QVOc466bjY/snlArV46iHzeWjHsOQ6H8taaxOEQ9AW9xGriZvVCNL/kiZ6PVQjde4gPKBsqcXng07R9KXa4i/A49PuruMcmyqxm96VM99VRPjL4FvSbqDGtZcn8T4i7L8op8yuVS2Fe1YQdKz6Rr26Bome00lw9B0M9BtcSO/VjbxNVOSmuiv6B7eB9zQdaqzO9YBUzl+13WaHtWzIvAgZRei1wGrAN+sa2RM0/EncP0p1zSiskHKWFzjPIApofWJvttUfMMtiqRHWFD7EmXDkb/BwrtA9XiNzShbnG3Awmu59PRmb8vwwaZJOnK0pqeRx6Zq/Alcf+D+7jok7WD7rH6Xo1dJ9NHprPsyj1998JIxX7Rk1xmo4YOTOKpntM0v6twYpNH4E7h+zxt59Etbckbr2+iHrS1uCc23/aWmgo81vA9odPhgDfHfU/37yh7jjGrQZ8ZK2gR4iu3zRhx/IfAn253mtNf3eq0+mvTlCprQ+hr9sLXFLQlJn6QkyB+w8BT/utqIB3r4YFNNH4PetCXpR8CHbV814vizgH+3vVuv1+i3ttToW5/oJ6otbXFLQtKNoxy2a9qcRdIc29OrhPxs249KusL2yC3Wpmr8pptW1hvteI0zVxuJL+li29uOca7RpX4nS1sSfeubbhbDkZTJSEPHzW+rOJDDB5tu+uhyBqUPQJQ+hg0pSyPX0rTVYPxVxzm3Qo+xp4qb+l2AOqRGXxnkDqM6SNoC2JyFOzO/XlPsgRw+2K9RPVW/0jtsv3kqx5f0HcraM8eNOP5mYAfb+/QSfzJJ2pCyRMq1tq/rd3nqlkRfactXtCUh6RPADEqin0WZbHSu7b1qit+K4YNNjxoaca2mdznqOb6kp1D6df5OGbEFMB1YFniV7T/1VsrmSDrN9p7V/T2A/wVmAy8A/sP21/pXuvol0VeGPNFfBWxF2WVnq+oP+JuubyvBgR4+ONaoHte3KFj3RtJLUTazX8P2TgMS/6XAFtXDa2yfXUfcJnV/g5d0PmXrzBslrQn8vK7+nakibfQL3NTvAvTRg1UH5nyVRdTuoKzQ2JNBHz7Y5dPA8xgxqqfG+N0bSc+ntKmfOijxbXe2D0TSiio7lO1n+xV1XaMB3TXcZWzfCGD7LkmP9qlMjRmKGn2VvKZ1jevtHN/S9pVjvGxoSDqasljavpRVMf9C2TO2p12mBn34YNd1Gh3VM+hU1up/BWXbzp0oHyLft930lnxLrGvGuYDlKPtH/7H6WeZM1mSyydL6RC/ptZT2tzso+34eZPvi6tzQNteMRdIGwMp1fgAO6vDBrvg/A/akfKCsSXkvbWv7BT3GnTneedu7T/H4O1L2ediRUqP/LvBF2xv0ErefJK0KPMNLuDT3VDUMif5yYJfq03o74OuUSR4/yEgb7QSsZPuUEcf3Au6va15B1xICCw3vq7GNu+n4TY3quRO4hbKD2EWMmIXpHvc0noT4j1IWeDuo0/QhaV5d8y8mQ9UftXb18Dbbt/ezPE0Zhjb6pV2tW27711X76o8krUsfNumdYg6j1FRHmg38kJrmFYwc3dEZ3ldH7MmIDxxWjeB5lLLTFKpnvfunUpZY3o/S7HEGZUOTa3qMO1nxn0Np7vuZpHmUzX2Wril2o1SWQvkS5UP7turwOpLuA95u+7K+Fa4Jtlt9A84HNh5xbCXg58DD/S5fn383c8Y5d2XD175qUOIDlzb9+6G0Ex8E3Am8s4HfR9PxXwB8EfgD8GPgkCb/f2so7+XAc0c5/jzKzm19L2Odt2Go0b+dx39lfUBl67PX9qdIU8bKkpaxPb/7oMruSbXNbBxjeN8fpnr8yRjVo7LW/Ssote4NgC9QxqbXoun4HbbPB86X9B7KloL7AMfWfZ0arWj7opEHbV9YNdW1SusTve0rxjj+D6DnjbUH3PeB4yS90/ZfAaqlBD5fnavLoA4f/DaldtrUqKGvU8afzwI+ZfvqXmNOZvyu6xxs+3gAlxFJPwem+taBP5Z0BqXP7pbq2LrAGxhjS8RB1vrO2A5Jr6asZ/NkSg2/to07BpWkZShjxN8M/J7yO1kXOB74uAdsI/WmNLgo2KMsWJOn+w+xlvdm0/G7rvNtyro3BwNrAF8FzrH9/jriN0XSLsAedHXGAjNtz+pfqZoxTIl+LrCb7d/0uyxTjaQVgE2qh3Nd02bqgz58sOs6jY7qaQNJ+wBHUT5Y9veINeqjv1rfdNPl9iT5Mc0HXgq8GEDSOcCXa6jRP59xhvfVoOn4wKSM6hlokjalbNJyKvAM4PXV0OW/9bdkY6sm232YUqN/CuWD/A7gdOAI2/f1sXi1G6Ya/ecpw81OY+HNNepsix5Ikr5CmUx2YnXo9cAj7n11w6VZMLxvS2oe3td0/EVcuxXrrddB0nXAP9v+uSQB/wq8aSp/45F0JnA2cKKrxdckPZUyMulltnfsY/FqN0yJ/qujHLbtN016YaaY0abz1z3Fvxr9sR/wn5SOwf+rK3bT8ZteFGzQSVrZ9p9HHNvM9m/7VaZFkXS97acv7rlBNTRNN+5x3ZaWe0TSxq7WApK0EV2bhPeiJcMHmx41NJAkHWr7s7b/LGlv2yd3nT6Isn7SVPV7SYdSavS3w2OzZA9iwSic1himGv1mlJlwT7G9haQtgd1tf7rPRes7SdtTRkrMo7Rzrw+80WVVwl7idg/vO6nh4YO1x4/xda8VNXLdqKm+jpSk1ShDZvegjMQDuB2YCRxZx/DZqWSYEv05wAeAY7xgHeqrbW8x/iuHQ1Uz7nxdvd72w+M9f4IxB3r44GSN6hlUWnhN94XWjRr2daSmmqFpugGe6LLWTfex+WM9eQhtQ2n6WAbYWhLucStB20vVUbB+xWeSRvUMMI9xf7THA0PSc2xf2u9y1GmYEv1dkjamegNWKzT+sb9FmhokfQPYmLL+R6dt3pRZg8Os6UXBBt1Wkv5M+QBcoboPC+YbDKq3A2/pdyHqNExNNxtR1t54AXAvcCNwgO2b+lmuqUDSb4DNPSxvhiXQ9KihiCYNTY3e9jzg5dWCRUvZfqDfZZpCrqbUXvMNZ4TJWhQsJl81aWpnFl4C4cy2TZaC4arRLwe8hgXt0ADYPrxfZZoqJP0C2Br4NQtPJhv2zsaM6mkpSW8APgH8lK716ClNdZ/qtX9qqhmmRP8T4H7gErrGiNv+774VaoqQ9JLRjrvHHYgG3WQtChaTT9L1lPXo7xtxfDXgItub9adkzRiaphtgHds797sQU5HtcyStD2xq+2eSnsiA7BTUpEkY1RP9I0YfGfQoLRxdNUyJ/nxJz7J9Vb8LMtVIegtwCLA6ZfTN2sCXge37Wa6IBn0GuFTST1kwE3Y9StPNv/WtVA1pfdNN1xKzywCbUmZ/PsyCr99b9rF4U4LKBurbUb6ydibAZNGuaLWqmWYnHt8Ze2//StWMYajRv7LfBRgAD9v+e2cyWbUhSbtrADHUJD3P9oWUDc1br/VtkLZ/37lRdr/ZA9idsvrg7/tbuinjHEkfoUx62QE4Gfhhn8sU0aSjO3ckXdDPgkyG1if6DkmHUdZbXwNYE/iqpI/1t1RTxoeAO4GrKG31Z9j+aH+LFNGo7g7XQZ7FOyGtb6PvqIZTbWX7oerxCsDlbVt3enFI2oMyGumo6vGvgWmUZptDbZ/Sz/JFNEXSFcAMSmX37Or+Y8m/batXDkMbfccfKJ/cD1WPl2PBRIlhdSiwb9fjZSmLmz2JsmxxEn201SqUOTWd5N69iJmBjSa9RA0apkR/P3CNpLMo/5E7AL+W9AUA2+/uZ+H6ZFnb3ZssnFvVZO6ploqIaCXbG/S7DJNpmJpuDhzvvO0TxzvfRpLm2t5kjHM32N54sssU0S/V6rb7A/tO5f1ul8Qw1ejvoXQyPtrvgkwhF0l6i+3jug9Keitl3ZuIVpO0FrAPJcE/C/gPFm7ObIVhqtF/k7KRxKnACbav63OR+k7Sk4HTKBPIOm2U21D6L/bs7KUZ0TaSDqGsSLo28L3qdrrtDftasIYMTaKHsls95T/3jZR2+q9SNpIY6iWLJb0M6HxVvcb22f0sT0TTJP0duAB4n+051bF5tlvVCdsxVIkeQNIawOuB9wK/ATYBvmD7i30tWERMmioP7E2p+D2VUqM/yPa6fS1YQ4Ym0UvanVKT34SyRd6Jtu+oVmq8dth64SOikLQOpZ1+P2BF4Ae2P9LfUtVrmBL9icDxtn85yrntbf+8D8WKiClE0qbAfm3bkGhoEn1ERIekV4933vb3J6ssk6H1iV7SAzx+d6CO7BIUMYSq3cMur27w+LzwpskvVXNan+gjIkaStCdlvPwmwOmU0Xdz+1uq5gxVope0FfBP1cNf2r6yn+WJiP6qlvrYg9IZuwbw0TbulTxMyxS/B/gW8OTq9i1J7+pvqSKizx6irIP1Z8pifq1csnhoavSSrgSeb/uv1eMVgQuylWDE8KkmCe5L2ULzZ8BJnYlTbTRMif4qYNuu9eiXBy7OvqgRw6fqjL0SOJcyWGOhRNi21WyHaVGzr1IW8fpB9XhP4Pg+lici+ueN/S7AZBqaGj2ApOcAL6oe/sr2Zf0sT0RMbZK+aHvg+/JaX6OvmmjeRhlGdRVwtO35/S1VRAyIF/a7AHUYhlE3JwLTKUl+F+C/+luciIjJ1foaPbB5p8NV0vFkQ42IGDLDUKP/R+dOmmwiYjFp0U+Z+oahRr+VpD9X9wWsUD0WWesmYihJWmaCFb/PN16YSTBUo24iIgAkXWr7OdX9VoysGc8wNN1ERIzU3STTipE140mij4hhNFRNGWm6iYihI+lvwFxKzX7j6j4s6Ltr1RpYw9AZGxEx0jP6XYDJlBp9RAwtSRsCz6weXmt7Xj/L05Qk+ogYOpJWBr5CmTXf2U5wa+AS4GDbfx7rtYMoiT4iho6krwE3AYfbfrQ6JuDjwCa239C/0tUviT4iho6k39nedHHPDaoMr4yIWFgrlj3olkQfEcPofEmHVc01j5H0ceCCPpWpMWm6iYihU3XGHg88h4U7Yy8D3mz7vn6VrQlJ9BExtCRtDGxePbzW9g2S1rL9h36Wq25J9BERXSTdbHu9fpejTmmjj4hYWDpjIyJarnXNHFnrJiKGjqQvMnpCF7DqJBencUn0ETGM5izhuYGUztiIiDG0ZfeptNFHRIytFbtPJdFHRLRcEn1ERMsl0UdEjK0VY+qT6CMiKpKWl7R316HP960wNcqom4gYapKWBnYC9gN2BH5le6/+lqpeGUcfEUNJ0kuA/YFdgV9TRthsaPtvfS1YA1Kjj4ihI+lW4GbgS8Bpth+QdKPtDftctEakjT4ihtEpwFrAPsBuklakhWvcdKRGHxFDqdpdagalbX5XYBXgYGCW7b/0sWi1S6KPiKEn6Qks6JDdyfaafS5SrZLoI2LoSPoAcJLtW0Y5t4LtB/tQrMakjT4ihtFalA3CfyXpHZKmdU60LclDavQRMaSqNvoXA/sCewJXAN8Bvm/7gX6WrW5J9BEx9KpJUy8HjgCebvuJfS5SrTJhKiKGmqRnUWr1+wB3AR/ub4nql0QfEUNH0qaU5L4v8AhwErCj7Xl9LVhD0nQTEUNH0g2U9viTbF/d7/I0LYk+IqIiaSlgP9vf6ndZ6pThlRExdCStLOnDkv5P0o4q3gXMA17b7/LVLTX6iBg6kk4H7gUuALYHnkzZZOQ9ti/vZ9makEQfEUNH0lW2n1XdXxr4I7Ce7Yf6W7JmpOkmIobRPzp3bD8C3NrWJA+p0UfEEJL0CPBXFuwJuwLwt+qxba/cr7I1IYk+IqLlMmEqIoaWpJcCz6weXm17dh+L05jU6CNi6EhaG/g+8BBwSXV4G0oTzqts39avsjUhiT4iho6kHwCn2/7aiONvAF5je4++FKwhSfQRMXQkXW/76Yt7blBleGVEDKNRc1+1BMLSk1yWxiXRR8QwOkPScZJW7Byo7n8ZmNW/YjUjiT4ihtEHgPuA30u6RNIlwE3An4H397NgTUiij4hhtDXwOWBd4CDga8BlwLLAk/pWqoYk0UfEMDoGeLjaCHw1yq5SxwD3A8f2s2BNyISpiBhGS9u+p7q/D3Cs7VOBUyW1bvXK1OgjYhgtLalT0d0eOLvrXOsqwK37gSIiJuA7wDmS7gIeBH4FIGkTSvNNq2TCVEQMJUnPA54G/NT2X6tjmwFPsn1pXwtXsyT6iIiWSxt9RETLJdFHRLRcEn1ERMsl0UdEtFwSfUREy/1/YJympULOA8sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "## Get at and plot the feature importance (tutorial: https://scikit-learn.org/stable/auto_examples/ensemble/plot_forest_importances.html)\n",
    "feature_labels = list(df_train) # columns names\n",
    "importances = forest.feature_importances_\n",
    "std = np.std([tree.feature_importances_ for tree in forest.estimators_],\n",
    "             axis=0)\n",
    "indices = np.argsort(importances)[::-1]\n",
    "\n",
    "## Print the detailed feature ranking and values if curious:\n",
    "# print(\"Feature ranking:\")\n",
    "#for f in range(df_train.shape[1]):\n",
    "#    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], importances[indices[f]]))\n",
    "#    print(feature_labels[indices[f]])\n",
    "\n",
    "    \n",
    "# Plot the feature importances of the forest\n",
    "plt.figure()\n",
    "plt.title(\"Feature importances\")\n",
    "plt.bar(range(df_train.shape[1]), importances[indices],\n",
    "       color=\"r\", yerr=std[indices], align=\"center\")\n",
    "plt.xticks(range(df_train.shape[1]), labels=df_train.columns[indices], rotation=90)\n",
    "plt.xlim([-1, df_train.shape[1]])\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}