week4/week4-explore-more-1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.1 Week4 - Explore more exercise 1\n",
    "\n",
    "* Check the precision and recall of the dummy classifier that always predict \"FALSE\"\n",
    "* F1 score = 2 P R / (P + R)\n",
    "* Evaluate precision and recall at different thresholds, plot P vs R - this way you'll get the precision/recall curve (similar to ROC curve)\n",
    "* Area under the PR curve is also a useful metric"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data and data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:18.650154Z",
     "iopub.status.busy": "2021-10-01T10:50:18.649113Z",
     "iopub.status.idle": "2021-10-01T10:50:19.744946Z",
     "shell.execute_reply": "2021-10-01T10:50:19.743993Z",
     "shell.execute_reply.started": "2021-10-01T10:50:18.65006Z"
    }
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "# import os\n",
    "# for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "#     for filename in filenames:\n",
    "#         print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:19.74699Z",
     "iopub.status.busy": "2021-10-01T10:50:19.746671Z",
     "iopub.status.idle": "2021-10-01T10:50:19.803291Z",
     "shell.execute_reply": "2021-10-01T10:50:19.802429Z",
     "shell.execute_reply.started": "2021-10-01T10:50:19.746956Z"
    }
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"WA_Fn-UseC_-Telco-Customer-Churn.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:19.804726Z",
     "iopub.status.busy": "2021-10-01T10:50:19.804496Z",
     "iopub.status.idle": "2021-10-01T10:50:19.857327Z",
     "shell.execute_reply": "2021-10-01T10:50:19.856459Z",
     "shell.execute_reply.started": "2021-10-01T10:50:19.804694Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerID</th>\n",
       "      <th>gender</th>\n",
       "      <th>SeniorCitizen</th>\n",
       "      <th>Partner</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>tenure</th>\n",
       "      <th>PhoneService</th>\n",
       "      <th>MultipleLines</th>\n",
       "      <th>InternetService</th>\n",
       "      <th>OnlineSecurity</th>\n",
       "      <th>...</th>\n",
       "      <th>DeviceProtection</th>\n",
       "      <th>TechSupport</th>\n",
       "      <th>StreamingTV</th>\n",
       "      <th>StreamingMovies</th>\n",
       "      <th>Contract</th>\n",
       "      <th>PaperlessBilling</th>\n",
       "      <th>PaymentMethod</th>\n",
       "      <th>MonthlyCharges</th>\n",
       "      <th>TotalCharges</th>\n",
       "      <th>Churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7590-VHVEG</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>No</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>29.85</td>\n",
       "      <td>29.85</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5575-GNVDE</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>34</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>One year</td>\n",
       "      <td>No</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>56.95</td>\n",
       "      <td>1889.5</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3668-QPYBK</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>53.85</td>\n",
       "      <td>108.15</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7795-CFOCW</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>45</td>\n",
       "      <td>No</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>One year</td>\n",
       "      <td>No</td>\n",
       "      <td>Bank transfer (automatic)</td>\n",
       "      <td>42.30</td>\n",
       "      <td>1840.75</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9237-HQITU</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>2</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>No</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>70.70</td>\n",
       "      <td>151.65</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7038</th>\n",
       "      <td>6840-RESVB</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>24</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>One year</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>84.80</td>\n",
       "      <td>1990.5</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7039</th>\n",
       "      <td>2234-XADUH</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>72</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>No</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>One year</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Credit card (automatic)</td>\n",
       "      <td>103.20</td>\n",
       "      <td>7362.9</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7040</th>\n",
       "      <td>4801-JZAZL</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>11</td>\n",
       "      <td>No</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>29.60</td>\n",
       "      <td>346.45</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7041</th>\n",
       "      <td>8361-LTMKD</td>\n",
       "      <td>Male</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>No</td>\n",
       "      <td>...</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>74.40</td>\n",
       "      <td>306.6</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7042</th>\n",
       "      <td>3186-AJIEK</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>66</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>Yes</td>\n",
       "      <td>...</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Two year</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Bank transfer (automatic)</td>\n",
       "      <td>105.65</td>\n",
       "      <td>6844.5</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7043 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      customerID  gender  SeniorCitizen Partner Dependents  tenure  \\\n",
       "0     7590-VHVEG  Female              0     Yes         No       1   \n",
       "1     5575-GNVDE    Male              0      No         No      34   \n",
       "2     3668-QPYBK    Male              0      No         No       2   \n",
       "3     7795-CFOCW    Male              0      No         No      45   \n",
       "4     9237-HQITU  Female              0      No         No       2   \n",
       "...          ...     ...            ...     ...        ...     ...   \n",
       "7038  6840-RESVB    Male              0     Yes        Yes      24   \n",
       "7039  2234-XADUH  Female              0     Yes        Yes      72   \n",
       "7040  4801-JZAZL  Female              0     Yes        Yes      11   \n",
       "7041  8361-LTMKD    Male              1     Yes         No       4   \n",
       "7042  3186-AJIEK    Male              0      No         No      66   \n",
       "\n",
       "     PhoneService     MultipleLines InternetService OnlineSecurity  ...  \\\n",
       "0              No  No phone service             DSL             No  ...   \n",
       "1             Yes                No             DSL            Yes  ...   \n",
       "2             Yes                No             DSL            Yes  ...   \n",
       "3              No  No phone service             DSL            Yes  ...   \n",
       "4             Yes                No     Fiber optic             No  ...   \n",
       "...           ...               ...             ...            ...  ...   \n",
       "7038          Yes               Yes             DSL            Yes  ...   \n",
       "7039          Yes               Yes     Fiber optic             No  ...   \n",
       "7040           No  No phone service             DSL            Yes  ...   \n",
       "7041          Yes               Yes     Fiber optic             No  ...   \n",
       "7042          Yes                No     Fiber optic            Yes  ...   \n",
       "\n",
       "     DeviceProtection TechSupport StreamingTV StreamingMovies        Contract  \\\n",
       "0                  No          No          No              No  Month-to-month   \n",
       "1                 Yes          No          No              No        One year   \n",
       "2                  No          No          No              No  Month-to-month   \n",
       "3                 Yes         Yes          No              No        One year   \n",
       "4                  No          No          No              No  Month-to-month   \n",
       "...               ...         ...         ...             ...             ...   \n",
       "7038              Yes         Yes         Yes             Yes        One year   \n",
       "7039              Yes          No         Yes             Yes        One year   \n",
       "7040               No          No          No              No  Month-to-month   \n",
       "7041               No          No          No              No  Month-to-month   \n",
       "7042              Yes         Yes         Yes             Yes        Two year   \n",
       "\n",
       "     PaperlessBilling              PaymentMethod MonthlyCharges  TotalCharges  \\\n",
       "0                 Yes           Electronic check          29.85         29.85   \n",
       "1                  No               Mailed check          56.95        1889.5   \n",
       "2                 Yes               Mailed check          53.85        108.15   \n",
       "3                  No  Bank transfer (automatic)          42.30       1840.75   \n",
       "4                 Yes           Electronic check          70.70        151.65   \n",
       "...               ...                        ...            ...           ...   \n",
       "7038              Yes               Mailed check          84.80        1990.5   \n",
       "7039              Yes    Credit card (automatic)         103.20        7362.9   \n",
       "7040              Yes           Electronic check          29.60        346.45   \n",
       "7041              Yes               Mailed check          74.40         306.6   \n",
       "7042              Yes  Bank transfer (automatic)         105.65        6844.5   \n",
       "\n",
       "     Churn  \n",
       "0       No  \n",
       "1       No  \n",
       "2      Yes  \n",
       "3       No  \n",
       "4      Yes  \n",
       "...    ...  \n",
       "7038    No  \n",
       "7039    No  \n",
       "7040    No  \n",
       "7041   Yes  \n",
       "7042    No  \n",
       "\n",
       "[7043 rows x 21 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:19.860117Z",
     "iopub.status.busy": "2021-10-01T10:50:19.859798Z",
     "iopub.status.idle": "2021-10-01T10:50:20.087279Z",
     "shell.execute_reply": "2021-10-01T10:50:20.086397Z",
     "shell.execute_reply.started": "2021-10-01T10:50:19.860076Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerid</th>\n",
       "      <th>gender</th>\n",
       "      <th>seniorcitizen</th>\n",
       "      <th>partner</th>\n",
       "      <th>dependents</th>\n",
       "      <th>tenure</th>\n",
       "      <th>phoneservice</th>\n",
       "      <th>multiplelines</th>\n",
       "      <th>internetservice</th>\n",
       "      <th>onlinesecurity</th>\n",
       "      <th>...</th>\n",
       "      <th>deviceprotection</th>\n",
       "      <th>techsupport</th>\n",
       "      <th>streamingtv</th>\n",
       "      <th>streamingmovies</th>\n",
       "      <th>contract</th>\n",
       "      <th>paperlessbilling</th>\n",
       "      <th>paymentmethod</th>\n",
       "      <th>monthlycharges</th>\n",
       "      <th>totalcharges</th>\n",
       "      <th>churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7590-vhveg</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>1</td>\n",
       "      <td>no</td>\n",
       "      <td>no_phone_service</td>\n",
       "      <td>dsl</td>\n",
       "      <td>no</td>\n",
       "      <td>...</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>month-to-month</td>\n",
       "      <td>yes</td>\n",
       "      <td>electronic_check</td>\n",
       "      <td>29.85</td>\n",
       "      <td>29.85</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5575-gnvde</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>34</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>dsl</td>\n",
       "      <td>yes</td>\n",
       "      <td>...</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>one_year</td>\n",
       "      <td>no</td>\n",
       "      <td>mailed_check</td>\n",
       "      <td>56.95</td>\n",
       "      <td>1889.50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3668-qpybk</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>2</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>dsl</td>\n",
       "      <td>yes</td>\n",
       "      <td>...</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>month-to-month</td>\n",
       "      <td>yes</td>\n",
       "      <td>mailed_check</td>\n",
       "      <td>53.85</td>\n",
       "      <td>108.15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7795-cfocw</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>45</td>\n",
       "      <td>no</td>\n",
       "      <td>no_phone_service</td>\n",
       "      <td>dsl</td>\n",
       "      <td>yes</td>\n",
       "      <td>...</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>one_year</td>\n",
       "      <td>no</td>\n",
       "      <td>bank_transfer_(automatic)</td>\n",
       "      <td>42.30</td>\n",
       "      <td>1840.75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9237-hqitu</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>2</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>fiber_optic</td>\n",
       "      <td>no</td>\n",
       "      <td>...</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>month-to-month</td>\n",
       "      <td>yes</td>\n",
       "      <td>electronic_check</td>\n",
       "      <td>70.70</td>\n",
       "      <td>151.65</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7038</th>\n",
       "      <td>6840-resvb</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>24</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>dsl</td>\n",
       "      <td>yes</td>\n",
       "      <td>...</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>one_year</td>\n",
       "      <td>yes</td>\n",
       "      <td>mailed_check</td>\n",
       "      <td>84.80</td>\n",
       "      <td>1990.50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7039</th>\n",
       "      <td>2234-xaduh</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>72</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>fiber_optic</td>\n",
       "      <td>no</td>\n",
       "      <td>...</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>one_year</td>\n",
       "      <td>yes</td>\n",
       "      <td>credit_card_(automatic)</td>\n",
       "      <td>103.20</td>\n",
       "      <td>7362.90</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7040</th>\n",
       "      <td>4801-jzazl</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>11</td>\n",
       "      <td>no</td>\n",
       "      <td>no_phone_service</td>\n",
       "      <td>dsl</td>\n",
       "      <td>yes</td>\n",
       "      <td>...</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>month-to-month</td>\n",
       "      <td>yes</td>\n",
       "      <td>electronic_check</td>\n",
       "      <td>29.60</td>\n",
       "      <td>346.45</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7041</th>\n",
       "      <td>8361-ltmkd</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>4</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>fiber_optic</td>\n",
       "      <td>no</td>\n",
       "      <td>...</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>month-to-month</td>\n",
       "      <td>yes</td>\n",
       "      <td>mailed_check</td>\n",
       "      <td>74.40</td>\n",
       "      <td>306.60</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7042</th>\n",
       "      <td>3186-ajiek</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>66</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>fiber_optic</td>\n",
       "      <td>yes</td>\n",
       "      <td>...</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>two_year</td>\n",
       "      <td>yes</td>\n",
       "      <td>bank_transfer_(automatic)</td>\n",
       "      <td>105.65</td>\n",
       "      <td>6844.50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7043 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      customerid  gender  seniorcitizen partner dependents  tenure  \\\n",
       "0     7590-vhveg  female              0     yes         no       1   \n",
       "1     5575-gnvde    male              0      no         no      34   \n",
       "2     3668-qpybk    male              0      no         no       2   \n",
       "3     7795-cfocw    male              0      no         no      45   \n",
       "4     9237-hqitu  female              0      no         no       2   \n",
       "...          ...     ...            ...     ...        ...     ...   \n",
       "7038  6840-resvb    male              0     yes        yes      24   \n",
       "7039  2234-xaduh  female              0     yes        yes      72   \n",
       "7040  4801-jzazl  female              0     yes        yes      11   \n",
       "7041  8361-ltmkd    male              1     yes         no       4   \n",
       "7042  3186-ajiek    male              0      no         no      66   \n",
       "\n",
       "     phoneservice     multiplelines internetservice onlinesecurity  ...  \\\n",
       "0              no  no_phone_service             dsl             no  ...   \n",
       "1             yes                no             dsl            yes  ...   \n",
       "2             yes                no             dsl            yes  ...   \n",
       "3              no  no_phone_service             dsl            yes  ...   \n",
       "4             yes                no     fiber_optic             no  ...   \n",
       "...           ...               ...             ...            ...  ...   \n",
       "7038          yes               yes             dsl            yes  ...   \n",
       "7039          yes               yes     fiber_optic             no  ...   \n",
       "7040           no  no_phone_service             dsl            yes  ...   \n",
       "7041          yes               yes     fiber_optic             no  ...   \n",
       "7042          yes                no     fiber_optic            yes  ...   \n",
       "\n",
       "     deviceprotection techsupport streamingtv streamingmovies        contract  \\\n",
       "0                  no          no          no              no  month-to-month   \n",
       "1                 yes          no          no              no        one_year   \n",
       "2                  no          no          no              no  month-to-month   \n",
       "3                 yes         yes          no              no        one_year   \n",
       "4                  no          no          no              no  month-to-month   \n",
       "...               ...         ...         ...             ...             ...   \n",
       "7038              yes         yes         yes             yes        one_year   \n",
       "7039              yes          no         yes             yes        one_year   \n",
       "7040               no          no          no              no  month-to-month   \n",
       "7041               no          no          no              no  month-to-month   \n",
       "7042              yes         yes         yes             yes        two_year   \n",
       "\n",
       "     paperlessbilling              paymentmethod monthlycharges  totalcharges  \\\n",
       "0                 yes           electronic_check          29.85         29.85   \n",
       "1                  no               mailed_check          56.95       1889.50   \n",
       "2                 yes               mailed_check          53.85        108.15   \n",
       "3                  no  bank_transfer_(automatic)          42.30       1840.75   \n",
       "4                 yes           electronic_check          70.70        151.65   \n",
       "...               ...                        ...            ...           ...   \n",
       "7038              yes               mailed_check          84.80       1990.50   \n",
       "7039              yes    credit_card_(automatic)         103.20       7362.90   \n",
       "7040              yes           electronic_check          29.60        346.45   \n",
       "7041              yes               mailed_check          74.40        306.60   \n",
       "7042              yes  bank_transfer_(automatic)         105.65       6844.50   \n",
       "\n",
       "      churn  \n",
       "0         0  \n",
       "1         0  \n",
       "2         1  \n",
       "3         0  \n",
       "4         1  \n",
       "...     ...  \n",
       "7038      0  \n",
       "7039      0  \n",
       "7040      0  \n",
       "7041      1  \n",
       "7042      0  \n",
       "\n",
       "[7043 rows x 21 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns = data.columns.str.lower().str.replace(' ','_')\n",
    "\n",
    "categorical_cols = list(data.dtypes[data.dtypes == 'object'].index)\n",
    "for c in categorical_cols:\n",
    "    data[c] = data[c].str.lower().str.replace(' ','_')\n",
    "\n",
    "data['totalcharges'] = pd.to_numeric(data['totalcharges'],errors='coerce')\n",
    "data['totalcharges'] = data['totalcharges'].fillna(0)\n",
    "\n",
    "data['churn'] = (data['churn'] == 'yes').astype(int)\n",
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:20.089255Z",
     "iopub.status.busy": "2021-10-01T10:50:20.088625Z",
     "iopub.status.idle": "2021-10-01T10:50:20.661865Z",
     "shell.execute_reply": "2021-10-01T10:50:20.661001Z",
     "shell.execute_reply.started": "2021-10-01T10:50:20.08921Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8034066713981547"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full_train, df_test = train_test_split(data,test_size=0.2,random_state=1)\n",
    "df_train, df_val = train_test_split(df_full_train,test_size=0.25,random_state=1)\n",
    "\n",
    "df_train.reset_index(drop=True)\n",
    "df_val.reset_index(drop=True)\n",
    "df_test.reset_index(drop=True)\n",
    "\n",
    "y_train = df_train['churn'].values\n",
    "y_val = df_val['churn'].values\n",
    "y_test = df_test['churn'].values\n",
    "\n",
    "del df_train['churn']\n",
    "del df_val['churn']\n",
    "del df_test['churn']\n",
    "\n",
    "categorical_cols = ['gender', 'seniorcitizen', 'partner', 'dependents','phoneservice', \n",
    "                    'multiplelines', 'internetservice','onlinesecurity', 'onlinebackup',\n",
    "                    'deviceprotection', 'techsupport','streamingtv', 'streamingmovies', \n",
    "                    'contract', 'paperlessbilling','paymentmethod']\n",
    "\n",
    "numerical_cols = ['tenure','monthlycharges','totalcharges']\n",
    "\n",
    "use_features = categorical_cols + numerical_cols\n",
    "\n",
    "dv=DictVectorizer(sparse=False)\n",
    "\n",
    "dict_train = df_train[use_features].to_dict(orient='records')\n",
    "X_train = dv.fit_transform(dict_train)\n",
    "\n",
    "dict_val = df_val[use_features].to_dict(orient='records')\n",
    "X_val = dv.transform(dict_val)\n",
    "\n",
    "model = LogisticRegression()\n",
    "model.fit(X_train,y_train)\n",
    "\n",
    "y_pred = model.predict_proba(X_val)[:,1]\n",
    "churn_decision = (y_pred >= 0.5)\n",
    "\n",
    "(y_val == churn_decision).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1.1 Precision and Recall of dummy classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us take a dummy model, where we do not do any actual predictions, but take all output as zeros and see what the precision and recall will be for the dummy model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.007455Z",
     "iopub.status.busy": "2021-10-01T10:50:21.006803Z",
     "iopub.status.idle": "2021-10-01T10:50:21.017882Z",
     "shell.execute_reply": "2021-10-01T10:50:21.016773Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.007423Z"
    }
   },
   "outputs": [],
   "source": [
    "y_zeros = np.zeros(len(y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.029523Z",
     "iopub.status.busy": "2021-10-01T10:50:21.029209Z",
     "iopub.status.idle": "2021-10-01T10:50:21.037868Z",
     "shell.execute_reply": "2021-10-01T10:50:21.037125Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.029487Z"
    }
   },
   "outputs": [],
   "source": [
    "actual_positive = (y_val == 1)\n",
    "actual_negative = (y_val == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.040968Z",
     "iopub.status.busy": "2021-10-01T10:50:21.040706Z",
     "iopub.status.idle": "2021-10-01T10:50:21.05087Z",
     "shell.execute_reply": "2021-10-01T10:50:21.049984Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.040933Z"
    }
   },
   "outputs": [],
   "source": [
    "predicted_positive = (y_zeros == 1)\n",
    "predicted_negative = (y_zeros == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.065208Z",
     "iopub.status.busy": "2021-10-01T10:50:21.064647Z",
     "iopub.status.idle": "2021-10-01T10:50:21.074727Z",
     "shell.execute_reply": "2021-10-01T10:50:21.07406Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.065176Z"
    }
   },
   "outputs": [],
   "source": [
    "tp = (actual_positive & predicted_positive).sum()\n",
    "tn = (actual_negative & predicted_negative).sum()\n",
    "fp = (predicted_positive & actual_negative).sum()\n",
    "fn = (predicted_negative & actual_positive).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion Matrix:\n",
    "\n",
    "         tn  |  fp\n",
    "         _________\n",
    "         fn  |  tp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.117036Z",
     "iopub.status.busy": "2021-10-01T10:50:21.116266Z",
     "iopub.status.idle": "2021-10-01T10:50:21.128256Z",
     "shell.execute_reply": "2021-10-01T10:50:21.127411Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.116996Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1023,    0],\n",
       "       [ 386,    0]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix = np.array([\n",
    "    [tn, fp],\n",
    "    [fn, tp]\n",
    "])\n",
    "\n",
    "confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.129738Z",
     "iopub.status.busy": "2021-10-01T10:50:21.129514Z",
     "iopub.status.idle": "2021-10-01T10:50:21.140944Z",
     "shell.execute_reply": "2021-10-01T10:50:21.140053Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.129712Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.73, 0.  ],\n",
       "       [0.27, 0.  ]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(confusion_matrix / confusion_matrix.sum()).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1 Precision and Recall for dummy mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.154092Z",
     "iopub.status.busy": "2021-10-01T10:50:21.153557Z",
     "iopub.status.idle": "2021-10-01T10:50:21.164515Z",
     "shell.execute_reply": "2021-10-01T10:50:21.163714Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.154062Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-12-14cf9d9238e4>:1: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  precision = tp / (tp + fp)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = tp / (tp + fp)\n",
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.16593Z",
     "iopub.status.busy": "2021-10-01T10:50:21.16571Z",
     "iopub.status.idle": "2021-10-01T10:50:21.177107Z",
     "shell.execute_reply": "2021-10-01T10:50:21.176267Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.165904Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Recall = looks at actual positives. fraction of correctly identified positives = tp / actual positives\n",
    "recall = tp / (tp + fn)\n",
    "recall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1.2 F1 score\n",
    "\n",
    "f1 = 2.P.R/(P+R)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#f1 of dummy model\n",
    "f1 = 2*precision*recall/(precision+recall)\n",
    "f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using the predictions on genuine model\n",
    "t = 0.5\n",
    "predicted_positive = (y_pred >= t)\n",
    "predicted_negative = (y_pred < t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.178668Z",
     "iopub.status.busy": "2021-10-01T10:50:21.178257Z",
     "iopub.status.idle": "2021-10-01T10:50:21.184628Z",
     "shell.execute_reply": "2021-10-01T10:50:21.184114Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.178637Z"
    }
   },
   "outputs": [],
   "source": [
    "tp = (actual_positive & predicted_positive).sum()\n",
    "tn = (actual_negative & predicted_negative).sum()\n",
    "fp = (predicted_positive & actual_negative).sum()\n",
    "fn = (predicted_negative & actual_positive).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.186281Z",
     "iopub.status.busy": "2021-10-01T10:50:21.185859Z",
     "iopub.status.idle": "2021-10-01T10:50:21.197064Z",
     "shell.execute_reply": "2021-10-01T10:50:21.196181Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.18625Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6752411575562701"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = tp / (tp + fp)\n",
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.198627Z",
     "iopub.status.busy": "2021-10-01T10:50:21.198391Z",
     "iopub.status.idle": "2021-10-01T10:50:21.208863Z",
     "shell.execute_reply": "2021-10-01T10:50:21.208086Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.198599Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5440414507772021"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall = tp / (tp + fn)\n",
    "recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6025824964131995"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#f1 score of real model\n",
    "f1 = 2*precision*recall/(precision+recall)\n",
    "f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.21064Z",
     "iopub.status.busy": "2021-10-01T10:50:21.209962Z",
     "iopub.status.idle": "2021-10-01T10:50:21.223582Z",
     "shell.execute_reply": "2021-10-01T10:50:21.222704Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.210597Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-20-8cc88ab28541>:16: RuntimeWarning: invalid value encountered in long_scalars\n",
      "  precision = tp / (tp + fp)\n"
     ]
    }
   ],
   "source": [
    "thresholds = np.linspace(0,1,101)\n",
    "\n",
    "scores = []\n",
    "\n",
    "for t in thresholds:\n",
    "    actual_positive = (y_val == 1)\n",
    "    actual_negative = (y_val == 0)\n",
    "    \n",
    "    predicted_positive = (y_pred >= t)\n",
    "    predicted_negative = (y_pred < t)\n",
    "    \n",
    "    tp = (actual_positive & predicted_positive).sum()\n",
    "    tn = (actual_negative & predicted_negative).sum()\n",
    "    fp = (predicted_positive & actual_negative).sum()\n",
    "    fn = (predicted_negative & actual_positive).sum()\n",
    "    precision = tp / (tp + fp)\n",
    "    recall = tp / (tp + fn)\n",
    "    f1 = 2*precision*recall/(precision+recall)\n",
    "    \n",
    "#     tpr = tp / (tp + fn)\n",
    "#     fpr = fp / (fp + tn)\n",
    "    \n",
    "    scores.append((t, tp, fp, fn, tn, precision, recall, f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.251218Z",
     "iopub.status.busy": "2021-10-01T10:50:21.250827Z",
     "iopub.status.idle": "2021-10-01T10:50:21.257849Z",
     "shell.execute_reply": "2021-10-01T10:50:21.256946Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.251176Z"
    }
   },
   "outputs": [],
   "source": [
    "columns = ['threshold','tp','fp','fn','tn', 'precision', 'recall', 'f1']\n",
    "df_scores = pd.DataFrame(scores, columns=columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.290165Z",
     "iopub.status.busy": "2021-10-01T10:50:21.289313Z",
     "iopub.status.idle": "2021-10-01T10:50:21.312499Z",
     "shell.execute_reply": "2021-10-01T10:50:21.311853Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.290125Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>tp</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tn</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>386</td>\n",
       "      <td>1023</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.273953</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.430084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.1</td>\n",
       "      <td>366</td>\n",
       "      <td>556</td>\n",
       "      <td>20</td>\n",
       "      <td>467</td>\n",
       "      <td>0.396963</td>\n",
       "      <td>0.948187</td>\n",
       "      <td>0.559633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.2</td>\n",
       "      <td>333</td>\n",
       "      <td>356</td>\n",
       "      <td>53</td>\n",
       "      <td>667</td>\n",
       "      <td>0.483309</td>\n",
       "      <td>0.862694</td>\n",
       "      <td>0.619535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.3</td>\n",
       "      <td>284</td>\n",
       "      <td>236</td>\n",
       "      <td>102</td>\n",
       "      <td>787</td>\n",
       "      <td>0.546154</td>\n",
       "      <td>0.735751</td>\n",
       "      <td>0.626932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.4</td>\n",
       "      <td>249</td>\n",
       "      <td>166</td>\n",
       "      <td>137</td>\n",
       "      <td>857</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.645078</td>\n",
       "      <td>0.621723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.5</td>\n",
       "      <td>210</td>\n",
       "      <td>101</td>\n",
       "      <td>176</td>\n",
       "      <td>922</td>\n",
       "      <td>0.675241</td>\n",
       "      <td>0.544041</td>\n",
       "      <td>0.602582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>150</td>\n",
       "      <td>53</td>\n",
       "      <td>236</td>\n",
       "      <td>970</td>\n",
       "      <td>0.738916</td>\n",
       "      <td>0.388601</td>\n",
       "      <td>0.509338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.7</td>\n",
       "      <td>76</td>\n",
       "      <td>20</td>\n",
       "      <td>310</td>\n",
       "      <td>1003</td>\n",
       "      <td>0.791667</td>\n",
       "      <td>0.196891</td>\n",
       "      <td>0.315353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.8</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>373</td>\n",
       "      <td>1022</td>\n",
       "      <td>0.928571</td>\n",
       "      <td>0.033679</td>\n",
       "      <td>0.065000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>386</td>\n",
       "      <td>1023</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>386</td>\n",
       "      <td>1023</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     threshold   tp    fp   fn    tn  precision    recall        f1\n",
       "0          0.0  386  1023    0     0   0.273953  1.000000  0.430084\n",
       "10         0.1  366   556   20   467   0.396963  0.948187  0.559633\n",
       "20         0.2  333   356   53   667   0.483309  0.862694  0.619535\n",
       "30         0.3  284   236  102   787   0.546154  0.735751  0.626932\n",
       "40         0.4  249   166  137   857   0.600000  0.645078  0.621723\n",
       "50         0.5  210   101  176   922   0.675241  0.544041  0.602582\n",
       "60         0.6  150    53  236   970   0.738916  0.388601  0.509338\n",
       "70         0.7   76    20  310  1003   0.791667  0.196891  0.315353\n",
       "80         0.8   13     1  373  1022   0.928571  0.033679  0.065000\n",
       "90         0.9    0     0  386  1023        NaN  0.000000       NaN\n",
       "100        1.0    0     0  386  1023        NaN  0.000000       NaN"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_scores[::10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1.3 Plot of P and R at different thresholds for PR curve\n",
    "\n",
    "Evaluate precision and recall at different thresholds, plot P vs R - this way you'll get the precision/recall curve (similar to ROC curve)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.314083Z",
     "iopub.status.busy": "2021-10-01T10:50:21.313692Z",
     "iopub.status.idle": "2021-10-01T10:50:21.507631Z",
     "shell.execute_reply": "2021-10-01T10:50:21.506854Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.314053Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f56aa518fd0>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABCOklEQVR4nO3dd3gU5fbA8e9JJ5SE3gIEkK50ASkCgjRRLKCAimBB7F4r3nu9itfrT716VSwgCiKgYqcpioAUAaVI71UIPYEUWiDJ+/tjJrjElCXZzWw5n+fZJzs7szNnZjd7Zt55ixhjUEopFbxCnA5AKaWUszQRKKVUkNNEoJRSQU4TgVJKBTlNBEopFeQ0ESilVJDTRODnRORWEZnjxnJjReTZ4oipsEQkXkSMiIQ5HUs2EVkgInfbz4eKyC9Ox+QOd78XXo6h2D5PezuXFOJ9+cYoIs+LyJSiR+jbNBF4kYjsEZHTInJCRA6LyEciUsqT2zDGfGKM6eHGciOMMf/25LaLW47jeUhEJnr6eHqLHeuLxbU9d78XnmR/Pt2Lc5vKMzQReN+1xphSQEvgcuCfORfwpTNgP5B9PJsDLYBnnA3H9/jr98lf4w4EmgiKiTFmPzAbuBTOX8o+ICLbge32a31FZI2IJIvIUhFpmv1+EakhIt+IyFERSRKRd+zXzxdXiOUNETkiIikisk5Esrd3wRmpiNwjIjtE5JiIzBCRai7zjIiMEJHtInJcRN4VEcltv0SkjYgss2M+KCLviEiEO+sSkVAReU1EEkVkF3DNRRzPQ8CPWAkhe1vt7OOWLCJrRaSLy7xy9hXZATuOafbrZUVkln1cj9vP49yNI8ex+NK+UkkRkUUi0sR+fThwK/CUfTUzM5f3jhWR13K8Nl1EHrOfjxSRnSKSJiKbROQGl+WGisgS+7M/BjwvOYqxRKS9iKywY1shIu1d5l1wJu9aHCIiUSIyxf7OJdvvrZxL/JOBmsBMex+fcpl9q4jstT/nf+TYzlf2+lOBoSISIyLj7e/SfhF5UURC7eUvEZGF9j4kisjnOcLonsf3LERE/ikif9j/G5NEJCaPz7C2vY00EfkJqOAyz61j4ZeMMfrw0gPYA3S3n9cANgL/tqcN8BNQDiiBdcVwBGgLhAJ32O+PtKfXAm8AJYEooKO9nqHAL/bznsAqIBYQoBFQ1Z43EXjRfn4VkGhvMxJ4G1jkErcBZtnrqQkcBXrlsY+tgHZAGBAPbAYedWddwAhgi31sygE/28uHuXE844D1wFv2dHUgCeiDdYJztT1d0Z7/HfA5UBYIBzrbr5cHbgKigdLAl8A0l20uAO7OeazziO9Oex2RwJvAGpd5549/Hu+9EtgHiD1dFjgNVLOnBwDV7H27BTjp8tkOBTKAh+zPoUSO70U54Dhwuz1/kD1dPudxtaefB6bYz+8FZtrHJ9T+vMsU9PnY0/H25/mBHVMzIB1o5LKdc8D19n6VAKYB72N9zysBy4F77eU/A/5hL3v+f8CN79mdwA6gDlAK+AaYnCPGMHt6GfA/+zO8EkgrzLHwt4fjAQTyw/7HOAEkA38A7wEl7HkGuMpl2THYScLlta1AZ+AK+4v9lx/IHP/wVwHbsH6YQ3IsN5E/E8F44FWXeaXsf8h4l9hc/8m+AEa6uc+PAt+6TOe5LmA+MMJlXg8KTgQn7H9OA8wDYu15T2f/c7ss/yNWQq0KZAFl3Yi/OXDcZXoBbiaCHOuJtWOMyXn881hegL3Alfb0PcD8fJZfA/RziWtvPt+L24HlOeYvA4a6HNe8EsGdwFKgqZvf99wSQZzLa8uBgS7bcT0BqYyVKEq4vDYI+Nl+PgkY57o+N79n84D7XeY1wPq+Z5+8GPt5TayEWtJl2U8Lcyz87aFFQ953vTEm1hhTyxhzvzHmtMu8fS7PawGP25ecySKSjHWmXM3++4cxJiO/DRlj5gPvAO8Ch0VknIiUyWXRaliJKft9J7DOnqu7LHPI5fkprGTxFyJS3y5OOWRf3r+Ey+V0AeuqxoXH4A8Kdr0xpjTQBWjosq1awIAcx68jVhKoARwzxhzPJf5oEXnfLjZIBRYBsdnFEe6yi7letotvUrF+FOGvxyJXxvqlmYr1wwcwGPjEZf1D5M9iw2SsIkbXdbsex5wu+Lxtf3Dh552XyVgJdapdrPaqiIS78T5X+X2Xcv4PhAMHXfbzfawrA4CnsBLmchHZKCJ3urmdnPv/B9YPf85inWpYJwEncyybzRPHwidpInCWa9ev+4D/2Ekj+xFtjPnMnldT3LiZZowZbYxpBTQB6gNP5rLYAax/OgBEpCRWEcn+QuzDGKzinXrGmDLA37H+Wd1xEOtHOltNdzdqjFmIdZadXa6+D+uKwPX4lTTGvGzPKycisbms6nGsM8S2dvxX2q+7uw/ZBgP9gO5ADNaZput63Onm9zOgv4jUwioi/BrAnv4AeBCrOCcW2JAjxvzWf8HnbavJn5/3SazijmxVzq/UmHPGmFHGmMZAe6AvMCSP7RSmK+Oc/wPpQAWXz7CMMaaJHcshY8w9xphqWMU074l7VUZz7n/2mf/hHMsdBMra/w+uy2Jv/2KOhV/RROA7PgBGiEhbsZQUkWtEpDTW5fRB4GX79SgR6ZBzBSJyuf3+cKx/7jNAZi7b+hQYJiLNRSQS6yz+N2PMnkLEXRpIBU6ISEPgvot47xfAwyISJyJlgZEXue03gatFpDkwBbhWRHraZ+dRItJFROKMMQexbtS/J9bN4XARyf7BL41VFp8sIuWA5y4yhmylsX7EkrB+VF/KMf8wVhl1nowxq7GKAD8EfjTGJNuzSmL9YB4FEJFh2JUO3PQ9UF9EBotImIjcAjTGKlMHq5hpoH1cWgP9s98oIl1F5DL7CikVq0glt++UW/uYH/tzmgO8LiJl7Ju8dUWksx3LAPnzRv5xrGOSVyyuPgP+Zt8ILoX12Xye8wrbGPMHsBIYJSIRItIRuDZ7/kUeC7+iicBHGGNWYpULv4P1Jd+BVc6LMSYT6wt5CVY5cgLWDcOcymAllONYl7RJ/HnG7LqtecCzWGecB4G6wMBChv4E1tlwmr3tnDU58vMB1qX2WuB3rJt4bjPGHMUqN37WGLMP64z871g/mPuwroayv+O3Y/3jbsG6Kf+o/fqbWDcpE4FfgR8uJgYXk7CO+X5gk70uV+OBxnaRx7R81vMZ1lXFp9kvGGM2Aa9jlesfBi4DlrgbmDEmCevs9XGs78RTQF9jTKK9yLNY34HjwCjXbWNdHXyF9cO3GViIlXRz83/AP+19fMLd+HIYAkRgHcPj9rar2vMuB34TkRPADOARY8xuN9Y5AatYZxGwG+sE6aE8lh2MdTV2DOukYJLLvIs5Fn4lu4aCUkqpIKVXBEopFeQ0ESilVJDTRKCUUkFOE4FSSgU5v+vkqUKFCiY+Pt7pMJRSyq+sWrUq0RhTMbd5fpcI4uPjWblypdNhKKWUXxGRPFvua9GQUkoFOU0ESikV5DQRKKVUkNNEoJRSQU4TgVJKBTmvJQIRmWAPC7chj/kiIqPFGi5xnYi09FYsSiml8ubNK4KJQK985vcG6tmP4Vj92iullCpmXmtHYIxZJCLx+SzSD5hkj8z0q4jEikhVu09yz0vcDuu+gEoNoVJjKFcXwiIKfp9Syqct2ZHIb7uScp1XOSaKW9vmHJNH5eRkg7LqXDhMXYL92l8SgYgMx7pqoGZNtwexutChdbD4NTBZ1nRIGJS/BCo2hEqN/vxbrg6EBsToc0oFhae/XkfC8dNILmPKNY2L1UTgBicTQW5DAeY6OIIxZhzWoNW0bt26cAMoXHoTNLgGErfB0S1wZLP19+Ba2DT9z02HRsIl3aDJDVC/F0TlNuSvUsoXHE1LJ+H4af7RpxH3XFnowdGCnpOJIIELx6uNwxpb1HvCo6BqU+vh6uypPxPEgdWweSZs/d5KCvWutpNCT4gs7dXwlFIXZ82+ZACa14x1NA5/52QimAE8KCJTsYaGS/Ha/YGCRERDtebWo9lA6Pl/kLACNn5jXS1smQVhUS5JoRdElCxorUopL1uz7zhhIcKl1WKcDsWveS0RiMhnQBeggogkYI3/GQ5gjBmLNaB2H6yxeU8Bw7wVy0ULCYGaba1Hz/+Dfb/9mRQ2z4SwEtDtX9DuPnItmFRKFYvVe5NpWLU0JSJCnQ7Fr3mz1tCgAuYb4AFvbd9jQkKg1hXWo9fLsHcZLBkNPz4DiVuhz2t6c1kpB2RmGdYlpHB9i2pOh+L3tGXxxQgJhfiOMGgqdHwMVk2EKTfCvhVwJtXp6JQKKjuPnuBEegbNa5R1OhS/53fjEfiEkBDo/hxUqAczH4Hx3a3Xy8RBnc7WfYTanbWdglJetGZvMgDNa8Q6Gkcg0ERQFM0HQ52uVk2jo5vh0AbYPAvWfAJRsdDydmj/CJTKdVAgpVQRrN6XTJmoMOpU0IobRaWJoKjKVLUeDftY0xnpsHM+rPsclr0LK8ZDm3ug2SBtzayUB63Zl0yzGrGEhGiFjaLSROBpYZHQoLf1SNwOC1+xbi4vectqzVyuLpSs8Ofyl3SDDn+zipuUUm45mZ7B1kOpXN31EqdDCQiaCLypQj246UPo+g+rXUJ2a+b0NGv+2RMw7wWrdfP1Y632DEqpAq3fn0KW0YZknqKJoDiUq209cjLGKj6a809I3gsDP7OKmZRS+TrfolhrDHmElkc4SQTaPwiDPoOj22BCD0jeV/D7lApya/YmU6t8NOVK6j03T9BE4Asa9Iahs+B0CnzcF1ISnI5IKZ+VkZnF8j3HaFlTrwY8RROBr6jeEm7/Fk4dg4l9IWW/0xEp5ZN+3XWMYyfP0rNJZadDCRiaCHxJXCsrGZxMhLEdYPHrkH7C6aiU8infrT9IdEQoXRpUcjqUgKGJwNfEtYa7foS4y60aRW81hV/HQFam05Ep5biMzCx+3HiIqxpWIipcO5rzFE0EvqhyE7j1S7hrLlS5DH4YCR/1hqSdTkemlKN+220VC/VtqrXrPEkTgS+rcTncPg1uGGe1PxjTAeaOsrrCTtyhVwkq6Mxap8VC3qDtCHydCDS7BWpfCbP+Br/87895UbHQqO+fndxpd9gqgGmxkPdoIvAXZarC4KlWq+Sj26xO7nYvho3TYfUUiK0FQ6bn3nBNqQCgxULeo0VD/iaytFW7qMVtcOP78OQOuHkypKfCx9fC8T+cjlApr9DaQt6jicDfhUdB4+usewnpaVYbhOS9TkellEdtO5zG9NX76d6oshYLeYEmgkBRrTkMmQbpKfDRNXBki9MRKeURx0+e5e6PV1IiIoyRvRs6HU5A0kQQSKq1sO4TZJyB8VfDjrlOR6RUkZzLzOL+T37nUMoZxg1pRbXYEk6HFJA0EQSaai3gnvkQWxM+GQDL3tNqpspv/ee7zSzblcRLN16mfQt5kSaCQBRbA+78Aer1hB+fgfeugA1fQ1aW05Ep5bZNB1KZuHQPQ9vH079VnNPhBDRNBIEqsjQM/BQGTLTaInx1J7x/JZw44nRkSrnlzbnbKB0Vxt+613c6lICniSCQhYRYjc3uWwo3fghJO2DqYDh32unIlMrX+oQU5mw6zN0d6xATrQ0lvU0TQTAICYWmA+DGcdaQmdPu12Ii5dPenLuNmBLhDOsY73QoQUFbFgeTxtdB9+dh7vNQvq41lrKI01GpAHPmXCYTluxm37HTpJ4+R1p6BtVjo7ikUmkaVilN29rlCAvN+xx09d7jzNtyhCd7NqBMlF4NFAdNBMGmw6NWEdGi/8LuRdDlGajTRROC8pjnZ2xk6op9VCgVSWx0ONERoaxPSOb4KWsY1svjy/L2oJZUiYkCIOXUOab89geJJ9IBWLYziXIlI7ijfbxTuxB0NBEEGxHo+5ZVzXTx/2Dy9VCrA1zzOlRq5HR0ys99uXIfU1fs4/4udXmq14WNvxJPpDNv82FGzdzENaMX898BTdl++ATv/ryDtPQMSkVaP0cCPN274flp5X1ijHE6hovSunVrs3LlSqfDCAwZ6fD7JFjwf1b3FF3/Du0ftu4pKHWRNh9M5fp3l9CyZlkm39Umz+KfHUfSuG/K72w/Yo2+17VBRZ7q1ZBGVcsUZ7hBR0RWGWNa5zpPE4HixFH47m/WOAdxl8OgqVCygtNRKT9yIj2DvqMXc+psJt893ImKpSPzXf7U2QzGL95Nq/iytK+r37XikF8i0FpDCkpVtHowvWk8HFpvVzE943RUyo+8PW87e5JO8fagFgUmAYDoiDAe6lZPk4CP8GoiEJFeIrJVRHaIyMhc5seIyEwRWSsiG0VkmDfjUfkQgcv6ww1jYd9vMONB8LOrReWM3YknmbBkNwNaxdG2Tnmnw1GF4LVEICKhwLtAb6AxMEhEGudY7AFgkzGmGdAFeF1EIrwVk3JDkxvgqmdh/Zew8BWno1F+4MVZm4gMC+XJXg2cDkUVkjdvy7cBdhhjdgGIyFSgH7DJZRkDlBYRAUoBx4AML8ak3NHpcUjaad1EDg2Hjo9p9VKVqwVbjzBvyxGe6d2QSqWjnA5HFZI3E0F1YJ/LdALQNscy7wAzgANAaeAWY8xfmryKyHBgOEDNmjW9EqxyIQLXvgWZZ2HeC5C43ZoOK7jsVwWPsxlZvDBrE7UrlGRYBx0i1Z958x5BbqeQOQudewJrgGpAc+AdEflLHTJjzDhjTGtjTOuKFSt6Ok6Vm7AIuOlDq/Xx2s+sYTDXfAYHVsPZU05Hp7zgzLlMNuxPIfXMuQKXTTl9jjsnrmDX0ZM827cREWFa78SfefOKIAGo4TIdh3Xm72oY8LKx6rDuEJHdQENguRfjUu4Sgc5PQflLYMbDMG2E9XpIONz0gXU/Qfm9zCzDtNX7eX3OVg6kWLXFqsVEUat8SSLDQwgLEUpFhtH+kgp0bVCJk+kZ3PXxCvYeO8WrNzXlqoaVHd4DVVTeTAQrgHoiUhvYDwwEBudYZi/QDVgsIpWBBsAuL8akCuPSG6HRdXBsFxzdDEvegukPQpWmVp9Fyi+dzchizqZDvPvzTjYfTKVpXAyP92jA4bQzbDuUxr7jpzl1NoOMLMPRtHSmrbHO46LCQygRHsqUu9pqLaEA4bVEYIzJEJEHgR+BUGCCMWajiIyw548F/g1MFJH12C3LjTGJ3opJFUFoGFSsbz2qt4KxHeHLO+CuuRCuNwn9Rcrpc2w5mMr8rUf4amUCSSfPUqt8NG8PasE1l1UlJCT3SgHGGDYfTGP+lsPsTjzFw90uoVb5ksUcvfIWbVmsCmfrbPhsILQZDn3+63Q0qgATl+zmg8W72Z9sjUURGiJ0b1SJQW1q0qleRULzSAAqcOTXslh7dVKF06A3XPEgLHsHoitYVU5D9evka4wxvDVvO2/O3U67OuW4rV0tGlUtzWXVYyhfSmuBKYv+56rC6/YcpB2CBS/Bttlw/RjtwdSHGGN4+YctvL9wF/1bxfHKTU31zF/lSut8qcILi4D+461xkZP3WmMib5zmdFRBzxjD0p2J3PXxSt5fuIvb2tXkVU0CKh96RaCKrskNUKsjfH4rfHsvxMRBXK5FkcqLsrIM09bs5/2Fu9h6OI2y0eE83ashIzrXQbRluMqHJgLlGaUqwsDP4MOrrJvId8+DsrWcjipo/LI9kZe+38ymg6k0rFKaV/s35bpm1YgK17ElVME0ESjPKVkeBn8J47vDp7fAXT9CVIzTUQWMORsPsffYKRpVLUOjqmVIOpHOT5sPM2fjYdbsSyaubAneGtica5tWy7MaqFK5yTMR2HX7c6tbKoAxxjT1WlTKf1WsDzdPgik3WfcM+r0H8R2cjsqvZWRm8dL3W5iwZHeu8y+rHsOzfRtzW7uaRIbpFYC6ePldEfQttihUYKnTBe6YCdPug4nXQNsR1jCYUToU4cVKPXOOhz5dzcJtRxnaPp77u9Rl2+ETbD6YSnRkKFc1rETVmBJOh6n8nDYoU95z9iTMfR6Wj4PQSKh3tXVjuX5PiCztdHQ+72R6BgPGLmPb4TRe6Hcpg9tqz7uq8ArVoExE0si/aEhP71T+IkparY6bDYJ1X8CmabBlFoRFuSSFXtZy6gJZWYbHvljDlkOpjL/jcro2rOR0SCqA5ZkIjDF6yqY8o3pL69HzJdj3q9XWYNM02DwTwkpA/R5WUqjXEyKinY7WJ7w1bzs/bjzMP69ppElAeZ3btYZEpBJwvncxY8xer0SkAldICNRqbz16/R/sXQYbv4VN061HeLR1hdDkBqjXIyg7s8vKMkxfu5+35m2nf6s47uqoA74o7yvwHoGIXAe8jjV4zBGgFrDZGNPE++H9ld4jCEBZmfDHEjspzIBTiVCxIQyaCuUC/4fQGMOSHUl8t/4g8zYf5khaOi1qxjJ1eDutBaQ8Jr97BO4kgrXAVcBcY0wLEekKDDLGDPd8qAXTRBDgMjNg2w8w/QEICYVbPoFaVzgdlVdYXUEk8dqcrazem0zJiFA6N6hIt4aV6X1ZFaIjtJmP8pyi9j56zhiTJCIhIhJijPlZRF7xcIxKWULDoFFfq/O6T2+GSddBl2eg6S0QU93p6Dxm19ET/HPaBpbuTKJqTBT/ueFS+reK0ysA5Qh3EkGyiJQCFgGfiMgRIMO7YamgV74u3D0Xvr4H5o2yHjXaQdt7rRHT/ETyqbNMWLKHstHhtK9bgdoVSvLB4l28NW87UWEhPHdtYwa1qaldQShHuZMI+gGngb8BtwIxwAveDEopAEqUhdu+gsQd1v2D9V/CV8PgwGro/rxVdOTD5m85zMiv13MkLf38axFhIZzNyKLPZVV4/romVCodfDfEle9xJxFUAg4aY84AH4tICaAykOTVyJTKVuES6PwkdPwb/PA0LB0NSTvhxnEQWcrp6ACrvP/XXcfYe+wkSSfPsvFAKt+tO0iDyqWZMPRyYkqEs2xXEmv2JdO5fkV6NqnidMhKnefOzeKVQHtjzFl7OgJYYoy5vBji+wu9Waz4bZyVEKq3gqHfW+MiOOx/c7Yyev6O89MlI0K5o308j3Svp+X+yicU9WZxWHYSADDGnLWTgVLOaDscSlawionmPg+9XnI0nLfnbWf0/B3c3DqOR7rXp1x0BCUi9Mdf+Q93Rig7arclAEBE+gGJ3gtJKTdceiO0GQ6/vgtbvnckhMwsw3sLdvD6T9u4sWV1Xr6xKdVjS2gSUH7HnSuCEVi1hd7F6nsoARji1aiUckePF2Hfb1YvpyN+gdgaXt/kqbMZTF2+jyU7Elm+5xhpZzK4tlk1/tu/mY4BoPxWgYnAGLMTaGdXIRVjTJr3w1LKDWGR0P8jeL8zfNIfevwHLukGXhqW8fe9x3n8i7XsTjxJnQol6du0Gu3rlqf3pVV0PGDl1wpMBCJSGXgJqGaM6S0ijYErjDHjvR6dUgUpXxdunggzH4VPboK4NtD1GajT1SMJISvLsCfpJF//nsCYBTupGlOCT+9pS/u6FYq8bqV8hTu1hmYDHwH/MMY0E5EwYLUx5rLiCDAnrTWkcpVxFtZMgUWvQ2oC1LwCuoyE2p0LlRDWJSTz6g9bWZuQTNoZq/1k/1Zx/OvaxpSJCvd09Ep5XVFrDVUwxnwhIs8AGGMyRCTToxEqVVRhEdD6Tmh+K6yebCWESf2gZntrdLTandxe1fQ1+3nqq3XERodzXbNqNI2LoUXNstSvrD2zq8DkTiI4KSLlsQepEZF2QIpXo1KqsMIi4fK7oflt8Psk+OV/8HFfiO8EVz5p/Q3JvbJcVpbhfz9t452fd9AmvhxjbmtJ+VKRxbwDShU/d4qGWgJvA5cCG4CKQH9jzDrvh/dXWjSkLsq5M7BqopUQThyG0tWgcT9oNhCqNT+/WEZmFk99tY5vVu9n4OU1eKHfpUSEuVO7Win/UKRuqO0VhAENsIap3Aq0McYs8WiUbtJEoArl3GnYPMvqs2jHT5CVYY13UL8n5zKz+Nvna5i17iB/616fh7tdgnip5pFSTinsmMWhwM1AdWC2MWajiPQFxgElgBbeCFYprwgvAU0HQNMBLFq3jdrf30rlqUNZ2HEyXybEMmfTYUb2bsiIznWdjlSpYpffte944G6gPPC2iHwE/Bd41RjjVhIQkV4islVEdojIyDyW6SIia0Rko4gsvNgdUMpdqWfO8cSXaxny6XaGnXmMY5mRNFl4D6s3beG5axtrElBBK7+bxa2BpsaYLBGJwupW4hJjzCF3VmxfUbwLXI3VGnmFiMwwxmxyWSYWeA/oZYzZa4+LrFSRbT2UxnsLdrDqj+OULxlBxdKRbDyQypG0dB7segkPd6uHHGpE6MReLK70PlEt+jgdslKOyS8RnDXGZAEYY86IyDZ3k4CtDbDDGLMLQESmYo1tsMllmcHAN8aYvfZ2jlxU9ErlsPPoCV6ZvYU5mw5TMiKULg0qkZaewf7kM1SPLcHY21rRrEastXBccxgwkagvbocPu8PgL6wur5UKMvklgoYikl0zSIC69rQAxhjTtIB1Vwf2uUwnAG1zLFMfCBeRBUBp4C1jzKScKxKR4cBwgJo1axawWRWsVuw5xl0TVwDwSLd6DOsQT2x0AR3lNugFd8yEqYPhw6vg5slQp3MxRKuU78gvETQq4rpzq3aRs4pSGNAK6IZ1A3qZiPxqjNl2wZuMGYd1k5rWrVsXXM1JBZ2fNh3mwU9/p3psCT6+sw01ykW7/+aa7eCe+fDpLTDlRujzGrQe5r1glfIxeSYCY8wfRVx3AuDaHWQccCCXZRKNMSexGq4tApoB21AqHwnHT7FkRyIHks+w79gppq3Zz2VxsXw09HLKlSzEcBll4+GuOfDVnTDrUUjcZvVu6uPDYSrlCe60LC6sFUA9EakN7AcGYt0TcDUdeMdupxCBVXT0hhdjUn7OGMOny/fy4qzNnD6XiQhULBXJtc2q8dINl1Eysghf6agYGPQ5zPkH/PoeJO2AAR9DxEVcXSjlh7yWCOw+iR4EfgRCgQl2W4QR9vyxxpjNIvIDsA7IAj40xmzwVkzKfxlj2JN0ihdmbuTnrUfpVK8C/+rbmFrlS3q2BXBoGPR+BSrUg++fhM9usZKDJgMVwNxqWexLtGVx4EvPyGT74RNsPZTGtsNpbDyQyoYDKSSfOkdkWAjP9G7IkCvivT8QzNrP4dt7rZvHg6ZajdKU8lOFbVm8nr/e3D3PjVpDSrltx5ET/LTpMEt2JLJizzHSM7IAiAgNoV7lUvRqUoVLq8fQuX7Fi7sRXBTNbgGTZY2A9unN0OZeqNTIup+g9w5UAMmvaKiv/fcB++9k+++twCmvRaSCysGU07w+Zxtf/56AMdCgcmkGt61Jq1plaVilNPHlSxIW6mDnb80HWclg5iOwe5H1WkQpa6yDdvdrQlABwZ3eR5cYYzoU9Fpx0aIh/5dy6hwr9hzjlx2JTF2xl6wsuKN9Le7pVIdKZaKcDi936Sfg6FY4uhk2zYDtP0KNtnD9GGuUNKV8XFEHpikpIh2NMb/YK2sPlPRkgCowZWUZ9h47xeaDqWw7fIJtR9LYdiiNHUdPYIxV7NP7sio80aNB8RX3FFZkKYhrZT2a3wrrvoDZT8KYDtBuBLR/GKLLOR2lUoXizhVBK2ACEGO/lAzcaYz53buh5U6vCHxTVpZhwbYjbDqQyq7Ek+xOPMm2Q2mcPGsNZicCNctFU69SKZrFxdKmdjma1YglKtyPi1ZSD8Kcf8KGryGiJLS9F654UBOC8klFHo/AXkkZe3lHRyfTROB7lu5M5KXvN7NhfyoAVcpEEV8hmgaVS9O4WhkaVS1DvUqlKRHhxz/6+TmyGRa+AhunWfcP2t0HV9wPJco6HZlS5xUpEYhIJHATEI9LUZIx5gUPxug2TQS+wRjDb7uP8f7Cnfy89SjVY0vwRM/69GxShegIb7ZT9GGHN8HCl2HTdIiMsRJCu/ugRKzTkSlV5HsE07HGKF4FpHsyMOV/MjKz+Ob3/UxYspsth9KIjQ7nqV4NuLNDbf8u5vGEyo3h5klwaAMs+D8rKfw2xiouav+QtkNQPsudRBBnjOnl9UiUz/t1VxLPTd/I1sNpNKxSmlduuox+zatrAsipyqUw8BM4uA4WvAw//weObIL+H1k3S5TyMe4kgqUicpkxZr3Xo1E+Jz0jk992HeOLlfuYte6g3ad/S3o2qaLj+hakalMY9Cks/h/MGwW1r4TWdzodlVJ/4U4i6AgMFZHdWEVD7o5HoPzU0bR0Fmw9wvwtR1i07Sgnz2ZSIjyUR7rVY0TnuoF709dbOjwKe36B2SMhro11xaCUD3EnEfT2ehTKcTuPnuCHDYeYs/EQaxOsimGVy0TSr0V1ujeqRPu6FbQIqLBCQuCG92FsR/hyKNwzz+rpVCkfUWAiyB6XwB5P2EebfaqLlZVlWL8/hZ82HWbOpkNsO3wCgGY1Ynn86vp0bViJJtXKaPGPp5SqCDd9CJOug9cbQv1e0OQGaNDH6vFUKQcV+A0UkeuA14FqwBGgFrAZaOLd0JQ3nMvMYvKyPxi3aBeHUs8QInB5fDmev7YxPZpUoVqs1mzxmtqd4M45sOYT2DwDNn5jtVK+/j2nI1NBzp1TkX8D7YC5xpgWItIVGOTdsJQ3LN5+lFEzN7HjyAna1y3PU70a0LVBJcoWZkQvVTg1LrcefV6Dec/D0rehcT+o39PpyFQQcycRnDPGJIlIiIiEGGN+FpFXvB6Z8pjth9N4efYW5m05Qq3y0Xw4pDXdGlXSYh8nhYbBVc/C9rkw81F44Fe9b6Ac404iSBaRUsAi4BMROQJkeDcs5Qm7E0/yweJdTF2+l5IRYTzdqyF3downMkxv+vqEsEi4/l34sDv8+A/o947TEakg5U4i6AecBv6GNRZBDOBI9xKqYAnHTzFz7UFmrj3ApoOphIUIQ66I5+Fu9Qo3qLvyruqtrJ5Ll7wJNdtBs8FWLSOlipEOVRkATqZnMGPtAb79fT/L9xwDoHmNWPo2rUrfptWoEqOVvXzauTPwUS84sBoqX2YNetPwGm2FrDyqqH0NKR9ljOG79Qf596xNHE5Np27FkjzZswHXNavm+/37qz+FR8Fdc2HDV1Yvpp/fCg37Wl1ShOlVnPI+TQR+anfiSZ6dtoFfdiTSpFoZ3hnckta1yuoNYH8VGgbNBsKl/WHZOzD3OfhqGAyYCKHhTkenApwmAj+TmWWY8MtuXpuzlYjQEEZd14Tb2tUiNEQTQEAIDYOOj0JYFPzwNHx1J/SfoMlAeZU7Dco6AM9jNSQL48++hup4NzSV04b9KTw7fQOr9ybTvVFl/nPDpVT21TF+VdG0GwEmC358Br4YAjd+YA2XqZQXuHNFMB6rxtAqINO74aiczmZkMXvDQSYt+4NVfxwnNjqctwY257pm1bQYKNBdcb91JTD7KZjQCwZPhZg4p6NSAcidRJBijJnt9UjUBc5lZvHlygTemb+dAylniC8fzT+vacSAVjWIidZigqDR5h4oGw9fDoMProJBU6F6S6ejUgHGnaEqXwZCgW9wGaFMB6/3jqwsw/S1+3njp+3sPXaK5jVieaR7PTrXq0iI3gcIXkc2wyc3Q8YZGPELlK7sdETKzxS1+mhb+6/rCgxwVVEDUxdyHQS+SbUyTBjamq4NtCsIBVRqBLd+AeO6wjf3wO3fQoi2EFee4U431F2LI5BgdjI9g6e/Xnd+BLA3b7HuAegVgLpApUbQ51WY8RD88j+48kmnI1IBwp1aQzHAc8CV9ksLgReMMSneDCxY7E06xT2TVrL9SBqPXV2f4VfW0QFgVN5a3A67F8HPL0HN9hDfwemIVABwp1OTCUAacLP9SAU+8mZQwWLhtqNc9+4vHEo9w8d3tuHhbvU0Caj8iUDfN6wbyLMehawspyNSAcCdewR1jTE3uUyPEpE1XoonKOw7doqXvt/M7A2HqF+5FB8MaU2t8iWdDkv5i8jS0OXv8M3dsP1HaKCjyaqiceeK4LSIdMyesBuYnXZn5SLSS0S2isgOERmZz3KXi0imiPR3Z73+KiMzizfnbqPb/xayYOtRHr+6PjMe7KhJQF28JtdDTA1YMtrpSFQAcOeK4D7gY/tegQDHgKEFvUlEQoF3gauBBGCFiMwwxmzKZblXgB8vLnT/ciTtDA99uprfdh/j2mbV+HufhlSN0WEhVSGFhkO7+62WxwkrIS7XWoFKuaXAKwJjzBpjTDOgKXCZMaaFMWatG+tuA+wwxuwyxpwFpmKNbZDTQ8DXWOMhB6SlOxO5ZvQvrE1I5vUBzXh7UAtNAqroWg6xRjVb8pbTkSg/l+cVgYjcZoyZIiKP5XgdAGPM/wpYd3Vgn8t0An+2ScheV3XgBqw2CZfnE8twYDhAzZo1C9is79h6KI3//riVuZsPU6dCSSbf1YaGVco4HZYKFJGloPVd8MsbkLQTytd1OiLlp/K7IsguuC6dx6MguVWCz9mM+U3gaWNMvn0YGWPGGWNaG2NaV6xY0Y1NO+tEegZPfbWWXm8t4rddSTzRoz6zHu6oSUB5Xtt7rWKiha+Cnw0ypXxHnlcExpj37b+jCrnuBKCGy3QccCDHMq2BqfZVRgWgj4hkGGOmFXKbjtt2OI37pqxid+JJ7ulUh/u71CU2WgcXUV5Sugq0u88qHjKZcN071kA3Sl0EdxqUvQq8iFVT6AegGfCoMWZKAW9dAdQTkdrAfmAgMNh1AWNMbZftTARm+WsSMMbw7er9/OPbDZSMDGXK3W1pX7eC02GpYNB9lHWvYN4LcPwPGPgJlKrkdFTKj7hTfbSHMSYV6It1ll8fKLBtuzEmA3gQqzbQZuALY8xGERkhIiOKELPPOZB8mrs/XsljX6zlsuoxfPdwJ00CqviIQKfH4ebJcGg9jO0E2wK6Ep7yMHeqj2b3edwH+MwYc8zdTtCMMd8D3+d4bWweyw51a6U+whjDvmOnmb3hIKPnbSfLwD+vacTQ9vGEhbqTX5XysMbXWS2OvxkOn94MzW+DXi9ZVwtK5cOdRDBTRLZgFQ3dLyIVgTPeDct3bT2UxsfL9rBw61H2J1vt6q6sX5H/XH+pDhivnFe1Kdy7EBa8DEvehH2/wfCfrdbISuWhwPEIAESkLJBqjMkUkWigjDHmkNejy4VT4xEs3ZnI+wt3sXDbUUqEh9KpXgU6XFKBDpeUp27FUtpVtPI9O+fDlJugyY1w04dWEZIKWoUaj0BErjLGzBeRG11ec13kG8+F6Ls2H0zlpe83s3h7IhVKRfJEj/rc2rYWZUtqTSDl4+peZfVJ9POLUKez1QBNqVzkVzTUGZgPXJvLPEOAJ4JTZzN4YeYmPl+5jzJR4fzzmkbc1q6W9g6q/Eunx2DPYvj+KajeGio3djoi5YPcKhryJcVRNJRy6hzDJi5nzb5khnWozUNXXaJtAZT/SjsMYztCibLW/YII7eQwGOVXNFRg9RYReUlEYl2my4rIix6Mz6ccST3DLeOWsWF/Ku/d2pJn+zbWJKD8W+nKcNMHkLjNujJQKgd36jn2NsYkZ08YY45jVSUNOAdTTjPg/WXsPXaKCUMvp9elVZ0OSSnPqNMFrnwC1kyBtVOdjkb5GHcSQaiIRGZPiEgJIDKf5f1Syqlz3DFhOUknzvLJ3W3pWE8bhKkA03mkNbzlrMcgcbvT0Sgf4k4imALME5G7RORO4CfgY++GVbzOnMvkro9XsCfxFOOGtKJFzbJOh6SU54WGQf/xEBYJXwyBU8ecjkj5CHfGI8jua6gR0AT4t/1aQMjIzOLBT1ezau9x3riluXYNoQJbmWrQfwIk7YDJN8DpZKcjUj7A3b4QNgM/GGMeBxaLSMA0U8weL2DUdU24pqneE1BBoG5XuGUKHN6oyUAB7tUaugf4Cnjffqk6MM2LMRWbWesO8P6iXdzerhZDroh3Ohylik/9nnCL3Und5BvgxFGnI1IOcueK4AGgA5AKYIzZDvh9H7dbD6Xx1FfraFWrLM/21UY2Kgg16G0lgyOb4YOr4PCmgt+jApI7iSDdHnMYABEJ468jjfmV1DPnGDFlFSUjw3jv1pZEhGlvoSpINegNw76HzLMwvgdsm+N0RMoB7vwCLhSRvwMlRORq4EtgpnfD8q4XZm5i77FTvHdrSyqX0dGcVJCr3hLumQ/lasPUQXBgjdMRqWLmTiJ4GjgKrAfuxRpf4J/eDMqb5m0+zFerErivc10ujy/ndDhK+YaY6nDHDIguD9MfgIyzBb9HBYx8E4GIhADrjTEfGGMGGGP628/9smgo+dRZnvlmPQ2rlOahbpc4HY5SvqVEWej7BhzeAL/8z+loVDHKNxEYY7KAtSJSs5ji8apRMzdx7ORZXhvQjMgw7UVUqb9oeA1c2h8W/RcObXA6GlVM3CkaqgpsFJF5IjIj++HtwDxtzsZDfLt6Pw9edQmXVteh+5TKU+9XrauD6fdDZobT0ahi4M5QlaO8HkUxuLR6DEPbx/NAVy0SUipfJctbyeCrYbD2M2h5u9MRKS/LczwCEYkCRgCXYN0oHm+Mcfz0wKmhKpUKKsbA+KshZT88/DuEl3A6IlVEhR2P4GOgNVYS6A287oXYlFK+SAS6j4K0A/DbWKejUV6WXyJobIy5zRjzPtAf6FRMMSmlfEF8B6jfCxa/oT2VBrj8EsG57Ce+UCSklHJAt+cgPVWrkwa4/BJBMxFJtR9pQNPs5yKSWlwBKqUcVLkxNB8Mv42DfSucjkZ5SZ6JwBgTaowpYz9KG2PCXJ6XKc4glVIO6v681fJ4yo2QoBU1ApH2tqaUyl+pSnDHLKv7ick3wP5VTkekPEwTgVKqYDHVYegsiC4Hk26AlASnI1IepIlAKeWemDi4/Vs4dwoW683jQKKJQCnlvnJ1oMVtsHqy1dhMBQSvJgIR6SUiW0Vkh4iMzGX+rSKyzn4sFZFm3oxHKeUBnR6zWh4vedPpSJSHeC0RiEgo8C5Wq+TGwCARyTkm5G6gszGmKfBvYJy34lFKeUhsTatK6aqPIfWg09EoD/DmFUEbYIcxZpc91OVUoJ/rAsaYpcaY4/bkr0CcF+NRSnlKp8fAZOpVQYDwZiKoDuxzmU6wX8vLXcDs3GaIyHARWSkiK48ePerBEJVShVI2HpoNhFUTIe2Q09GoIvJmIpBcXsu1q1MR6YqVCJ7Obb4xZpwxprUxpnXFihU9GKJSqtA6PW4Neq+d0vk9byaCBKCGy3QccCDnQiLSFPgQ6GeMSfJiPEopTypXBxpdCysmQHqa09GoIvBmIlgB1BOR2iISAQwELhjZzB4C8xvgdmPMNi/GopTyhvaPQHoK/D7J6UhUEXgtEdg9lj4I/AhsBr4wxmwUkREiMsJe7F9AeeA9EVkjItqRiVL+JK4V1OoAy96DzHMFL698Up4jlPkqHaFMKR+z9Qf47Ba48QNoerPT0ag8FHaEMqWUKli9HlCxISx5y2popvyOJgKlVNGEhED7h+DwBvh1jNPRqELQRKCUKrqmA6FhX/jxGVj+gdPRqIukiUApVXShYdD/I2jQB75/AlaMdzoidRE0ESilPCMsAgZ8bA14/91jsG2O0xEpN2kiUEp5TlgE3DwJKjSA2U/CuTNOR6TcEOZ0AJ5w7tw5EhISOHNGv3SFERUVRVxcHOHh4U6HogJBWCT0fgUmXw9L34bOTzodkSpAQCSChIQESpcuTXx8PCK5dXGk8mKMISkpiYSEBGrXru10OCpQ1O0KjfvB4teh2S1W19XKZwVE0dCZM2coX768JoFCEBHKly+vV1PK83r8x/r749+djUMVKCASAaBJoAj02CmviK0BVz4Om2fC+q+cjkblI2ASgVLKB7V/GGpeAd8Mh03TnY5G5UETgYeEhobSvHlzLr30UgYMGMCpU6eKvM5//etfzJ07N8/5Y8eOZdIk7fVR+bCwSLj1S4hrDV/daV0dKJ8TEJ3Obd68mUaNGjkUkaVUqVKcOHECgFtvvZVWrVrx2GOPnZ+fmZlJaGioU+EVyBeOoQpgZ1Jhyk1w4HcrMdS9yumIgk5+nc4FRK0hV6NmbmTTgVSPrrNxtTI8d20Tt5fv1KkT69atY8GCBYwaNYqqVauyZs0a1q9fz8iRI1mwYAHp6ek88MAD3HvvvQC8+uqrTJ48mZCQEHr37s3LL7/M0KFD6du3L/3792fkyJHMmDGDsLAwevTowWuvvcbzzz9PqVKleOKJJ1izZg0jRozg1KlT1K1blwkTJlC2bFm6dOlC27Zt+fnnn0lOTmb8+PF06tTJo8dHqQJFlYHbvoIJveDre2DEL1CmqtNRKVvAJQKnZWRkMHv2bHr16gXA8uXL2bBhA7Vr12bcuHHExMSwYsUK0tPT6dChAz169GDLli1MmzaN3377jejoaI4dO3bBOo8dO8a3337Lli1bEBGSk5P/st0hQ4bw9ttv07lzZ/71r38xatQo3nzzzfMxLV++nO+//55Ro0blW9yklNdExcCAiTCuC3xzDwyZDiG+e5UcTAIuEVzMmbsnnT59mubNmwPWFcFdd93F0qVLadOmzfn6+XPmzGHdunV89ZVVgyIlJYXt27czd+5chg0bRnR0NADlypW7YN1lypQhKiqKu+++m2uuuYa+ffteMD8lJYXk5GQ6d+4MwB133MGAAQPOz7/xxhsBaNWqFXv27PH4vivltooN4JrXYdp9sPBV6PqM0xEpAjAROKVEiRKsWbPmL6+XLFny/HNjDG+//TY9e/a8YJkffvgh3yqcYWFhLF++nHnz5jF16lTeeecd5s+f73ZskZGRgHVDOyMjw+33KeUVzQfD7kWw8BUoEQut77RuKivHaK2hYtSzZ0/GjBnDuXPWkH7btm3j5MmT9OjRgwkTJpyvaZSzaOjEiROkpKTQp08f3nzzzb8knJiYGMqWLcvixYsBmDx58vmrA6V8Up/XIL4j/DASRreAFR9CxlmnowpaekVQjO6++2727NlDy5YtMcZQsWJFpk2bRq9evVizZg2tW7cmIiKCPn368NJLL51/X1paGv369ePMmTMYY3jjjTf+su6PP/74/M3iOnXq8NFHHxXnril1cSJLwR0zYdfP8PP/wXePw/qv4ZYpULK809EFHa0+qgA9hspBxsD6L2HGQ1CqMgz+Aio1dDqqgKNjFiulfJeINej90O/g3GkYfzXsWuh0VEFFE4FSyjfEtYZ75kNMHHx6iyaDYqSJQCnlO2JrWPcOytW2ksHuxU5HFBQ0ESilfEvJCjBkBpStBZ/eDNu1AaS3aSJQSvmeUhWtK4Oy8fDJTTDrb5B+wumoApYmAqWUbypVybpncMWDsPIjGHMFrPtSE4IXaCLwENduqK+99tpc+wMqivj4eBITEwGrp1OlgkJ4Cej5H7jzBwiNgG/uhv/Whc9vh/2rnI4uYGgi8JDsLiY2bNhAuXLlePfdd50OSanAUbMdPLAChs2GlkPgj6Uw6QY4utXpyAJC4LUsnj0SDq337DqrXAa9X3Z78SuuuIJ169YBsHPnTh544AGOHj1KdHQ0H3zwAQ0bNuTw4cOMGDGCXbt2ATBmzBjat2/P9ddfz759+zhz5gyPPPIIw4cP9+y+KOWvQkKgVnvr0f5h+OAq+GSAVXxUsoLT0fk1vSLwsMzMTObNm8d1110HwPDhw3n77bdZtWoVr732Gvfffz8ADz/8MJ07d2bt2rX8/vvvNGli9Zo6YcIEVq1axcqVKxk9ejRJSUmO7YtSPiu2BgyaCicOw9TBcO6M0xH5tcC7IriIM3dPyu6Ges+ePbRq1Yqrr76aEydOsHTp0gu6hE5PTwdg/vz554eZDA0NJSYmBoDRo0fz7bffArBv3z62b99O+fLa94pSfxHXCm54H768A95rB9WaQ8VGUKMNxHeC0MD7efMWrx4pEekFvAWEAh8aY17OMV/s+X2AU8BQY8zv3ozJW7LvEaSkpNC3b1/effddhg4dSmxsbK7dU+dmwYIFzJ07l2XLlhEdHU2XLl04c0bPdJTKU5PrIfND2PA1HFgNG6cBBqLLQ6PrrC6va7RxOEjf57WiIREJBd4FegONgUEi0jjHYr2BevZjODDGW/EUl5iYGEaPHs1rr71GiRIlqF27Nl9++SVgjUewdu1aALp168aYMdbuZmZmkpqaSkpKCmXLliU6OpotW7bw66+/OrYfSvmNpgNg8FR4ZC08k2D1YFqnC6z7wuq3aPINsG+F01H6NG9eEbQBdhhjdgGIyFSgH7DJZZl+wCRjdYH6q4jEikhVY8xBL8bldS1atKBZs2ZMnTqVTz75hPvuu48XX3yRc+fOMXDgQJo1a8Zbb73F8OHDGT9+PKGhoYwZM4ZevXoxduxYmjZtSoMGDWjXrp3Tu6KUf4ksBY2utR5nT8KK8bDkLRjfHcrVhdBwpyMsmha3Q/sHPb5abyaC6sA+l+kEoK0by1QHLkgEIjIc64qBmjVrejxQTzhx4sJGLjNnzjz//IcffvjL8pUrV2b69Ol/eX327Nm5rt91iMmc21JK5SKiJHR42BoBbeUE2L+y4Pf4ulKVvLJabyaC3MZezDn4gTvLYIwZB4wDazyCooemlAoakaWshKDy5M3qowlADZfpOOBAIZZRSinlRd5MBCuAeiJSW0QigIHAjBzLzACGiKUdkFLY+wP+NtKaL9Fjp1Rw81rRkDEmQ0QeBH7Eqj46wRizUURG2PPHAt9jVR3dgVV9dFhhthUVFUVSUhLly5fHqpGq3GWMISkpiaioKKdDUUo5JCDGLD537hwJCQla576QoqKiiIuLIzzcz2tUKKXylN+YxQHR9C48PJzatWs7HYZSSvkl7WtIKaWCnCYCpZQKcpoIlFIqyPndzWIROQr8Uci3VwASPRiOP9B9Dg66z8GhKPtcyxhTMbcZfpcIikJEVuZ11zxQ6T4HB93n4OCtfdaiIaWUCnKaCJRSKsgFWyIY53QADtB9Dg66z8HBK/scVPcIlFJK/VWwXREopZTKQROBUkoFuYBMBCLSS0S2isgOERmZy3wRkdH2/HUi0tKJOD3JjX2+1d7XdSKyVESaORGnJxW0zy7LXS4imSLSvzjj8wZ39llEuojIGhHZKCILiztGT3Pjux0jIjNFZK29z4XqxdhXiMgEETkiIhvymO/53y9jTEA9sLq83gnUASKAtUDjHMv0AWZjjZDWDvjN6biLYZ/bA2Xt572DYZ9dlpuP1eV5f6fjLobPORZrXPCa9nQlp+Muhn3+O/CK/bwicAyIcDr2IuzzlUBLYEMe8z3++xWIVwRtgB3GmF3GmLPAVKBfjmX6AZOM5VcgVkSqFnegHlTgPhtjlhpjjtuTv2KNBufP3PmcAR4CvgaOFGdwXuLOPg8GvjHG7AUwxvj7fruzzwYoLdZgJKWwEkFG8YbpOcaYRVj7kBeP/34FYiKoDuxzmU6wX7vYZfzJxe7PXVhnFP6swH0WkerADcDYYozLm9z5nOsDZUVkgYisEpEhxRadd7izz+8AjbCGuV0PPGKMySqe8Bzh8d+vgBiPIIfchijLWUfWnWX8idv7IyJdsRJBR69G5H3u7PObwNPGmMwAGbnOnX0OA1oB3YASwDIR+dUYs83bwXmJO/vcE1gDXAXUBX4SkcXGmFQvx+YUj/9+BWIiSABquEzHYZ0pXOwy/sSt/RGRpsCHQG9jTFIxxeYt7uxza2CqnQQqAH1EJMMYM61YIvQ8d7/bicaYk8BJEVkENAP8NRG4s8/DgJeNVYC+Q0R2Aw2B5cUTYrHz+O9XIBYNrQDqiUhtEYkABgIzciwzAxhi331vB6QYYw4Wd6AeVOA+i0hN4Bvgdj8+O3RV4D4bY2obY+KNMfHAV8D9fpwEwL3v9nSgk4iEiUg00BbYXMxxepI7+7wX6woIEakMNAB2FWuUxcvjv18Bd0VgjMkQkQeBH7FqHEwwxmwUkRH2/LFYNUj6ADuAU1hnFH7LzX3+F1AeeM8+Q84wftxzo5v7HFDc2WdjzGYR+QFYB2QBHxpjcq2G6A/c/Jz/DUwUkfVYxSZPG2P8tntqEfkM6AJUEJEE4DkgHLz3+6VdTCilVJALxKIhpZRSF0ETgVJKBTlNBEopFeQ0ESilVJDTRKCUUkFOE4EKGiJS3u6Vc42IHBKR/fbzZBHZ5IXtPS8iT1zke07k8frEQOg9VfkmTQQqaBhjkowxzY0xzbH6H3rDft4cq859vkQk4NrdKAWaCJTKFioiH9j92c8RkRIAdudtL9n9+j8iIq1EZKHdoduP2b0+isjDIrLJ7h9+qst6G9vr2CUiD2e/KCKPicgG+/FozmDsVqPv2Ov8Dqjk3d1XwUzPcJSy1AMGGWPuEZEvgJuAKfa8WGNMZxEJBxYC/YwxR0XkFuA/wJ3ASKC2MSZdRGJd1tsQ6AqUBraKyBigKVZr0LZYLWF/E5GFxpjVLu+7AaurhMuAylhjDEzwxo4rpYlAKctuY8wa+/kqIN5l3uf23wbApVi9W4LV5UF2Hy/rgE9EZBowzeW93xlj0oF0ETmC9aPeEfjW7hgOEfkG6AS4JoIrgc+MMZnAARGZX/RdVCp3mgiUsqS7PM/E6sI520n7rwAbjTFX5PL+a7B+vK8DnhWRJnmsN4zcuxHOjfb/ooqF3iNQyn1bgYoicgWAiISLSBMRCQFqGGN+Bp7CGi6yVD7rWQRcLyLRIlISqxhocS7LDBSRUPs+RFcP74tS5+kVgVJuMsactatwjhaRGKz/nzex+vqfYr8mWLWRkvMaDMcY87uITOTP/vI/zHF/AOBbrIFW1tvr9/tB6JXv0t5HlVIqyGnRkFJKBTlNBEopFeQ0ESilVJDTRKCUUkFOE4FSSgU5TQRKKRXkNBEopVSQ+3+DbkcN8mv93AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(df_scores['threshold'], df_scores['precision'], label='Precision')\n",
    "plt.plot(df_scores['threshold'], df_scores['recall'], label='Recall')\n",
    "plt.xlabel('Threshold')\n",
    "plt.ylabel('Precision and Recall')\n",
    "plt.title('Precision and Recall at various thresholds')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T10:50:21.509129Z",
     "iopub.status.busy": "2021-10-01T10:50:21.50892Z",
     "iopub.status.idle": "2021-10-01T10:50:21.516438Z",
     "shell.execute_reply": "2021-10-01T10:50:21.515574Z",
     "shell.execute_reply.started": "2021-10-01T10:50:21.509104Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f56aa473f70>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAy60lEQVR4nO3deXxU5dn/8c+Vfd8JkARI2DdZwyaiqJVFUVCxirYoTyni49bFR22fVrS2/amlFdfyoFVxRavVUkWwiIhKBYLse9jDThYSSAJZrt8fM2AIAYYkkzPJXO/XK6/MnHPPmW8myVxz7nPOfYuqYowxxn8FOB3AGGOMs6wQGGOMn7NCYIwxfs4KgTHG+DkrBMYY4+esEBhjjJ+zQmD8gogsFJGJ7tt3iMjXTmeqDyLSWkSOikjgedrdJiKfNVQu07hYITANTkR2iEiJ+w1sv4i8JiJRPpArTEQKROSKGtY9LSLvX+D2XhORE+6fM09E/i0inesvMajqLlWNUtWK87R7S1WH1edzm6bDCoFxyrWqGgX0AnoDv3I2DqhqKfAuML7qcven7XHAzFps9in3z5kGHAReq95AXOx/0TjG/viMo1R1PzAPV0EAQEQGishi96fzVSIytMq6BBF5VUT2iki+iHzkXh4vIh+LyCH38o9FJK0WkWYCN4pIRJVlw3H9r3zqfq6HRGSPiBSJyCYRudKDn7MYeBvo7t7GQhH5g4h8AxQDbUWks3uvIc+93R9W+bnDReTPIrJTRI6IyNfuZekioiIS5G53h4hsc2fbLiK3VVn+dZXtXSwiy9zbWiYiF1dZt1BEHheRb9zb+UxEkmrxWppGwgqBcZT7zXokkO2+nwp8AvweSAAeAD4QkWbuh7wBRADdgGTgaffyAOBVoA3QGigBnr/QPKq6GNgH3FBl8Y+Bt1W1XEQ6AfcA/VQ1GleR2OHBzxkF3AasqLbdSUA0cAj4N65ikYxrD+RFEenmbjsV6AtcjOt1eRCorPYckcCzwEh3touBlTVkScD1Gj8LJAJ/AT4RkcQqzW4FJrizhOD6PZgmygqBccpHIlIE7MbVZTLFvfxHwBxVnaOqlar6byALuFpEWuIqGpNVNV9Vy1T1SwBVzVXVD1S1WFWLgD8Al9Uy2+u4u4dEJAYYzffdQhVAKNBVRIJVdYeqbj3Hth4QkQJchS4KuKPKutdUdZ2qlgMjgB2q+qqqlqvqd8AHwFh3t9F/Afer6h5VrVDVxap6vIbnqwS6i0i4qu5T1XU1tLkG2KKqb7if6x1gI3BtlTavqupmVS0B3qPKHptpeqwQGKeMcX9qHQp0Bk52PbQBbnJ3CxW430QvAVoCrYA8Vc2vvjERiRCR/3N3nRQCi4C4851NcxavA5e7907GAtmqugJAVbOBnwGPAgdFZJaIpJxjW1NVNU5VW6jqddWKxu4qt9sAA6r93LcBLXC9NmHAuQoOqnoMuBmYDOwTkU/OcnA6BdhZbdlOILXK/f1VbhfjKmKmibJCYBzl/kT/Gq6uD3C9Ob7hfvM8+RWpqk+41yWISFwNm/ol0AkYoKoxwKXu5VKLTLuAr3C9Ef8YV2Gouv5tVb0E15u3Ak9e6HOc3FSV27uBL6v93FGqehdwGCgF2nmQfZ6qXoWrcG4EXqqh2V539qpaA3tq80OYxs8KgfEF04CrRKQX8CZwrYgMF5FA9ymdQ0UkTVX34Tpg+6L74HCwiJx8w4/GdVygwN0HPqWG57kQM3EdCxgMvHVyoYh0EpErRCQU15tzCa7uorr6GOgoIj92/1zBItJPRLqoaiXwCvAXEUlxvy6D3BlOEZHmInKd+1jBceDoWbLNcT/XrSISJCI3A13dGYwfskJgHKeqh3B96v6tqu7G1Sf/a1wHUHcD/8P3f6s/Bspwfdo9iKubBlzFJBzXp+dvgbl1jPU+EA987i5AJ4UCT7ifZz+ug6m/ruNz4T6uMQy4Bdcn9v249jROvtk/AKwBlgF57nXV/38DcO0Z7XW3uQz47xqeKxcY5W6bi+vA8yhVPVzXn8M0TmIT0xhjjH+zPQJjjPFzVgiMMcbPWSEwxhg/Z4XAGGP8XJDTAS5UUlKSpqenOx3DGGMaleXLlx9W1WY1rWt0hSA9PZ2srCynYxhjTKMiItWvJj/FuoaMMcbPWSEwxhg/Z4XAGGP8XKM7RmCMObeysjJycnIoLS11OopxQFhYGGlpaQQHB3v8GCsExjQxOTk5REdHk56ejsgFD75qGjFVJTc3l5ycHDIyMjx+nNe6hkTkFRE5KCJrz7JeRORZEckWkdUi0sdbWYzxJ6WlpSQmJloR8EMiQmJi4gXvDXrzGMFruGZdOpuRQAf31yTgr17MYoxfsSLgv2rzu/da15CqLhKR9HM0GQ28rq7hT78VkTgRaVltyN96s/lAER+v2ktUWBBRocFEhQURHRpEZGgQUaFBRIe5vkeGBhESZMfQjTH+w8ljBKmcPlVfjnvZGYVARCbh2mugdevWtXqyzQeKeHZBtkdtw4IDiAkLJiY8mJiwIPf3YGLDg4kJDyImLJj4iBBiI1zf4yKCXV/hIVZEjAECAwO56KKLKC8vp0uXLsycOZOIiIg6bfORRx7h0ksv5Qc/+EGN66dPn05ERATjx4+v0/NkZGQwd+5cOnXqdGrZz372M1JSUnjwwQfP+/j09HSio6MREeLj43n99ddp06b6hHC1N3ToUKZOnUpmZuapC2yTkpLO/8BzcLIQ1LT/UuPkCKo6A5gBkJmZWasJFEb1SOHq7i0pLqvgaGk5R4+XUVRazrHjFVVul1NUWs7R4+UUlpZxpKSMwpJy8o6dYMfhYxSWlnOkpIyKyrNHiA0PpkNyFB1bRNOpeTQdm0fTqUU0CZEhtYltTKMUHh7OypUrAbjtttuYPn06v/jFL06tr6ioIDDwwqaT/t3vfnfO9ZMnT77gnDW55ZZbmDVrFlOmuCa5q6ys5P333+ebb77xeBtffPEFSUlJTJkyhd///ve89FJNM4b6DicLQQ6uychPSsM1s5LXBAQIUe6uINdc4BdOVTl2ooKC4hMUFJeR7/5+8v7+wlK2HDjKJ6v38XbJrlOPaxETxpAOSVzeOZlLOiQRE+b5qV3GNGZDhgxh9erVLFy4kMcee4yWLVuycuVK1qxZw8MPP8zChQs5fvw4d999N3feeScATz31FG+88QYBAQGMHDmSJ554gjvuuINRo0YxduxYHn74YWbPnk1QUBDDhg1j6tSpPProo0RFRfHAAw+wcuVKJk+eTHFxMe3ateOVV14hPj6eoUOHMmDAAL744gsKCgr429/+xpAhQ07LO27cOG6++eZThWDRokWkp6fTpk0b1q1bx4QJEzhx4gSVlZV88MEHdOjQ4aw/+6BBg3j22WcBOHToEJMnT2bXLtf7wrRp0xg8eDBHjx7l3nvvJSsrCxFhypQp3Hjjjdx1110sW7aMkpISxo4dy2OPPeaNXw/gbCGYDdwjIrOAAcARbx0fqE8i3xeTtPizt1NVDhYdZ9P+IjYfKGLF7gLmrdvP35fnEBggDGqbyHW9UhjRvYUVBeM1j/1rHev3FtbrNrumxDDl2m4etS0vL+fTTz9lxAjXeSNLly5l7dq1ZGRkMGPGDGJjY1m2bBnHjx9n8ODBDBs2jI0bN/LRRx+xZMkSIiIiyMvLO22beXl5fPjhh2zcuBERoaCg4IznHT9+PM899xyXXXYZjzzyCI899hjTpk07lWnp0qXMmTOHxx57jPnz55/22B49ehAQEMCqVavo2bMns2bNYty4cYCr++n+++/ntttu48SJE1RUnHu66rlz5zJmzBgA7r//fn7+859zySWXsGvXLoYPH86GDRt4/PHHiY2NZc2aNQDk5+cD8Ic//IGEhAQqKiq48sorWb16NT169PDodb9QXisEIvIOMBRIEpEcXJOJBwOo6nRcE2hfDWQDxcAEb2VxgojQPCaM5jFhXNrRNeBfeUUlK3YXsGDjQT5ZvY8H31/Nbz5ay29HdeXHA+uvD9EYp5WUlNCrVy/AtUfwk5/8hMWLF9O/f/9T57d/9tlnrF69mvfffx+AI0eOsGXLFubPn8+ECRNOHVNISEg4bdsxMTGEhYUxceJErrnmGkaNGnXa+iNHjlBQUMBll10GwO23385NN910av0NN9wAQN++fdmxY0eN+ceNG8esWbPo1q0b//znP091Sw0aNIg//OEP5OTkcMMNN5x1b+Dyyy/nwIEDJCcn8/vf/x6A+fPns379+lNtCgsLKSoqYv78+cyaNevU8vh41yfM9957jxkzZlBeXs6+fftYv3594ysEqjruPOsVuNtbz++LggID6JeeQL/0BB4c3omVuwuY+tkmHv94PYPaJtA+OdrpiKaJ8fSTe32reoygqsjIyFO3VZXnnnuO4cOHn9Zm7ty55zwFMigoiKVLl/L5558za9Ysnn/+eRYsWOBxttDQUMB1QLu8vLzGNuPGjWPYsGFcdtll9OjRg+TkZABuvfVWBgwYwCeffMLw4cN5+eWXueKKK854/BdffEFkZCR33HEHjzzyCH/5y1+orKzkP//5D+Hh4ae1VdUzft7t27czdepUli1bRnx8PHfccYdXrxS3U1wcIiL0bh3PtJt7ExESyIPvr2bLgSKKT9T8h2lMUzN8+HD++te/UlZWBsDmzZs5duwYw4YN45VXXqG4uBjgjK6ho0ePcuTIEa6++mqmTZt2RsGJjY0lPj6er776CoA33njj1N6Bp9q1a0diYiIPP/zwqW4hgG3bttG2bVvuu+8+rrvuOlavXn3WbYSHhzNt2jRef/118vLyGDZsGM8///yp9SdzV1+en59PYWEhkZGRxMbGcuDAAT799NMLyn+hbIgJhzWLDuXRa7vxs3dXctXTiwBIjAwhLSGCri2j6ZkWR4+0ODo2jyIo0Oq2aTomTpzIjh076NOnD6pKs2bN+OijjxgxYgQrV64kMzOTkJAQrr76av74xz+eelxRURGjR4+mtLQUVeXpp58+Y9szZ848dbC4bdu2vPrqqxecb9y4cfzqV7/i+uuvP7Xs3Xff5c033yQ4OJgWLVrwyCOPnHMbLVu2ZNy4cbzwwgs8++yz3H333fTo0YPy8nIuvfRSpk+fzm9+8xvuvvtuunfvTmBgIFOmTOGGG26gd+/edOvWjbZt2zJ48OALzn8hxNVD03hkZmZqU5yYZv3eQrYcLCInv4Sc/BJ25h5j7Z4jFJa69hDCggO4KDWWsX3TGN0rlbDgCzv1zviPDRs20KVLF6djGAfV9DcgIstVNbOm9rZH4CO6psTQNSXmtGWqys7cYlblFLBq9xEWbz3MQx+s4U/zNnPHxW24pkcKLWPDrCgYY+rECoEPExHSkyJJT4pkdK9UVJXFW3OZsWgbUz/bzNTPNgOui9haxITRPDaMqNDvi0KACCO6t+Dq7i0JCLCxZ4wxNbNC0IiICIPbJzG4fRJbDhSxcncBBwpLOVB4nP2FpRwoLGVfwffnNReWlvHx6n10SN7CPVe0Z1SPFAKtIPiFms5EMf6hNt39VggaqQ7No+nQ/Nynm1ZUKnPW7OO5BVu4f9ZKnpnvKgjX9UyxA89NWFhYGLm5uTYUtR86OR9BWNiFjZxgB4v9QGWlMm/dfp75fAsb9xeR2SaeGeMzbfyjJspmKPNvZ5uh7FwHi60Q+JHKSuXDFXv41YdrSIkN49UJ/clIijz/A40xjd65CoH1D/iRgADhxr5pvPPTARSWlnPDi9+wbEfe+R9ojGnSrBD4ob5tEvjwvy8mPiKE215awm8+WsOu3GKnYxljHGKFwE+1SYzkg7su5sa+qby3LIehU7/g3ndWsOPwMaejGWMamB0jMBwoLOWVr7fz1hLXOOl/GtuDkRe1dDiVMaY+2cFi45E9BSX891vfsWp3AbcPasOwbi1IjQunZVwYoUF29bIxjZkVAuOxE+WV/HHOBl5bvOO05cnRofRIi+OS9okMbp9E++QoO0fdmEbECoG5YHsLStiZW8yeghL25JewK6+YrJ157HQfVG4eE8rUm3oypEMzh5MaYzxhg86ZC5YSF05KXPgZy3fnFbN462Fe+mo7976zgjn3DamxnTGm8bCzhswFaZUQwc39WvPS+EzKyiu5950VlFVUOh3LGFMHVghMrWQkRfL/buzB8p35PPnpRqfjGGPqwLqGTK1d1zOF5TvyePnr7XRoHsXN/Vo7HckYUwu2R2Dq5LejujKkQxL/++FaFmcfdjqOMaYWrBCYOgkKDOCF2/qQkRTJ+FeWct87K1ixK9/pWMaYC2CFwNRZTFgwb/90ILdfnM4XGw9y/YuLGfPCN6zcXeB0NGOMB6wQmHrRLDqU347qyn9+fSWPXtuVg4Wl3PbSt3y7LdfpaMaY87BCYOpVVGgQdwzO4MO7B9MyLpw7Xl3Kl5sPOR3LGHMOVgiMVzSPCePdSQPJSIripzOz+Pf6A05HMsachRUC4zWJUaHM+ulAuqTEcNeby5m5eEetJtY2xniXFQLjVbERwbz5k/5c2rEZU2av4+fvrqS0rMLpWMaYKqwQGK+LDgvm5fGZ/PKqjvxz1V5++nqWFQNjfIgVAtMgAgKEe6/swJM39uDr7MNMnGnFwBhfYYXANKgfZrbiT2N78nX2YaZ/udXpOMYYvFwIRGSEiGwSkWwRebiG9bEi8i8RWSUi60RkgjfzGN8wtm8aw7s15+WvtpN/7ITTcYzxe14rBCISCLwAjAS6AuNEpGu1ZncD61W1JzAU+LOIhHgrk/EdvxzWiWMnypm+yPYKjHGaN/cI+gPZqrpNVU8As4DR1dooEC2uOQ+jgDyg3IuZjI/o2DyaMb1Smbl4Bzn5xU7HMcavebMQpAK7q9zPcS+r6nmgC7AXWAPcr6pnzHIiIpNEJEtEsg4dsqtUm4pfXNWRoIAA7n1nBSfKbXIbY5zizUJQ08zm1a8mGg6sBFKAXsDzIhJzxoNUZ6hqpqpmNmtmc+Q2Fa0SInjixotYsauAP87ZwNHjtjNojBO8OTFNDtCqyv00XJ/8q5oAPKGuy02zRWQ70BlY6sVcxoeM6pHCsu15vLZ4B68t3kFiZAitEyPon5HAQ8M7ExBQ0+cJY0x98mYhWAZ0EJEMYA9wC3BrtTa7gCuBr0SkOdAJ2ObFTMYH/XZUVy5un8TWQ0fZnVdM9sGj/N+X22geHcZ/XZLhdDxjmjyvFQJVLReRe4B5QCDwiqquE5HJ7vXTgceB10RkDa6upIdU1aa58jNBgQEM79bi1H1V5Sczs3hq3kYu75xMRlKkg+mMafqksQ0ClpmZqVlZWU7HMF62/0gpw57+ko7No3n3zkEEWheRMXUiIstVNbOmdXZlsfFJLWLDmHJtN7J25vPs51uorGxcH1iMaUysEBifdUOfVK7p0ZJnPt/CmBe/YfnOPKcjGdMkWSEwPktEeH5cb6bd3IuDhce58a//4b53VlB8wk4zNaY+WSEwPk1EGNM7lQUPXMY9l7dn9qq9vLds9/kfaIzxmBUC0yhEhATxwPBOtEoIZ/HWXKfjGNOkWCEwjcrFbZP4dlsuFXbw2Jh6Y4XANCoXt0+ksLSclbvznY5iTJNhhcA0KkM7JRMfEcxTczfR2K6BMcZXWSEwjUpseDC/HNaJJdvzmGUHjY2pF1YITKMzrn9rBrdP5NcfruHFhdmUV9gQ1sbUhRUC0+gEBgh/u70fI7u34Km5mxj9wje8/NU2th066nQ0YxolG2vINFqqyr9W7+PZz7eQfdBVBNITI7gpsxUTh2QQGhTocEJjfMe5xhqyQmCahN15xXyx6SDz1u3nm+xc2iZF8tqE/rROjHA6mjE+wQadM01eq4QIxg9K562JA3ltQj/2HSnl2QVbnI5lTKNghcA0OUM7JXNj31Rmr9zL4aPHnY5jjM+zQmCapDsuzuBERSUPvb+aotIyp+MY49OsEJgmqX1yFL8b3Y2Fmw8x8pmv+HTNPrsAzZizsEJgmqzxg9J5d9JAokKDuOut77hlxres31vodCxjfI4VAtOkZaYn8PG9l/D7Md3ZfKCIUc99xf9+uMYuQjOmCisEpskLCgzgRwPbsPCByxnXvzVvLdnFwk2HnI5ljM+wQmD8RmxEML++ugsAG/ZZF5ExJ1khMH4lMjSINokRbNhvhcCYk6wQGL/TpUUMa/cUUminlRoDQJDTAYxpaL1bxzF33X56PfYZPVvFMbhdEoPbJ9GnTZyNT2T8ko01ZPxOZaWydEcei7MP83X2YVblHKGiUgkLDuDyTsncfXl7uqfGOh3TmHplg84Zcw6FpWUs2ZbH11sO8eGKPRSWlvODLs355bCOdGkZ43Q8Y+qFFQJjPFRYWsZr3+xg+pdbSYgM4euHrnA6kjH1wkYfNcZDMWHBTLq0LQEihAYF8OXmQzZWkWny7GCxMTW4KDWWpTvyuP2VpQQIdE2J4Z7L2zOie0unoxlT76wQGFNNWHAg70wayLHj5azYVcDS7bk8uyCbBRsPWiEwTZJ1DRlzFpGhQVzSIYmRF7ne/PulJzicyBjv8GohEJERIrJJRLJF5OGztBkqIitFZJ2IfOnNPMbUxn+25gIwqF2iw0mM8Q6vFQIRCQReAEYCXYFxItK1Wps44EXgOlXtBtzkrTzG1NaqnAJE4O0lu8jJL3Y6jjH1zpvHCPoD2aq6DUBEZgGjgfVV2twK/ENVdwGo6kEv5jGmViZd2pZjxyuY/uVWpn+5lSs6J/OjgW24tEMzAgLE6XjG1Jk3C0EqsLvK/RxgQLU2HYFgEVkIRAPPqOrr1TckIpOASQCtW7f2SlhjzqZbSiwv357JnoIS3lmyi1nLdjF/w0FaJ0Two4GtGde/NdFhwU7HNKbWvHmMoKaPStWvXgsC+gLXAMOB34pIxzMepDpDVTNVNbNZs2b1n9QYD6TGhfPA8E4sfvhKnhvXm5axYfxxzkYuefILps3fTEHxCacjGlMr3twjyAFaVbmfBuytoc1hVT0GHBORRUBPYLMXcxlTJyFBAVzbM4Vre6awOqeA5xdkM23+Fl7+ajs/GtiGiUMySIoKdTqmMR7z5h7BMqCDiGSISAhwCzC7Wpt/AkNEJEhEInB1HW3wYiZj6lWPtDhmjM9k7s+GcHnnZP5v0VYueXIBS7blOh3NGI95rRCoajlwDzAP15v7e6q6TkQmi8hkd5sNwFxgNbAUeFlV13orkzHe0rlFDM+N6838X1xGbHgwz3+R7XQkYzzm1SuLVXUOMKfasunV7v8J+JM3cxjTUNo1i+JHA9rw539vJvvgUdonRzkdyZjzsiEmjKln4wa05rkF2Ux+czk90+JIiQsjJS6cTi2i6dM63ul4xpzBCoEx9SwpKpTHRnfj/eU5/GfrYfYXllLpPl/uqRt78MN+rc69AWMa2DkLgYgUceYpn+A6NVRV1WbtMKYG4/q7ri8AKK+o5EDRcR58fxW/+eda2jaLpG+beETsYjTjG2xiGmMaSO7R41z73NfsPVJKi5gwMtPjyWwTT2Z6Al1axhBoVykbLzrXxDTn2yM453CLqppXl2DG+JPEqFA+unsw89btZ9mOfJbvzOfj1fsAiAoNIjM9nj9cfxGpceEOJzX+5px7BCKyHVfXUI1XCatqW28FOxvbIzBNyZ6CErJ25JG1I593s3Zza//WPHpdN6djmSao1nsEqprhnUjGGHANW5HaK5XRvVI5fPQ4b367k5z8Ykb3SuUHXZoTHhLodETjBzw+a0hE4oEOQNjJZaq6yBuhjPFHvxvdnbT4cGav2sv8DQeJDAnkmh4tmXJtNyJD7QQ/4z0eXVksIhOBRbiuEn7M/f1R78Uyxv80iw7lf6/pyuKHr+TtiQPo0Dya97Jy2F9Y6nQ008R5OsTE/UA/YKeqXg70Bg55LZUxfiwwQBjULpGSExX0SIulXTO7Otl4l6eFoFRVSwFEJFRVNwKdvBfLGP/23a58Nh0o4tb+Nv+G8T5POx5z3NNKfgT8W0TyOXNIaWNMPXl7yW6CAoQhHW3+DeN9HhUCVb3effNREfkCiMU1aqgxxgv2HSmhvFK55MkFdGoezcC2iYztm0b31Fino5kmyKMri0VkILBOVYvc96OBrqq6xMv5zmDXERh/cLy8glW7j7BkWy7fbs9l+c58ggMCmHP/EFolRDgdzzRC57qOwNNCsALoo+7GIhIAZKlqn3pN6gErBMYf7c4rZuQzX9G5RTSzJg0kKNCbc0qZpuhchcDTvybRKhVDVSuxkUuNaTCtEiJ4fEw3snbm8+LCrU7HMU2Mp4Vgm4jcJyLB7q/7gW3eDGaMOd31vdMY3SuFZz7fwne78p2OY5oQTwvBZOBiYA+uCecHAJO8FcoYU7PHx3SnRUwY989awe68YqfjmCbCo0KgqgdV9RZVTVbV5qp6q6oe9HY4Y8zpYsKCef7W3hwpLmPUc1/zxSb7NzR15+kQEx1F5HMRWeu+30NEfuPdaMaYmvRuHc/sey6hZWwYE15dxqjnvuKNb3dSWFrmdDTTSHl61tCXwP8A/6eqvd3L1qpqdy/nO4OdNWSMS8mJCt7L2s2sZbvZsK+QsOAArrkohVv6tyLTZkAz1dR6GOoqIlR1abU/rPI6JzPG1Fp4SCC3X5zO+EFtWLPnCLOW7Wb2yr188F0ObZtFcku/VtzYJ43EqFCnoxof52khOCwi7XDPXywiY4F9XktljPGYiNAjLY4eaXH85poufLJ6H+8u280f52xk6mebuaVfK+68rJ3NfGbOytOuobbADFxnDuUD24HbVHWnd+OdybqGjPHMlgNF/O3r7XzwXQ6qcH3vVO4a2o62NpqpX6rzlcVVNhSJ6wBzCXCzqr5VPxE9Z4XAmAuzt6CEGYu28c7SXZRVVHJNjxT+eH13osOCnY5mGlCtrywWkRgR+ZWIPC8iVwHFwO1ANvDD+o9qjKlvKXHhPHpdN75+6ApGXtSSf63ay+YDRU7HMj7kfMcI3sDVFfQf4KfAg0AIMEZVV3o3mjGmPjWLDiU2PJjIkEAuSo1zOo7xIecrBG1V9SIAEXkZOAy0PjkKqTGm8VBVFm48yCUdkggJskHrzPfO99dw6goVVa0AtlsRMKZx2nzgKHuPlHJ5p2Snoxgfc749gp4iUui+LUC4+74AqqoxXk1njKk32w8fBaBdsp01ZE53zkKgqoENFcQY41390hMQgcXZufRLT3A6jvEhXu0oFJERIrJJRLJF5OFztOsnIhXuC9WMMV6QGBVKz7Q4FthAdaYarxUCEQkEXgBGAl2BcSLS9SztngTmeSuLMcblis7JrM4p4PDR405HMT7Em3sE/YFsVd2mqieAWcDoGtrdC3wA2McUY7zsis7JqMLCTYecjmJ8iDcLQSqwu8r9HPeyU0QkFbgemH6uDYnIJBHJEpGsQ4fsD9iY2uqWEkNydKjNY2BO481CUNMYuNXHs5gGPOQ+NfWsVHWGqmaqamazZs3qK58xfkdEGNqpGYs2H2LfkRKn4xgf4c1CkAO0qnI/DdhbrU0mMEtEdgBjgRdFZIwXMxnj98b1b01FpTLyma+Yt26/03GMD/BmIVgGdBCRDBEJAW4BZldtoKoZqpququnA+8B/q+pHXsxkjN/r3TqeT+4bQqv4CO58Yzn/++EaSsvOuVNumjivFQJVLQfuwXU20AbgPVVdJyKTRWSyt57XGHN+GUmRfHDXxdx5aVveWrKL0c9/Q/ZBGzTAX13QMNS+wIahNqZ+Ldp8iJ+/u5LiExX8fkx3buyb5nQk4wW1HobaGNP0XdqxGXPuH0KPtFh++fdV3PP2d+QdO+F0LNOArBAYY2geE8ZbEwfwy6s6Mm/dfoY9/SVz19pstP7CCoExBoCgwADuvbIDs++5hOYxYUx+8zvufWeF7R34ASsExpjTdGkZw0d3D+YXV3Vk7tp9tnfgB6wQGGPOEBwYwH3V9g5eWrTN6VjGS6wQGGPO6uTeQWabeN7L2n3+B5hGyQqBMeacggMDGNG9BVsOHmVPgQ1L0RRZITDGnNfQTq4xvhbaYHVNkhUCY8x5tWsWRWpcOHPW7KOxXYRqzs8KgTHmvESEOy5O55vsXF5cuNXpOKaeWSEwxnhk4pAMRvdK4U/zNjF3rY1a2pRYITDGeEREePLGHvRqFcfP313J4uzDTkcy9cQKgTHGY2HBgcwY35eUuDBu+9sS/vzZJsorKp2OZerICoEx5oIkR4fxr3sv4aa+aTy3IJubZ3xLTn6x07FMHVghMMZcsIiQIJ4a25NnbunFpv1FTJxpQ8M3ZlYIjDG1NrpXKqlx4STHhDkdxdSBFQJjTK0dKS5j04Ei+rWJdzqKqQMrBMaYWsvamQdAv4wEh5OYurBCYIyptaU78ggOFHq1inM6iqkDKwTGmFrbevAo7ZpFERYc6HQUUwdWCIwxtXaiQgm1ItDoWSEwxtRaeUUlwQHidAxTR1YIjDG1Vl6pBAVaIWjsrBAYY2qtvKKSoAB7G2ns7DdojKm1krJK2yNoAqwQGGNqZdHmQ2zYV0i/dLuGoLGzQmCMuWClZRX89p9raZsUycQhGU7HMXUU5HQAY0zjM2PRNnbmFvP2xAGEBtnpo42d7REYYy5YpXve4k0HihxOYuqD7REYYy7YvVd0YMO+Qn738XpS4sIZ3q2F05FMHXh1j0BERojIJhHJFpGHa1h/m4isdn8tFpGe3sxjjKkfgQHCtJt70zMtjvtnrWDFrnynI5k68FohEJFA4AVgJNAVGCciXas12w5cpqo9gMeBGd7KY4ypX+Ehgbx8eybJ0WFMnJnFztxjTkcyteTNPYL+QLaqblPVE8AsYHTVBqq6WFVPfpT4FkjzYh5jTD1LigrltQn9qFBlwqvLyD92wulIpha8WQhSgd1V7ue4l53NT4BPvZjHGOMFbZtF8fL4THIKSvjp61nkWTFodLxZCGq63FBrbChyOa5C8NBZ1k8SkSwRyTp06FA9RjTG1IfM9ASe/mEvVu4u4Io/L+TtJbuorKzx3934IG8WghygVZX7acDe6o1EpAfwMjBaVXNr2pCqzlDVTFXNbNasmVfCGmPq5poeLfnkviF0ah7Nrz9cw/V/XczqnAKnYxkPeLMQLAM6iEiGiIQAtwCzqzYQkdbAP4Afq+pmL2YxxjSATi2imTVpINNu7sWe/BJGv/AN//vhGgqKrbvIl3ntOgJVLReRe4B5QCDwiqquE5HJ7vXTgUeAROBFEQEoV9VMb2UyxnifiDCmdypXdEnm6X9vZubiHXy6dj8Pj+zM2D5pBNj8BT5HVBtXP15mZqZmZWU5HcMY46H1ewt55J9rydqZT5/WcTw+pjvdUmKdjuV3RGT52T5o2xATxhiv6poSw3t3DmLqTT3ZmVvMtc99zaOz13GkpMzpaMbNhpgwxnhdQIAwtm8aV3Vpzp//vYnX/7ODj1fv5db+rbm+TxoZSZFOR/Rr1jVkjGlwa/cc4al5m/hqyyFUoW+beG7ok8qoi1KIjQh2Ol6TdK6uISsExhjH7D9Sykcr9/DB8hy2HDxKSFAAV3Vpzg19Urm0YzOCA633ur5YITDG+DRVZe2eQj74LofZq/aSd+wESVEhPDSiMzdltjr/Bsx52cFiY4xPExEuSovl0eu6seTXV/LS+ExCgwL5e1aO09H8ghUCY4xPCQ4M4KquzenVKo7Dx447HccvWCEwxvikxKgQco/aFckNwQqBMcYnJUaGcqSkjLKKSqejNHlWCIwxPikxKgTA5jhoAFYIjDE+KffoCUSwsYkagBUCY4zPUVX+tXov/dITSIoKdTpOk2eFwBjjc9bvKyT74FGu7ZnidBS/YIXAGONT1u8t5Kczs4gMCeTq7i2cjuMXrBAYY3zGv9cfYOz0xVQqvHvnIBKtW6hB2OijxhjHqSozFm3jibkbuSg1lpfGZ9I8JszpWH7DCoExxlHlFZX86h9r+PvyHK7p0ZKpY3sSHhLodCy/Yl1DxhhHbTpQxN+X5xAUIHRtGcPx8gqnI/kdKwTGGEd1S4nltQn9GNg2kT/N28Sg/7eA97J2Ox3Lr1ghMMY4bminZN6cOIBP7x9CZGggX2465HQkv2KFwBjjM7q0jCEuIgSlcc2T0tjZwWJjjE8JDQpgzpr9DH96Ef0zEuifkcCAjASS7Swir7FCYIzxKc+O683ctftZsj2Pf3yXwxvf7gQgIymS/ukJDGjrKg5p8REOJ206bKpKY4zPKq+oZN3eQpZuz2PJ9lyWbs+jsLQcgNS4cAac3GNom0h6YgQiNkDd2dicxcaYJqGyUtm4v4il23NZsj2PpdvzyHUPU90iJozB7ZMY3D6Rwe2T7IK0aqwQGGOaJFVl66FjfLstl8VbD7N4ay4FxWUAtE+OYnA7V1EYkJFITHiQX+8xWCEwxviFykpl/b5Cvsk+zDdbc1m6PZfSMtcMZ8GBQlRoEJGhQUSFBhEdVu12SBBRYa77UaFnvx0ZGkRwYOM74dIKgTHGLx0vr2DFrgJW7CqgsLSMo6XlHD3u/nLfPna8nCL3/ZIyz65qjo8IJiUunNS4cFLjXd/T4sNJjYsgNT6c+Ihgn9v7OFchsLOGjDFNVmhQIAPbJjKwbaJH7csrKjl2ooJj7mJRVFp+6vbJwlFUWs7BolL2FJSw/fAxvs4+TPGJ0wtIeHDgqQKRcqpIfF80mseEEehDM69ZITDGGLegwABiwwOIDQ/2+DGqSkFxGXsKSsjJL2FPQQl78kvYW+C6vWbPEfKqzbscFCC0iA07VRzS3AWjavEIC264gfesEBhjTB2ICPGRIcRHhtA9NbbGNsUnytl7lkLx7dZc9heWUlmtlz4pKvRUkUiNd+1VXHNRS6/M0eDVQiAiI4BngEDgZVV9otp6ca+/GigG7lDV77yZyRhjGlpESBDtk6Npnxxd4/qyikr2Hyk9VSROFYsjJWzYV8j8DQc4Xl7J4aLj/GJYp3rP57VCICKBwAvAVUAOsExEZqvq+irNRgId3F8DgL+6vxtjjN8IDgygVUIErRJqvlpaVcn41RxOVHjn5B5vngPVH8hW1W2qegKYBYyu1mY08Lq6fAvEiUhLL2YyxphGR0S4rmcKnVvUvEdRV97sGkoFqg4qnsOZn/ZrapMK7KvaSEQmAZMAWrduXe9BjTHG1z07rrfXtu3NPYKazo2qvl/jSRtUdYaqZqpqZrNmzeolnDHGGBdvFoIcoFWV+2nA3lq0McYY40XeLATLgA4ikiEiIcAtwOxqbWYD48VlIHBEVfdV35Axxhjv8doxAlUtF5F7gHm4Th99RVXXichk9/rpwBxcp45m4zp9dIK38hhjjKmZV68jUNU5uN7sqy6bXuW2And7M4Mxxphza3xD6BljjKlXVgiMMcbPWSEwxhg/1+jmIxCRQ8DOBniqJOBwAzxPXVnO+mU565flrF91ydlGVWu8EKvRFYKGIiJZZ5vEwZdYzvplOeuX5axf3sppXUPGGOPnrBAYY4yfs0JwdjOcDuAhy1m/LGf9spz1yys57RiBMcb4OdsjMMYYP2eFwBhj/JzfFwIRGSEim0QkW0QermH9aBFZLSIrRSRLRC7xxZxV2vUTkQoRGduQ+ao8//lez6EicsT9eq4UkUd8Mae7zVB3xnUi8mVDZ3RnON/r+T9VXsu17t99gg/mjBWRf4nIKvfr2eADTHqQMV5EPnT/vy8Vke4NndGd4xUROSgia8+yXkTkWffPsVpE+tT5SVXVb79wjYq6FWgLhACrgK7V2kTx/bGUHsBGX8xZpd0CXAP9jfXFnMBQ4ONG8HuPA9YDrd33k30xZ7X21wILfDEn8GvgSfftZkAeEOJjGf8ETHHf7gx83tCvpfu5LwX6AGvPsv5q4FNcE3sNBJbU9Tn9fY/gvPMqq+pRdb/6QCQ1zKDWADyZ/xngXuAD4GBDhqvC05xO8yTnrcA/VHUXgKo68Zpe6Os5DninQZKdzpOcCkSLiOD6cJUHlPtYxq7A5wCquhFIF5HmDZgR93MvwvX6nE29z/Xu74XgbHMmn0ZErheRjcAnwH81ULaqzptTRFKB64HpOMej1xMY5O4i+FREujVMtNN4krMjEC8iC0VkuYiMb7B03/P09UREIoARuD4INDRPcj4PdME1A+Ea4H5VrWyYeIBnGVcBNwCISH+gDa5ZE32Nx38XnvL3QuDpnMkfqmpnYAzwuLdD1cCTnNOAh1S1wvtxzsqTnN/hGvOkJ/Ac8JG3Q9XAk5xBQF/gGmA48FsR6ejtYNV49Pfpdi3wjaqe65Okt3iScziwEkgBegHPi0iMd2OdxpOMT+Aq/itx7V2voGH3Wjx1IX8XHvHqxDSNwAXNmayqi0SknYgkqWpDDlDlSc5MYJZrz5sk4GoRKVfVjxokoct5c6pqYZXbc0TkRR99PXOAw6p6DDgmIouAnsDmhol4KoOnf5+34Ey3EHiWcwLwhLubNVtEtuPqh1/aMBE9/tucAK4DssB295evqf+53p04GOIrX7gK4TYgg+8PIHWr1qY93x8s7gPsOXnfl3JWa/8azhws9uT1bFHl9ewP7PLF1xNXN8bn7rYRwFqgu6/ldLeLxdWnHNnQv/MLeD3/Cjzqvt3c/X+U5GMZ43AfwAZ+iqsfvsFfT/fzp3P2g8XXcPrB4qV1fT6/3iNQz+ZVvhEYLyJlQAlws7p/Gz6W03Ee5hwL3CUi5bhez1t88fVU1Q0iMhdYDVQCL6tqjafzOZnT3fR64DN17b00OA9zPg68JiJrcL2BPaQNuBfoYcYuwOsiUoHrjLGfNFS+qkTkHVxn1yWJSA4wBQiukrPe53q3ISaMMcbP+fvBYmOM8XtWCIwxxs9ZITDGGD9nhcAYY/ycFQJjjPFzVgiMX3KP0nlyxM6/u4doqOs2fyciPzjH+skODVVhzDnZ6aPGL4nIUVWNct9+C1iuqn+psj5QnR2uw5gGY3sExsBXQHv3/ANfiMjbwBoRCRSRP4nIMve473eefICIPCgia9yD5z3hXvaauOeBEJEnRGS9+3FT3cseFZEH3Ld7ici37vUfiki8e/lCEXnSPR7+ZhEZ0tAvhvE/fn1lsTEiEgSMBOa6F/XHNZTEdhGZBBxR1X4iEgp8IyKf4RojZwwwQFWLpdpEMO771wOdVVVFJK6Gp34duFdVvxSR3+G6evRn7nVBqtpfRK52Lz9rd5Mx9cH2CIy/CnePMpmFa7yjv7mXL1XVkwONDcM1vMhKYAmQCHTA9cb8qqoWA+iZI34WAqXAyyJyA65hAE4RkVggTlVPzno2E9dkJCf9w/19Oa4xZ4zxKtsjMP6qRFV7VV3gHrm16ng9gutT+7xq7UZwjmF/3ePa9AeuxDUq6D3AFReQ7bj7ewX2P2oagO0RGHN283ANkBcMICIdRSQS+Az4r5NnGtXQNRQFxKrqHFzdPb2qrlfVI0B+lf7/HwOOzIlsDNinDWPO5WVcXTPfucenPwSMUdW5ItILyBKRE7hGg/x1lcdFA/8UkTBcexU/r2HbtwPT3cVkG/UwgqQxtWWnjxpjjJ+zriFjjPFzVgiMMcbPWSEwxhg/Z4XAGGP8nBUCY4zxc1YIjDHGz1khMMYYP/f/AZ/RiXIeyGQpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(df_scores['precision'], df_scores['recall'], label='Precision Vs Recall')\n",
    "plt.xlabel('Precision')\n",
    "plt.ylabel('Recall')\n",
    "plt.title('Recall Vs Precision')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1.4 Area under the PR curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "x is neither increasing nor decreasing : [0.27395316 0.29683886 0.31630972 0.33333333 0.34762774 0.35620301\n 0.36319846 0.36722606 0.37703252 0.38883035 0.39696312 0.4087346\n 0.41762014 0.43010753 0.43866171 0.44656489 0.45300261 0.46318608\n 0.47252747 0.47807638 0.48330914 0.49180328 0.50077519 0.50552923\n 0.51050081 0.515      0.51965812 0.52879581 0.5353902  0.54392523\n 0.54615385 0.54901961 0.5515873  0.55533199 0.56378601 0.56784969\n 0.57051282 0.57359307 0.58426966 0.58604651 0.6        0.60740741\n 0.60598504 0.61636829 0.61417323 0.62912088 0.64183381 0.64497041\n 0.65558912 0.65944272 0.67524116 0.67647059 0.68686869 0.68292683\n 0.69285714 0.70075758 0.70542636 0.72427984 0.73706897 0.73423423\n 0.73891626 0.74742268 0.7540107  0.76878613 0.77160494 0.77777778\n 0.7862069  0.78358209 0.78333333 0.77876106 0.79166667 0.81818182\n 0.83823529 0.83928571 0.83673469 0.825      0.83333333 0.83333333\n 0.86363636 0.88235294 0.92857143 1.         1.         1.\n        nan        nan        nan        nan        nan        nan\n        nan        nan        nan        nan        nan        nan\n        nan        nan        nan        nan        nan].",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-b97c1b7f8920>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m##When calculating auc with precision first and recall second, getting error - x is neither increasing nor decreasing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mauc_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mauc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_scores\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'precision'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdf_scores\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'recall'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mauc_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/metrics/_ranking.py\u001b[0m in \u001b[0;36mauc\u001b[0;34m(x, y)\u001b[0m\n\u001b[1;32m     95\u001b[0m             \u001b[0mdirection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 97\u001b[0;31m             raise ValueError(\"x is neither increasing nor decreasing \"\n\u001b[0m\u001b[1;32m     98\u001b[0m                              \": {}.\".format(x))\n\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: x is neither increasing nor decreasing : [0.27395316 0.29683886 0.31630972 0.33333333 0.34762774 0.35620301\n 0.36319846 0.36722606 0.37703252 0.38883035 0.39696312 0.4087346\n 0.41762014 0.43010753 0.43866171 0.44656489 0.45300261 0.46318608\n 0.47252747 0.47807638 0.48330914 0.49180328 0.50077519 0.50552923\n 0.51050081 0.515      0.51965812 0.52879581 0.5353902  0.54392523\n 0.54615385 0.54901961 0.5515873  0.55533199 0.56378601 0.56784969\n 0.57051282 0.57359307 0.58426966 0.58604651 0.6        0.60740741\n 0.60598504 0.61636829 0.61417323 0.62912088 0.64183381 0.64497041\n 0.65558912 0.65944272 0.67524116 0.67647059 0.68686869 0.68292683\n 0.69285714 0.70075758 0.70542636 0.72427984 0.73706897 0.73423423\n 0.73891626 0.74742268 0.7540107  0.76878613 0.77160494 0.77777778\n 0.7862069  0.78358209 0.78333333 0.77876106 0.79166667 0.81818182\n 0.83823529 0.83928571 0.83673469 0.825      0.83333333 0.83333333\n 0.86363636 0.88235294 0.92857143 1.         1.         1.\n        nan        nan        nan        nan        nan        nan\n        nan        nan        nan        nan        nan        nan\n        nan        nan        nan        nan        nan]."
     ]
    }
   ],
   "source": [
    "##When calculating auc with precision first and recall second, getting error - x is neither increasing nor decreasing\n",
    "auc_score = auc(df_scores['precision'],df_scores['recall'])\n",
    "auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since tp and fp are 0 beyond a certain threshold, precision calculation turns out to be a NaN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f56aa4c3580>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAu9UlEQVR4nO3deXhV5bX48e/KRAYyQhKGBMIsAcIUUEEFHABHxOFKrKKoVZyq3quVTli1vbXWtqhoEZUq+rtF62xVVBAUFWSQeY5EICAkYchEQqb1++McYwghOYHsnJyc9XmePDl773fvs16DZ52930lUFWOMMf4rwNsBGGOM8S5LBMYY4+csERhjjJ+zRGCMMX7OEoExxvi5IG8H0Fjt27fXlJQUb4dhjDE+ZdWqVXmqGl/XMZ9LBCkpKaxcudLbYRhjjE8RkZ0nOmaPhowxxs9ZIjDGGD9nicAYY/ycz7URGGPqV15eTnZ2NqWlpd4OxXhBaGgoSUlJBAcHe3yOJQJjWpns7GwiIyNJSUlBRLwdjmlGqsqBAwfIzs6mW7duHp/n2KMhEZkjIjkisuEEx0VEnhKRTBFZJyJDnIrFGH9SWlpKu3btLAn4IRGhXbt2jb4bdLKN4CVgfD3HLwR6uX9uBf7hYCzG+BVLAv7rZP72jj0aUtUvRCSlniITgLnqmgd7mYjEiEhHVf3BiXi27ivkg3V7XRsiTBzcmW7tI5x4K2OM8Sne7DXUGdhdYzvbve84InKriKwUkZW5ubkn9WaZOUU8vSiTpxdl8tTC7fz1k60ndR1jTMMCAwMZNGgQ/fv35+qrr+bIkSOnfM3p06ezYMGCEx6fNWsWc+fOPeX36datG1u3Hvv5cO+99/L44497dH5KSgoDBgwgLS2NUaNGsXPnCcdxnZTRo0dXD6pNSUkhLy/vlK/pzURQ1/1LnavkqOpsVU1X1fT4+DpHSDfo4rSOZP3pYrL+dDFXDU1iyfY8KiqrTupaxpj6hYWFsWbNGjZs2EBISAizZs065nhlZWWjr/nII49w/vnnn/D41KlTmTx5cqOvW9ukSZOYN29e9XZVVRVvvPEG11xzjcfXWLRoEevWrWP06NH84Q9/OOWYnObNRJANJNfYTgL2Nscbj+4TT35JOWuzDzfH2xnj184++2wyMzNZvHgxY8aM4dprr2XAgAFUVlbywAMPMGzYMNLS0njuueeqz3n88ccZMGAAAwcOZNq0aQDceOONvPHGGwBMmzaN1NRU0tLSuP/++wH4/e9/zxNPPAHAmjVrOOOMM0hLS2PixIkcOnQIcH2bfvDBBxk+fDi9e/dmyZIlx8WbkZFxTCL44osvSElJoWvXrmzcuJHhw4czaNAg0tLS2L59e711P/PMM9mzZw8Aubm5XHnllQwbNoxhw4bx1VdfAVBUVMSUKVOq7yLefPNNAG6//XbS09Pp168fDz30UOP/wzeCN7uPvgfcJSLzgNOBfKfaB2o7u2c8gQHC4q25DO0a1xxvaYxXPPz+RjbtLWjSa6Z2iuKhS/t5VLaiooKPPvqI8eNd/UaWL1/Ohg0b6NatG7NnzyY6OpoVK1Zw9OhRRo4cydixY9myZQvvvPMO33zzDeHh4Rw8ePCYax48eJC3336bLVu2ICIcPnz4uPedPHkyTz/9NKNGjWL69Ok8/PDDzJgxozqm5cuX8+GHH/Lwww8f97gpLS2NgIAA1q5dy8CBA5k3bx4ZGRmA6/HTPffcw89+9jPKysoavLOZP38+l19+OQD33HMP9913H2eddRa7du1i3LhxbN68mUcffZTo6GjWr18PUJ20/vjHPxIXF0dlZSXnnXce69atIy0tzaP/7o3lWCIQkX8Bo4H2IpINPAQEA6jqLOBD4CIgEzgCTHEqltqiw4MZ0iWGRVtz+J+xfZrrbY3xGyUlJQwaNAhw3RHcfPPNfP311wwfPry6f/snn3zCunXrqr/l5+fns337dhYsWMCUKVMIDw8HIC7u2C9rUVFRhIaGcsstt3DxxRdzySWXHHM8Pz+fw4cPM2rUKABuuOEGrr766urjV1xxBQBDhw7l+++/rzP+H+8K+vXrx7vvvssjjzwCuL7h//GPfyQ7O5srrriCXr161Xn+mDFj2L9/PwkJCdWPhhYsWMCmTZuqyxQUFFBYWMiCBQuOuQOJjY0F4PXXX2f27NlUVFTwww8/sGnTJt9LBKqa0cBxBe506v0bMrpPAn/5eCs5haUkRIZ6KwxjHOXpN/em9mMbQW0RET/11FNVnn76acaNG3dMmfnz59fbBTIoKIjly5ezcOFC5s2bx8yZM/nss888jq1NmzaAq0G7oqKizjIZGRmMHTuWUaNGkZaWRkJCAgDXXnstp59+Oh988AHjxo3jhRde4Nxzzz3u/EWLFhEREcGNN97I9OnT+dvf/kZVVRVLly4lLCzsmLKqelx9s7KyeOKJJ1ixYgWxsbHceOONjo4U99u5hkb1djU6f7Ht1FvcjTGNN27cOP7xj39QXl4OwLZt2yguLmbs2LHMmTOnuqdR7UdDRUVF5Ofnc9FFFzFjxozjEk50dDSxsbHVz/9feeWV6rsDT/Xo0YN27doxbdq06sdCADt27KB79+784he/4LLLLmPdunUnvEZYWBgzZsxg7ty5HDx4kLFjxzJz5szq4z/GXXv/oUOHKCgoICIigujoaPbv389HH33UqPgby28TQb9OUcRHtmHx1hxvh2KMX7rllltITU1lyJAh9O/fn9tuu42KigrGjx/PZZddRnp6OoMGDapuAP5RYWEhl1xySXX3zL///e/HXfvll1/mgQceIC0tjTVr1jB9+vRGx5eRkcGWLVuYOHFi9b7XXnuN/v37M2jQILZs2dJgL6WOHTuSkZHBM888w1NPPcXKlStJS0sjNTW1uifVb3/7Ww4dOkT//v0ZOHAgixYtYuDAgQwePJh+/fpx0003MXLkyEbH3xjiekLjO9LT07WpFqZ54N9r+WTTflb99nyCAv02J5pWZvPmzfTt29fbYRgvquvfgIisUtX0usr79aff6D4J5JeUs2b3YW+HYowxXuPXieCsXu2ru5EaY4y/8utEEB3m6ka6eJu1E5jWxdce+ZqmczJ/e79OBOB6PLRhTwE5hbaIh2kdQkNDOXDggCUDP/TjegShoY3rEu/3C9OM7hPPXz7eyudbc7k6PbnhE4xp4ZKSksjOzuZkJ2g0vu3HFcoaw+8TQWrHKBIi27B4myUC0zoEBwc3anUqY/z+0ZCIMKp3PEu25dpspMYYv+T3iQBgzGkJFJRWWDdSY4xfskQAjOzp6ka6yEYZG2P8kCUCXN1Ih3aJtfEExhi/ZInAbVSfeDbuLeBgcZm3QzHGmGZlicAttWMUAFl5xV6OxBhjmpclArekWNcc4dmHTn2RbWOM8SWWCNw6VyeCEi9HYowxzcsSgVt4SBDt24bYHYExxu84mghEZLyIbBWRTBGZVsfxWBF5W0TWichyEenvZDwN6RwbbncExhi/41giEJFA4BngQiAVyBCR1FrFfg2sUdU0YDLwpFPxeCIpNozdB+2OwBjjX5y8IxgOZKrqDlUtA+YBE2qVSQUWAqjqFiBFRBIdjKleybHh7DlcQlWVzdpojPEfTiaCzsDuGtvZ7n01rQWuABCR4UBX4Lhp80TkVhFZKSIrnZxRMSk2jPJKZb9NSW2M8SNOJgKpY1/tr9qPAbEisga4G1gNVBx3kupsVU1X1fT4+PgmD/RHSdZzyBjjh5ychjobqDmvcxKwt2YBVS0ApgCIiABZ7h+vSI4LB1xjCYalxHkrDGOMaVZO3hGsAHqJSDcRCQEmAe/VLCAiMe5jALcAX7iTg1d0jnHdEew+aHcExhj/4dgdgapWiMhdwMdAIDBHVTeKyFT38VlAX2CuiFQCm4CbnYrHE6HBgcRHtrGxBMYYv+LoCmWq+iHwYa19s2q8Xgr0cjKGxkrtGMXCzTkcPlJGTHhIwycYY4yPs5HFtfxyfB8Ol5Tz2EdbvB2KMcY0C0sEtfTrFM1NI1OYt2I3K74/6O1wjDHGcZYI6nDv+b3pHBPGr99aT1mFrWNsjGndLBHUIaJNEI9M6Mf2nCKeX7LD2+EYY4yjLBGcwHl9ExnfrwNPf7adPYetO6kxpvWyRFCP31zcF1Ws4dgY06pZIqhHclw4t53TnffX7mV5ljUcG2NaJ0sEDZg6ugcdokJ5+P2NVNqspMaYVsgSQQPCQ4L41UWnsXFvAW+s2t3wCcYY42MsEXjgsoGdGNo1lr98vJWC0nJvh2OMMU3K0SkmWgsR4aFLU7ls5lfM/CyTX1/Ut1HnFx+tICuvuPrn+7xi9hWUEhwYQGhwAKHBgYQFBxIaHEib4ABCgwLpEhfOFUM645qU1RhjnGOJwENpSTFcPTSJf36VxdjURBKjQquPdYoJIzDg+A/svYdLeHZxJq+t2E155U/tC52iQ+kYE0bx0QpKy6soraikpKyS0vJKSiuqqgexfX+gmP8Z28f5yhlj/JolgkZ4YHwfPtqwj6tmLT1m/9Cuscy5cRjRYcEA/JBfwrOLvuO1FbtRlKuGJjOqd3tS2kfQNS6CsJDAet+nskr5zdvrefqzTDrHhDFpeBfH6mSMMZYIGiEhMpR37hzBmt351fsOFB3liU+2kjF7GX+5Oo3XVuxm3nJXArg6PZk7RvcgKTa8Ue8TGCA8enl/9uaX8pt3NtAxJoxRvZ1bmc0Y499E1be6RKanp+vKlSu9HcYxFm/N4bZXVnG0ooqgAOHq9GTuHNP4BFBb0dEK/mvWUnYeKOb1qWfSr1N0E0VsjPE3IrJKVdPrPGaJoGms2nmQTzbu57ozulYvedkU9heUMvGZr6hU5e07RtLJvYqaMcY0Rn2JwLqPNpGhXeP41UV9mzQJACRGhfLPKcM5crSSjOeXsWb34Sa9vjHGWCLwAX06RPLSTcMpr6jiyn98zYwF26iotOmxjTFNwxKBjxjaNZaP7j2HS9M6MmPBdq6atZSsvGJvh2WMaQUcTQQiMl5EtopIpohMq+N4tIi8LyJrRWSjiExxMh5fFx0WzIxJg3k6YzA7cou46Mkl/N83u/C1dh5jTMviWCIQkUDgGeBCIBXIEJHUWsXuBDap6kBgNPBXEbEV4xtw6cBOfHzfOQzpGsOv317PLS+vJLfwqLfDMsb4KCfvCIYDmaq6Q1XLgHnAhFplFIgU1zwKbYGDQIWDMbUaHaPDeOWm05l+SSpLMvMYP+MLPt2039thGWN8kJOJoDNQc7rObPe+mmYCfYG9wHrgHlU9rhVURG4VkZUisjI3N9epeH1OQIBw01nd+M/dZ5EYFcrP565k2pvrKD5qudQY4zknE0Fds6XVfpg9DlgDdAIGATNFJOq4k1Rnq2q6qqbHx9sI29p6J0by9p0jmDqqB6+t3M34J7/ghSU7yCks9XZoxhgf4GQiyAaSa2wn4frmX9MU4C11yQSygNMcjKnVahMUyLQLT2Pez88gLjyEP3ywmTP/9BlT/rmc99fupbS80pH3raxScgpL2bAnn68y8xx7H2OMc5yca2gF0EtEugF7gEnAtbXK7ALOA5aISCLQB9jhYEyt3und2/HuXWeRmVPIm9/u4e1v93D3v1YTGRrEJWkduWJIEuldY4+Z3rq8sop12fkszzpIYWk5VQqqSpUqVYrrd5XrdXllFXlFR8kpPMr+glLyisqOWbmtc0wYv7roNC4e0NGm0DbGRzg6xYSIXATMAAKBOar6RxGZCqCqs0SkE/AS0BHXo6THVPXV+q7ZUqeYaKkqq5RlOw7w5qpsPtqwj5Lyyuq1Dtq2CeLr7w6wPOsgRe52hZDAAEQgQIQA928RV3tEgAiBAUK7iBASo0JJiGzj+h3VhoRI17TcTy7czuYfChieEsf0S1Pp39nmRzKmJbC5hgzgWiDnow37eOvbbJbuOIAqdGsfwYge7RjZsz1ndm9HbMSp9d6trFJeW7GbJz7ZyqEjZYzo0Y4JAzszrn+H6mm6jTHNzxKBOc7+glIqq9SxSezyS8r551dZvL16DzsPHCEkKIBz+yRw4YAO9IhvS+eYMGLCg+3xkTHNxBKB8RpVZV12Pu+s2cP7a38gr+ingW/hIYF0igmjc0wYSbFhXDU0icFdYr0YrTGtlyUC0yJUVimbfygg+9AR9hwuZc+hEvYcPsKewyV8n3eE4rIKJp/RlfvH9SEy1B4jGdOU6ksEtkKZaTaBAUL/ztF1NiAXlpbz10+28fLS7/l4434emdCPsf06eCFKY/yPzT5qWoTI0GB+f1k/3rp9BDHhwdz6yipuf3UV+wtsUJwxTrNEYFqUwV1ief/us/jl+D58tiWH8//6OXOXfm/rLxjjIEsEpsUJDgzgjtE9+eS+cxiYHMP0dzdy4ZNLWLw1x9uhGdMqWSIwLVbXdhG8cvNwnrt+KOWVVdz4zxVMnrOczJxCb4dmTKtiicC0aCLCuH4d+OS+Ufz24r6s3nWI8TOW8Mj7m8gvKfd2eMa0CtZ91PiUA0VHeeKTbcxbsYu48BBuG9WdPh2iSIp1jUcIDQ70dojGtEg2jsC0Ohv25PPw+xtZ8f2hY/a3b9vGlRRiXYPUkmLCSIoNJz6yDQeLy9hzuMQ9fsH1e29+CWd2b8ejl/e3JGJaNUsEplVSVX7IL2XP4RKyDx0h+2CJ+/VPH/RldfQ2CgwQOkSF0jkmjOjwYD7dtJ9hKbE8PzmdmHBbKdW0TjagzLRKIkKnmDA6xYQxLCXuuONVVUpe0VF2Hyohp6CUuIgQOseG0SEqlKDAn5rH3l+7l/95fS1X/ONrXp4ynOS48OashjFeZ4nAtFoBAUJCVCgJUaH1lrt0YKfqpT4nPvsVc24cRlpSTPMEaUwLYL2GjAGGd4vjzdtHEBocyPUvLmfrPuuiavyHJQJj3HomtOVfPz+D0OAArn/xG3YdOOLtkIxpFpYIjKkhOS6cV24+nbLKKq578RtybK4j4wcsERhTS+/ESP554zDyio4yec5y8o/YwDXTujmaCERkvIhsFZFMEZlWx/EHRGSN+2eDiFSKyPHdP4xpZoO7xDL7+nR25BYz5aXlFJZaMjCtl0eJQERGisinIrJNRHaISJaI7GjgnEDgGeBCIBXIEJHUmmVU9S+qOkhVBwG/Aj5X1YMnVRNjmthZvdrzVMYg1uw+zOi/LOafX2VxtKLS22EZ0+Q87T76InAfsArw9P+E4UCmqu4AEJF5wARg0wnKZwD/8vDaxjSL8f078vYdI/nz/C08/P4mnlq4nTF9EjivbyLn9G5vK6mZVsHTRJCvqh818tqdgd01trOB0+sqKCLhwHjgrhMcvxW4FaBLly6NDMOYUzMwOYb/d8vpfJV5gLe+zWbR1hzeWr2H4EDh9G7tOK9vAuf3TbSBaMZneTTFhIg8BgQCbwHVq4+r6rf1nHM1ME5Vb3FvXw8MV9W76yh7DXCdql7aUCw2xYTxtsoq5dtdh1iweT8LN+eQmVMEwIge7Xj5puEEB1ofDNPyNMUUEz9+k695EQXOreecbCC5xnYSsPcEZSdhj4WMjwgMEIalxDEsJY5fXdiXnQeKeevbPTy5cDsvf/09t5zd/ZjyFZVVfH/gCNFhwcRHtvFS1MacmEeJQFXHnMS1VwC9RKQbsAfXh/21tQuJSDQwCrjuJN7DGK/r2i6C+y7ozbrsw8xYsJ2EqFB25hWzLaeI7fsL2ZFbTFllFeEhgbwxdQSpnaK8HbIxx/C011C0iPxNRFa6f/7q/gA/IVWtwPXM/2NgM/C6qm4UkakiMrVG0YnAJ6pafLKVMKYl+N0lqRytqOQX/1rNXz/dxupdh+gUE8aUs1J4/Mo0okKDueXlFeQU2iA107J42kbwJrABeNm963pgoKpe4WBsdbI2AtOSbdpbQFllFb0S2hLR5tgb7g178rl61lJ6J7blF+f1omu7CLq3jyAgQLwUrfEnp7wegYiscff1r3dfc7BEYHzZJxv3cef/fUt5pev/u8sGduKpjMFejsr4g/oSgafdG0pE5KwaFxwJlDRFcMb4k7H9OrDiN+fz1h0juO6MLry3di/f7Djg7bCMn/M0EdwOPCMi34vITmAmMLWBc4wxdYgJD2FIl1h+c1EqHaJC+d8PN1NV5VsrBZrWxaNEoKprVHUgkAYMUNXBqrrW2dCMad3CQgK5f1wf1mbn8/66E/WsNsZ59XYfFZHrVPVVEfnvWvsBUNW/ORibMa3exMGdmfNlFn/4YDMHisq4cEAHOkaHeTss42cauiOIcP+OPMGPMeYUBAYIf74yjXYRITzyn02c+afPyJi9zKa+Ns3Ko15DLYn1GjKt1Xe5Rby7eg9PfZbJY1cMYNJwm1fLNJ1T7jUkIo+LSJSIBIvIQhHJExEbCWxME+oR35b7LuhNh6hQlmzP83Y4xo942mtorKoWAJfgmkOoN/CAY1EZ46dEhLN7tWfhlv3cO281sz7/jsVbc9iXX4qv3b0b3+HppHM/Trp+EfAvVT34Y4OxMaZpTRnZjQPFZSzPOsg7a37qTTQoOYbZk4eSEBnqxehMa+RpInhfRLbgGkR2h4jEAzZhijEOSO0UxZwbhwGQf6ScLfsKWJt9mL9/up2Jz3zNS1OG0SvR+mqYpuNxY7GIxAIFqlrpXkgmSlX3ORpdHayx2Pir9dn53PTyCkrLK3nu+qGM6NHe2yEZH3LSjcUicq779xXAGGCC+/V4YERTB2qMObEBSdG8fccIOkSFcsOc5by7Zo+3QzKtREOPhkYBnwF1rRymuFYsM8Y0k6TYcN64fQS3zl3Jva+tQRUuH9zZ22EZH1dvIlDVh9y/pzRPOMaYhkSHBfPSlOFMeWk5//36GoIDA7g4raO3wzI+zNNxBP8rIjE1tmNF5A+ORWWMqVdYSCAv3jCMIV1i+cW81czfcHxznapSVlHlheiMr/F0PYLVqjq41r5vVXWIY5GdgDUWG/OTwtJyrn9xORv35vOz07tSUFrOvvxS109BKSXllcy5YRhjTkvwdqjGy5piPYJAEaledVtEwgBbhdsYL4sMDeblm4YzKDmGV5ft5JsdByktr6RvxygyhnehS1w4j/xnk90ZmHp5Oo7gVWChiPwTVyPxTfy0bOUJich44EkgEHhBVR+ro8xoYAauQWt5qjrKw5iMMbjaDF6/7UxUOW7Zy5E923HTSyv500ebSYgM5UDRUW4cmUJSbLiXojUtUWPGEYwHzgcE12LzHzdQPhDYBlyAa1qKFUCGqm6qUSYG+BoYr6q7RCRBVXPqu649GjLGc6rK5DnLq+cuCgwQYsODmT05nSFdYr0cnWlO9T0a8vSOAGAzUKGqC0QkXEQiVbWwnvLDgUxV3eEOYh4wAdhUo8y1wFuqugugoSRgjGkcEWHmtUPIyiume3wEOQVHuemlFUyavYyZGYMZ26+Dt0M0LYCnvYZ+DrwBPOfe1Rl4p4HTOgO7a2xnu/fV1BuIFZHFIrJKRCaf4P1vFZGVIrIyNzfXk5CNMW7RYcEMSo4hKjSYnglteefOkfRJjOT+f68lt/Cot8MzLYCnjcV3AiOBAgBV3Q401A2hrlnpaj+HCgKGAhcD44DfiUjv405Sna2q6aqaHh8f72HIxpi6xEWE8PdrBlFaXsUfPtjU8Amm1fM0ERxV1bIfN0QkiOM/1GvLBpJrbCcBtRdmzQbmq2qxquYBXwADPYzJGHOSeia05fbRPXh3zV6+2GZ32f7O0zaCz0Xk10CYiFwA3AG838A5K4BeItIN2ANMwtUmUNO7wEx3YgkBTgf+7mnwxpiTd/voHry/di/3zFvNgKQY4sKDiY0IITY8hNiIEOLbhnBO73jCQxrTlGh8kad/4QeBW4D1wG3Ah8AL9Z2gqhUichfwMa7uo3NUdaOITHUfn6Wqm0VkPrAOqMLVxXTDyVXFGNMYocGBPJUxmL9/uo28oqNk5RVxuLicwqMV1WV6xEcw89oh9O0Y5cVIjdMa7D4qIgHAOlXt3zwh1c+6jxrjrLKKKg6XlLFhTz7T3lzP4ZJyfntxX64/oyu2IJXvOqWRxapaBawVEVtJ2xg/EBIUQEJkKOeelshH95zNyB7tmP7uRm57ZRU5hbYeVWvk6VxDnwHDgOVA8Y/7VfUy50Krm90RGNO8qqqUOV9l8ef5WwgMEK5JT2Zwl1j6doyie3wEwYGe9jkx3lTfHYGniaDOaR9U9fNTjK3RLBEY4x3f5xXzt0+3MX/DPsoqXXMXhQQGMKxbLI9O6E/3+LZejtDU56QTgYiEAlOBnrgail9U1YoTntAMLBEY413llVXsyC1m8w8FbPqhgNdW7KasooqnMwZzfmqit8MzJ3AqieA1oBxYAlwI7FTVexyJ0kOWCIxpWfbll3LL3BXsPljCJ/edQ2JUqLdDMnU4lcbiVFW9TlWfA64Czm7y6IwxPq1DdChPTRrM0YpK7v/3Wo5WVHo7JNNIDSWC8h9fePuRkDGm5eoe35bfXZLKku15XPzUl3y5PQ9PZzY23tfQgLKBIlLgfi24RhYXuF+rqtooE2MMAD87vSudosP4zdvrue7FbzitQyTn903k9O5xDO0aayOUWzCP1yNoKayNwJiWrbS8knfX7GHeit2sy86nskoJChA6xoTSMSqMDtGhdIwOpUN0KKd1iOLMHu28HbJfOOXuoy2JJQJjfEfR0QpW7TzEiqyD7D50hB9+XE85v7S6C+oVgzvzyOX9advG7hic1FQL0xhjTKO0bRPEqN7xjOp97PTxqsrB4jLmLt3J059tZ/Xuwzw1aTADkqK9FKl/szsCY4xXLdtxgHvnrSG36ChDu8Yyqnc8o/vEk9oxyuY2akL2aMgY06IdKi7j+SU7WLw1l00/uPqnxEe2qb6bGNevAyFBNpXFqbBEYIzxGTkFpXy+LZfPt+WyZHse+SXl3Dgihd9f1s/bofm0U5p91BhjmlNCVChXpycz89ohfPu7Cxibmsh/1u2lssq3vrT6EksExpgWKzBAuHRgJ/KKynh8/ha27S+0gWoOsF5DxpgW7by+CZzTO57nvtjBc1/soHv7CMb268AZ3eMYnBxLdHiwt0P0edZGYIzxCTkFpXyyaT8fb9zH0u8OUOF+VNQ9PoIhXWIZ3CWGET3a0619hJcjbZm81lgsIuOBJ3GtWfyCqj5W6/hoXAvYZ7l3vaWqj9R3TUsExpiioxWs232Y1bsPs3rXIb7ddZiDxWUAnNWzPTeMSOHc0xIIDLDupz/yyoAyEQkEngEuALKBFSLynqpuqlV0iape4lQcxpjWp22bIEb0bM+Inu0B1wC1nQeO8MH6H3h12U5+PnclXeLC+e8LenPZwE4EWEKol5ONxcOBTFXdoaplwDxggoPvZ4zxUyJCSvsI7hzTkyW/HMOzPxtCZGgQ9762hktnfsn8Dfus11E9nEwEnYHdNbaz3ftqO1NE1orIRyJSZ0dhEblVRFaKyMrc3FwnYjXGtBJBgQFcNKAj7991FjOuGURBaTlTX13F6CcW8eKXWRSWljd8ET/jWBuBiFwNjFPVW9zb1wPDVfXuGmWigCpVLRKRi4AnVbVXfde1NgJjTGNUVFbx6ab9vPhlFit3HqJtmyD+Kz2ZKSNTSI4L93Z4zcZbA8qygeQa20nA3poFVLVAVYvcrz8EgkWkvYMxGWP8TFBgABcO6Mgbt4/g3TtHcn7fBOYu/Z7z//Y5i7bmeDu8FsHJRLAC6CUi3UQkBJgEvFezgIh0EPesUiIy3B3PAQdjMsb4sYHJMcyYNJglD46hZ0Jbbpu7ik827vN2WF7nWCJwL215F/AxsBl4XVU3ishUEZnqLnYVsEFE1gJPAZPU1wY2GGN8TsfoMP7vljPo2ymKO/7ft/z2nfWs2nnIb0ct24AyY4zfKiwt56F3N/LB+h84WlFFl7hwLh/cmYsHdKR3YttWNQ22zT5qjDH1KCwt5+ON+3ln9R6+/i6PKoWUduGM69eBsf0SGZwc6/NjESwRGGOMh3IKS1mwKYf5G/ex9Ls8yiuVpNgwXrvtTDrHhHk7vJNmicAYY05Cfkk5n23Zz6/f2sDInu14fnK6zz4usvUIjDHmJESHBTNxcBL3XdCLBZtz+Hjjfm+H5AhLBMYY04ApI7vRO7EtMxdt93YojrBEYIwxDQgODGDCoM5s2FNATkGpt8NpcpYIjDHGA6P7xAPw5rd7Wt14A0sExhjjgdSOUQxKjuHP87eQ8fwy1mfnezukJmOJwBhjPCAi/HvqmTwyoR/b9xdx6cwvmf7uBioqq7wd2imzRGCMMR4KDgxg8pkpLH5gNDeN7MbcpTuZ+uq3lJRVeju0U2KJwBhjGikyNJjpl6by6IR+LNyyn4znl7F292Fvh3XSLBEYY8xJuv7MFJ69dghZecVMeOYrbpiznFU7D3o7rEazRGCMMafgwgEd+fLBMfxyfB/W78nnyn8sJWP2Mp9KCJYIjDHmFEWGBnPH6J58+eAYfntxX7bnFPFfzy3jlWU7vR2aRywRGGNMEwkPCeKWs7uz+IHRnNOrPb97ZwN/+M8mKqta9rgDSwTGGNPE2rYJ4vnJ6dxwZlde+DKLqa+u4khZhbfDOiFLBMYY44CgwAAentCfhy5NZeHm/dz32hpvh3RCQd4OwBhjWrMpI7uxPOsgW/cXejuUE3L0jkBExovIVhHJFJFp9ZQbJiKVInKVk/EYY4w3BLbw1c0cSwQiEgg8A1wIpAIZIpJ6gnJ/xrXIvTHGtDqRocHkFR6lqoU2Gjt5RzAcyFTVHapaBswDJtRR7m7gTSDHwViMMcZrhqXEUlBawZZ9LfPxkJOJoDOwu8Z2tntfNRHpDEwEZjkYhzHGeNXp3dsB8E3WAS9HUjcnE0FdD8Vq3xfNAB5U1XpnbBKRW0VkpYiszM3Nbar4jDGmWXSOCaN7fATPLv6OrS3wrsDJRJANJNfYTgL21iqTDswTke+Bq4BnReTy2hdS1dmqmq6q6fHx8Q6Fa4wxzpl13VACBP7ruaV8u+uQt8M5hpOJYAXQS0S6iUgIMAl4r2YBVe2mqimqmgK8Adyhqu84GJMxxnhF78RI3pg6gpjwYCa/uJy8oqPeDqmaY4lAVSuAu3D1BtoMvK6qG0VkqohMdep9jTGmpUqOC+fFG4ZRXFbB80t2eDucao4OKFPVD4EPa+2rs2FYVW90MhZjjGkJeia05dK0TryydCe3ndODuIgQb4dkU0wYY0xzu/vcnpSUV/LSV1neDgWwRGCMMc2uV2IkI3q048MN+7wdCmCJwBhjvOK80xLJzCniu9wib4diicAYY7zh/L6JBAUI42d8wTXPLeWZRZls2JPvlVgsERhjjBd0aRfOm7eP4OazulNYWsFfPt7KJU9/yd8+3dbssdg01MYY4yUDk2MYmBzDtAtPI7fwKH/6cDNPLdxOz4S2XDawU7PFYXcExhjTAsRHtuGxK9MY3i2OX76xtlnbDiwRGGNMCxESFMDMjMG0CQrk/n+vpaKyqlne1xKBMca0IAlRoTx6eX9W7zrMy0t3Nst7WiIwxpgW5tK0jpzRPY45X2ZR2QyL2VgiMMaYFkZEuP6MFPYcLuGLbc5PvW+JwBhjWqALUhNp37YNry5z/vGQJQJjjGmBQoICmDQsmc+25pBTWOroe1kiMMaYFuqc3vGowsa9BY6+jyUCY4xpoXontgVgm8PLW1oiMMaYFiomPITEqDZs3W+JwBhj/FbvxEi2WSIwxhj/1Tsxku37i6hycDyBJQJjjGnBesS35WhFFT8UONdzyNFEICLjRWSriGSKyLQ6jk8QkXUiskZEVorIWU7GY4wxvqZb+wgAsnKLHXsPxxKBiAQCzwAXAqlAhoik1iq2EBioqoOAm4AXnIrHGGN8Ufd4VyLYkefcbKRO3hEMBzJVdYeqlgHzgAk1C6hqkar++OArAnB+Ug1jjPEhCZFtiAgJZIcv3hEAnYHdNbaz3fuOISITRWQL8AGuu4LjiMit7kdHK3NznZ93wxhjWgoRoVt8BDvyfDMRSB37jvvGr6pvq+ppwOXAo3VdSFVnq2q6qqbHx8c3bZTGGNPCdYkLJ/vgEceu72QiyAaSa2wnAXtPVFhVvwB6iEh7B2MyxhifkxwbTvbhEse6kDqZCFYAvUSkm4iEAJOA92oWEJGeIiLu10OAEOCAgzEZY4zPSYoLp6yiityio45c37HF61W1QkTuAj4GAoE5qrpRRKa6j88CrgQmi0g5UAJcU6Px2BhjDJAUGwbA7oNHSIwKbfLrO5YIAFT1Q+DDWvtm1Xj9Z+DPTsZgjDG+Ljk2HIDsQyWkpzT99W1ksTHGtHA17wicYInAGGNauNDgQCYM6kRyXLgj13f00ZAxxpim8eSkwY5d2+4IjDHGz1kiMMYYP2eJwBhj/JwlAmOM8XOWCIwxxs9ZIjDGGD9nicAYY/ycJQJjjPFz4mtzvIlILrCzkae1B/IcCKcl88c6g3/W2+rsP06l3l1Vtc4FXXwuEZwMEVmpqunejqM5+WOdwT/rbXX2H07V2x4NGWOMn7NEYIwxfs5fEsFsbwfgBf5YZ/DPelud/Ycj9faLNgJjjDEn5i93BMYYY07AEoExxvi5VpUIRGS8iGwVkUwRmVbHcRGRp9zH14nIEG/E2ZQ8qPPP3HVdJyJfi8hAb8TZlBqqc41yw0SkUkSuas74nOJJvUVktIisEZGNIvJ5c8fY1Dz49x0tIu+LyFp3nad4I86mJCJzRCRHRDac4HjTf46paqv4AQKB74DuQAiwFkitVeYi4CNAgDOAb7wddzPUeQQQ6359oT/UuUa5z4APgau8HXcz/a1jgE1AF/d2grfjboY6/xr4s/t1PHAQCPF27KdY73OAIcCGExxv8s+x1nRHMBzIVNUdqloGzAMm1CozAZirLsuAGBHp2NyBNqEG66yqX6vqIffmMiCpmWNsap78nQHuBt4EcpozOAd5Uu9rgbdUdReAqvp63T2pswKRIiJAW1yJoKJ5w2xaqvoFrnqcSJN/jrWmRNAZ2F1jO9u9r7FlfElj63Mzrm8SvqzBOotIZ2AiMKsZ43KaJ3/r3kCsiCwWkVUiMrnZonOGJ3WeCfQF9gLrgXtUtap5wvOaJv8ca02L10sd+2r3jfWkjC/xuD4iMgZXIjjL0Yic50mdZwAPqmql64tiq+BJvYOAocB5QBiwVESWqeo2p4NziCd1HgesAc4FegCfisgSVS1wODZvavLPsdaUCLKB5BrbSbi+JTS2jC/xqD4ikga8AFyoqgeaKTaneFLndGCeOwm0By4SkQpVfadZInSGp/++81S1GCgWkS+AgYCvJgJP6jwFeExdD88zRSQLOA1Y3jwhekWTf461pkdDK4BeItJNREKAScB7tcq8B0x2t7qfAeSr6g/NHWgTarDOItIFeAu43oe/GdbUYJ1VtZuqpqhqCvAGcIePJwHw7N/3u8DZIhIkIuHA6cDmZo6zKXlS51247oAQkUSgD7CjWaNsfk3+OdZq7ghUtUJE7gI+xtXbYI6qbhSRqe7js3D1ILkIyASO4Po24bM8rPN0oB3wrPsbcoX68KyNHta51fGk3qq6WUTmA+uAKuAFVa2zC6Iv8PBv/Sjwkoisx/XI5EFV9enpqUXkX8BooL2IZAMPAcHg3OeYTTFhjDF+rjU9GjLGGHMSLBEYY4yfs0RgjDF+zhKBMcb4OUsExhjj5ywRGFMH96yla0Rkg3t2y5gmvv73ItLe/bqoKa9tTGNZIjCmbiWqOkhV++OaAOxObwdkjFMsERjTsKW4J/USkR4iMt89qdsSETnNvT9RRN52z4u/VkRGuPe/4y67UURu9WIdjDmhVjOy2BgniEggrikMXnTvmg1MVdXtInI68CyuCc+eAj5X1Ynuc9q6y9+kqgdFJAxYISJvtoL5nkwrY4nAmLqFicgaIAVYhWtWy7a4Fvr5d41ZTdu4f58LTAZQ1Uog373/FyIy0f06GegFWCIwLYolAmPqVqKqg0QkGvgPrjaCl4DDqjrIkwuIyGjgfOBMVT0iIouBUCeCNeZUWBuBMfVQ1XzgF8D9QAmQJSJXQ/XasT+uAb0QuN29P1BEooBo4JA7CZyGa1lBY1ocSwTGNEBVV+NaL3cS8DPgZhFZC2zkp6UT7wHGuGfBXAX0A+YDQSKyDtcsmcuaO3ZjPGGzjxpjjJ+zOwJjjPFzlgiMMcbPWSIwxhg/Z4nAGGP8nCUCY4zxc5YIjDHGz1kiMMYYP/f/AZ0ryrn/7OVtAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plot of Recall (x-axis) Vs Precision (y-axis)\n",
    "plt.plot(df_scores['recall'], df_scores['precision'], label='Precision Vs Recall')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#When calculating auc with recall first and precision second, dont get the above error\n",
    "auc_score = auc(df_scores['recall'],df_scores['precision'])\n",
    "auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AUC is NaN, since precision is NaN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since tp and fp are 0 beyond a certain threshold and precision, auc calculations are NaNs. \n",
    "\n",
    "Calculating Precision, Recall and plot of these 2 scores for the following:\n",
    "1. Filling NaNs in Precision with 1 (*Since precision converges to 1 beyond that certain threshold, filling all NaNs with 1 for precision to see the graphs and auc*)\n",
    "2. Dropping NaNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_scores['precision'].isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filling NaNs in precision with 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scores_1 = df_scores.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scores_1 = df_scores_1.fillna(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>tp</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tn</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>386</td>\n",
       "      <td>1023</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.273953</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.430084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.1</td>\n",
       "      <td>366</td>\n",
       "      <td>556</td>\n",
       "      <td>20</td>\n",
       "      <td>467</td>\n",
       "      <td>0.396963</td>\n",
       "      <td>0.948187</td>\n",
       "      <td>0.559633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.2</td>\n",
       "      <td>333</td>\n",
       "      <td>356</td>\n",
       "      <td>53</td>\n",
       "      <td>667</td>\n",
       "      <td>0.483309</td>\n",
       "      <td>0.862694</td>\n",
       "      <td>0.619535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.3</td>\n",
       "      <td>284</td>\n",
       "      <td>236</td>\n",
       "      <td>102</td>\n",
       "      <td>787</td>\n",
       "      <td>0.546154</td>\n",
       "      <td>0.735751</td>\n",
       "      <td>0.626932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.4</td>\n",
       "      <td>249</td>\n",
       "      <td>166</td>\n",
       "      <td>137</td>\n",
       "      <td>857</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.645078</td>\n",
       "      <td>0.621723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.5</td>\n",
       "      <td>210</td>\n",
       "      <td>101</td>\n",
       "      <td>176</td>\n",
       "      <td>922</td>\n",
       "      <td>0.675241</td>\n",
       "      <td>0.544041</td>\n",
       "      <td>0.602582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>150</td>\n",
       "      <td>53</td>\n",
       "      <td>236</td>\n",
       "      <td>970</td>\n",
       "      <td>0.738916</td>\n",
       "      <td>0.388601</td>\n",
       "      <td>0.509338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.7</td>\n",
       "      <td>76</td>\n",
       "      <td>20</td>\n",
       "      <td>310</td>\n",
       "      <td>1003</td>\n",
       "      <td>0.791667</td>\n",
       "      <td>0.196891</td>\n",
       "      <td>0.315353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.8</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>373</td>\n",
       "      <td>1022</td>\n",
       "      <td>0.928571</td>\n",
       "      <td>0.033679</td>\n",
       "      <td>0.065000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>386</td>\n",
       "      <td>1023</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>386</td>\n",
       "      <td>1023</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     threshold   tp    fp   fn    tn  precision    recall        f1\n",
       "0          0.0  386  1023    0     0   0.273953  1.000000  0.430084\n",
       "10         0.1  366   556   20   467   0.396963  0.948187  0.559633\n",
       "20         0.2  333   356   53   667   0.483309  0.862694  0.619535\n",
       "30         0.3  284   236  102   787   0.546154  0.735751  0.626932\n",
       "40         0.4  249   166  137   857   0.600000  0.645078  0.621723\n",
       "50         0.5  210   101  176   922   0.675241  0.544041  0.602582\n",
       "60         0.6  150    53  236   970   0.738916  0.388601  0.509338\n",
       "70         0.7   76    20  310  1003   0.791667  0.196891  0.315353\n",
       "80         0.8   13     1  373  1022   0.928571  0.033679  0.065000\n",
       "90         0.9    0     0  386  1023   1.000000  0.000000  1.000000\n",
       "100        1.0    0     0  386  1023   1.000000  0.000000  1.000000"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_scores_1[::10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f56aa353be0>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAxb0lEQVR4nO3de3zO9f/H8cd7Z9vY2OY4sxE2ZA7LWYiYQ0QUKikllc4nfb/fDvp961t9OyGRkKLyjcqpSIjkTGbYmDlkc5gd7GQ2O7x/f3ykxdiF69rnuq697rfbdbNrn8+u6/WeeXrv/Xl/3m+ltUYIIYTjczG7ACGEENYhgS6EEE5CAl0IIZyEBLoQQjgJCXQhhHASbma9cWBgoA4NDTXr7YUQwiHt2LEjTWsdVNYx0wI9NDSU7du3m/X2QgjhkJRSf1zumAy5CCGEk5BAF0IIJyGBLoQQTkICXQghnIQEuhBCOIlyA10pNVspdUoptecyx5VSarJSKlEpFauUamP9MoUQQpTHkh76HCD6Csf7Ao3PP8YC066/LCGEEFer3HnoWutflVKhVzhlEPCFNtbh3ayU8ldK1dFan7BWkX+TdgBiv4Ga4RAUAQE3gJuHTd5KCFFxNiSmseVQutllVIio0Brc3KTMe4OuizVuLKoHJJV6nnz+c5cEulJqLEYvnpCQkGt7t5OxsP5d0CXGcxc3I9SDwqFmxF9/1mgIru7X9h5CiAr34rexJJ8+i1JmV2J747o1sttAL+vbX+auGVrrGcAMgKioqGvbWaPFHdC0P6QlQOo+OBVv/HliF8Qt/uutXT3hhp7QfDA0iQavatf0dkII20vNKSD59Fn+2S+Ch25uaHY5DssagZ4M1C/1PBg4boXXvTx3L6jT0niUdi7vr6A/vhPil8L+H41wb3zr+XDvA55VbVqeEOLqxCRlAtAqxN/UOhydNQJ9CTBeKTUfaA9k2Wz8vDwe3lC3lfGIHA59/gPJ22Dvd0bvfd8ycPMqFe7R4OFjSqlCiL/EJJ3GzUXRoq6f2aU4tHIDXSn1NdAdCFRKJQOvAu4AWuvpwI9APyARyAPut1WxV83FBULaG48+/4GkLX+Fe/xScKsCPV+BDo9QKQbuhLBTO49mEl6nKlU8XM0uxaFZMstlRDnHNfCY1SqyFRcXaNDReES/BUc3wYbJ8NNLkLYf+r0rF1GFMEFxiSY2OYvbW9c1uxSHVznvFHVxhdAuMGI+dHkGdsyBeUMgaRvkZ5tdnRCVysHUXHILimhVv7rZpTg809ZDtwsuLtDrVQhsDEufhFm9jM9XC4aG3Yxx9rBuMs9dCBuKOZoJQKv6/qbW4Qwqd6D/qdVIaNjDmBmTGg8n90D8Moj5Erz8oc290OlJ8LX+vFEhKrudSZlU83KjYaBMULheEuh/qlbHeIT3M54XFcDBNRD7P9g0FbbNgnYPQeQIqNFIeu1CWElMUiaR9f1xcZGJCddLAv1y3DyhaV/jkXYA1r1tXETdMMm4O7VGI/AJ/Ov8G3pC56eNYRwhhEXOFBSx/2Q2t/a4wexSnIIEuiUCG8MdM6HHP4157X/enVqQYxw/lwurXzfuVr19ujEfXghRrt3HsijRckORtUigX40aYcbjYlobwzIr/wWZR2H418bwjRDiii7cISozXKxCxgesQSnoNB5GfA2pCTC7N2Qmlf91QlRyMUczaRDgTQ0fuSZlDRLo1tS0L4xeBmez4PMBkJVsdkVC2K2i4hK2HsmgTYj0zq1FAt3a6rWBe7+HvAyYMwCyjpldkRB2afOhDDLOnKNP81pml+I0JNBtIbitEepn0mB6Z1j/HhTkml2VEHblh90n8PZwpXvTmmaX4jQk0G0lOArG/ATBNxkzYCa1hM3ToKTY7MqEMF1RcQk/7T3JLeE18XKXBbmsRQLdlmo1h7sXwJhVUPtGWDEBPusL6QfNrkwIU205bAy3DGgps8GsSQK9ItS/Ce5dBINnGPPXp3WGVRONJXzTEqXXLiqdZbEy3GILMg+9oigFkXdB2M2w7Gn47f2/jnn5Q8SAvxYDk2V8hROT4RbbkUCvaNXqwMj5xl2mqQnGYmCH18PexbBzHvg3gFGLy76BSQgnIMMttiNDLmbxrGrMhml9Dwz5BJ5PhDvnQkE2fH4bnP7D7AqFsAmZ3WI7Euj2wt0Lmg00xtoLcow57JlHza5KCKtKSMlh8c5j9IqoJcMtNiCBbm/qtoJRi6AgCz7rD6f2mV2REFZx+sw5Hvx8O1U83JjQN9zscpySBLo9qtvaGEcvyodZt0LiKrMrEuK6FBaX8OiXv3MyK58Zo9pS17+K2SU5JQl0e1W3NTy0BvxD4MthsOljmd4oHNYbP8Sz6VA6bw65UdZusSEJdHvmXx8eWAGN+8BPL8HHHWHPt1BSYnZlQlgs7ng2czYeYXSnUIa2DTa7HKcmgW7vPKvC8K9g2BxjLvvCB+CTmyH3lNmVCWGRD1clUNXLjad7NTG7FKcnge4IXFyMm44e2QhDZkJ6IswfCYVnza5MiCvanZzFyrgUHuzSED9vuWHO1iTQHYmLK7QcBkNmGFvhLXpUhl+EXftwVQJ+Vdy5v0uo2aVUCnKnqCNqNhB6vQarXoOARsZep0p2TBfWlV9YzOwNh0nKOEv22UJyCoqo5+/FDTWrEl67Ku3DauDmevk+4c6jp1m97xTP92lKNS/pnVcECXRH1fkpY+jl1//C4V+h+0vQsLsEu7Ca15bsZf62JAJ9PfH3dsfbw5XdyZmczjO2V7wptDpTRrShtp8XAFl5hczb8gdpuQUAbDqYTg0fD+7rFGpWEyodCXRHpRQMmGRMb1z/Psy9HRp0hv7vQc0Is6sTDm7B9iTmb0vi0e6NeCH67zcBpeUWsDo+hYlL4+g/eT3/HdaSAym5TP0lkZyCInw9jVhRwIt9wy88F7antNamvHFUVJTevn27Ke/tdIoK4PcvYO1/jGUDevwDOj1hjLkLcZXiT2Rz+9QNtAmpztwx7S47rJJ4KodH5v3OgVPGblw9mgbxQnQ4EXWqVWS5lY5SaofWOqrMYxLoTiQ3FX542lhnPfgmGDEffALNrko4kNyCIgZMXk/euWJ+eKIrQVU9r3h+3rkiZq0/TNvQ6nRqJD9rFeFKgS6zXJyJb5CxYuMds+Dk7vNTG/PNrko4kCmrD3AkPY8pI1qXG+YA3h5uPN6zsYS5nbAo0JVS0Uqp/UqpRKXUhDKO+ymlliqldiml9iql7rd+qcIiSsGNQ2HwdEjaAkvGg0m/hQnHcjjtDLM3HGZY22DaNwwwuxxxDcoNdKWUKzAV6As0A0YopZpddNpjQJzWOhLoDrynlPKwcq3iajQfDLe8DLsXwLq3za5GOIB/L4vD082V56Obml2KuEaWXH5uByRqrQ8BKKXmA4OAuFLnaKCqUkoBvkAGUGTlWsXV6vqssSH12v8Y29p1eUamNYoyrd1/itX7TvFS33BqVvUyuxxxjSwJ9HpAUqnnyUD7i875CFgCHAeqAndprS+5hVEpNRYYCxASEnIt9YqroRTcNgmKz8Hq1yHtgPHcrfyxUVF5nCsq4fVlcYQF+nB/Z9n60JFZMoZeVpfu4kHZPkAMUBdoBXyklLpk7pLWeobWOkprHRUUFHSVpYpr4uYBd8w07ibd9bWxvV3M13B8J5zLM7s6YQP5hcXsOZZFdn5huedmnS3kgTnbOJR6hpcHRODhJvMkHJklPfRkoH6p58EYPfHS7gfe0sYcyESl1GEgHNhqlSrF9VEKur0AATfAkidg0Tjj8y7ucMenxni7cHjFJZpFO4/x3sr9HM8yZjfV9fOiQYAPnu4uuLkofD3d6HRDID2a1uRMQRFjPt/G0Yw83rmjJbeE1zK5BeJ6WRLo24DGSqkw4BgwHBh50TlHgZ7AeqVULaApcMiahQoraDEEIgZCxiFIjYcNk2DxeKjd0lgTRjikc0UlrIw7ydRfDhJ/IpuWwX4827spKTn5JJzMIen0WfLOFVFUoknNKWBRjNEf83J3oYq7K/PGtJdZLU6i3EDXWhcppcYDPwGuwGyt9V6l1Ljzx6cD/wfMUUrt5vwdv1rrNBvWLa6VqxsENTEe9drC9C6w4D4Ys8rYqFo4hKyzhew7kc2a/adYuD2Z9DPnaBDgzZQRrel/Yx1cXMq++K21Jv5EDmv2pXA4LY8net5AgwCfCq5e2IrcKVrZ7V8OXw+HdmOh33/NrkaUY86Gw3y6/jDHMo218F1dFL0iajKiXQhdGwfhepkgF87jSneKyqo5lV3TvtBxPGz6CLwDjamOrvJjYW+01kxafYAPVx2gQ8Ma3NOhARF1qnJjPT8CfGXWkjDIv1wBPV+FnJOw9k1IWA63T5MVG+2I1pq3Vuzjk3WHGNo2mLfvaCk9cVEmmaMkjKmNQ2cZ+5ZmHjX2LN27yOyqKj2tNRsPpjHm8+18su4Q93QI4R0Jc3EF0kMXf2k+GBp0gf/dDd8/DH7BEFzmUJ2woZISzaKYY3yy7hD7U3Ko7u3Oi9HhjOvWECV3+oorkEAXf+cbBMO/hpm3GBdLH1wN1RuYXVWl8duBNN78MZ64E9mE167KO0NbMjCyLl7usra9KJ8EuriUTwCMXACzesFXd8GYn8DLz+yqnMbKvSc5mpFHRJ1qRNSpRnpuAT/Hp7BybwoxSZkEV6/CpOGtuK1l3ctOPxSiLBLoomxBTeDOL2DeHcaY+qCPIbSz2VU5tKLiEt78cR+zNxwu8/iN9fx4eUAz7ukQgqeb9MjF1ZNAF5fXsDvctxQWPQJz+kP7ccb2dl6yxdjVys4v5PGvdrIuIZXRnUJ5tHsjElJyiT+RjbenK7eE16SOXxWzyxQOTm4sEuU7dwZWvQZbZ4CrJzS+1biA2qQPeFY1uzq7d6agiGHTN5GQksPrg1owsr2sNCqundxYJK6Ph49xF2nkCIj9BuIWwb5l4OZVKtyjjfPE35SUaJ75JoZ9J7OZdd9N9AivaXZJwolJoAvL1WtjPPq8CUmbjbnqcYuMTandqkCT3ka4N+4DHt5mV2sXJq0+wE97U/hX/wgJc2FzEuji6rm4QINOxiP6P3B0E+z9HuIWGw93b6PH3nwwNO5dKRf9KinRLN51jEmrDzC0bTBjusjGEcL2ZAxdWE9JMfyx4Xy4L4G8NAgKhxHzoYbzB5rWmg2J6fyw+wSr41M4lVNA6xB/5o/tILNWhNVcaQxdAl3YRnERJKyAxY+Biyvc9SU06Gh2VTZh3KKfzrsr97PzaCY+Hq50axpEz/Ba9L2xNt4e8ouwsB65KCoqnqsbRAwwFvn66k74YiB0fwla3gV+9cyuzmoOpebyr0V72HgwnTp+XrwxuAVD2wZLj1yYQnrowvbOnoZvH4LEn43n9TtA+4eNHZQcRGbeOWZvOEJ1b3c6NQokLNCHT9cfYtLqA3i5ufD0rU0Y0S5EbtEXNic9dGGuKtXhnoWQlmiMr+9eAAvvNzaq7vWaMSRjx9bsS2HCt7s5lVNw4XMebi6cKyqh3421eW1gc2pWrXwXfoX9kUAXFSfwBuj2PHR5Gla8CBsnQ/pBGDIDPH3Nrg4wxsM3H8rgaMYZ0s+cY+/xbH6IPUHTWlWZPfom/Kq4s+lQOjFJmXRrEkSf5rXNLlmIC2TIRZhnywwj2Ou1hdE/Guuym+z9lfuZvCbxwnMfD1fu6xTKk70ay7i4sAsy5CLsU/ux4BNoDL+seg2i3zS1nCmrDzB5TSJ3RgXzZK8m1PD2oIqHhLhwHBLowlwthhg3Jm2eCqFdILxfhZdQXKL55NeDvPdzAkPa1OOtIS1l2VrhkCTQhfl6/xuSthirOo77Dfzr2/wt884VMX9rEhsS09h6JIOc/CJui6zLf4dGSpgLhyWBLszn5glDP4NPusGXQ6H3G3BDT7DRdmu/Hz3Ns9/s4nDaGRoG+jCgZV06NQqgb4vasl+ncGgS6MI+BDSCO+fA0qfgyzsguB30eAka9rBKsJeUaI6kn+Hb35OZtvYgdfyq8NVD7enUKPC6X1sIeyGBLuzHDb3g8d8hZh78+h7MHQwhHaH7BAjrdk3BHpucyTsr9rMrOZOc/CIAhrYN5pXbmlHNy93aLRDCVDJtUdinogLYOdcI9pzjENLJ2C0prKvFL7E45hgvLIzF39udXhG1aBnsR+uQ6jSpJZtyCMcl0xaF43HzhJsehFb3wO9fwG/vw+cDILQr3Py88aeLS5lfWlKief/nBD76JZF2oTWYdk8bAnw9K7gBQlQ86aELx1CYDzvmGMGemwJV60KzQRA5HOq2unBaUXEJLyyM5budxxh+U31eH9QCD7eyg18IR3SlHrr8pAvH4O4FHcbBk7tgyEyo2xq2z4JPe0DCTwAUFpfw1P9i+G7nMZ7u1YT/DLlRwlxUKtJDFw7r19gEwn68m1rnklnXZS4Lkv1ZGZfChL7hjOvWyOzyhLCJ6+6hK6WilVL7lVKJSqkJlzmnu1IqRim1Vym17noKFuJKsvMLeW7BLkZ9dYD7858ho9iT5useYmfcPl69rZmEuai0yr0oqpRyBaYCtwLJwDal1BKtdVypc/yBj4ForfVRpZTshiusYv/JHD5em8iOP04T4ONBUFVP9h7P5lROAeN73MATPRujTkbgOiea9TU/wat1xS8dIIS9sGSWSzsgUWt9CEApNR8YBMSVOmck8J3W+iiA1vqUtQsVlcvB1FzeXr6PlXEp+Hi40r1pTXIKijiWmU89/ypMv6ctkfX9jZODW8GwOXh9cy/M7AUjvzGW6hWikrEk0OsBSaWeJwPtLzqnCeCulFoLVAUmaa2/uPiFlFJjgbEAISEh11KvqAS2HclgzJxtADzZszH3dw7F37ucpXWbRsN9S2H+SJh5C9w5Fxp2q4BqhbAfloyhl3V73sVXUt2AtkB/oA/wslKqySVfpPUMrXWU1joqKCjoqosVzu/nuBTumbmFQF9PfniiK0/f2qT8MP9TSAd4aA1UrQPzhsD2z2xbrBB2xpIeejJQevm7YOB4Geekaa3PAGeUUr8CkUCCVaoUTiv5dB4bEtM4nplPUkYei2KOcWOwP5+NvokaPtew4UX1UBizEhY+AMuegrQEYzVHO9/mTghrsCTQtwGNlVJhwDFgOMaYeWmLgY+UUm6AB8aQzAfWLFQ4F601X209yr+XxXO2sBilIMjXk9si6/Lm4Bvx8byOm5i9/GDE/2DlP2Hzx5CeCMM+Bw9v6zVACDtU7r8arXWRUmo88BPgCszWWu9VSo07f3y61jpeKbUCiAVKgJla6z22LFw4Jq01R9LzeH3pXn7Zn0rXxoG8MqAZDQJ8rHsTkKsb9H0bAhvDj8/D13cZIS+hLpyY3FgkbKagqJgDKbnsP5lDQkoOe49ns+d4Fpl5hXi6ufBS33BGdQy1/YYSu/4H3z9sXCQdMR/cq9j2/YSwIVmcS1SYxFO5/ByXwobENLYdyaCgqAQAD1cXGtfyJbp5bVrU86NbkyDq16ig3nLkXaBLjB2RvroT2j0MNSOM8XYZWxdORAJdWMWJrLO8tzKBb39PRmtoWqsqI9uH0LZBdcJrVyU0wAc3VxPXVWk1wgj1pU/C4V+Nz3n4Gmutd3hUgl04BQl0cc2y8grZdiSD3xLTmL/tKCUl8GCXMB7q2pCa1bzMLu9Sre82VmhM3Q+p8RC3BFb+C+KXwu3TjF2ThHBgMoYuylVSojmakUf8iWwSUnJJOJVDwskcElNz0doYTul7Y22e69204oZRrEFriP0Glj8PReeM1Rw7PQHeNcyuTIjLutIYugS6+JuSEs3ahFPEHc/mUNoZDqedIeFkDmfOFQPGLnAhNbxpXNOXyGB/2oXVILK+P17uDjxkkX3C6Knv+RY8fKD9w9BxvAS7sEsS6MIiGw+m8eaP8ew5lg1A7WpehAZ607RWVZrVrUZEnWo0rlmVKh4OHN5Xcioe1r0NexcZ4+sdHoGOj0KV6mZXJsQFEujisrTWbDmcwSfrDvLL/lTq+VfhuT5N6NO8Nt4elfQSS0ocrHsL4haDp58R7B0egSr+ZlcmhAS6uFRRcQnf/X6M2RsOs+9kDv7e7oy9uSEPdA5z7OETazq5B9b+B/YtM+4+7TgeOj0u89iFqWQeuvibzYfSeXXxXvan5BBeuypv33Ejg1rVkyC/WO0WMPxLOBELa9+CX96AU3Ew9DPjYoIQdkYCvZIoKCpmy6EMvtmexLLYE+fXFG9Dn+a1URJOV1anJYz4Cta/D6snQtjNEPWA2VUJcQkJdCeWmlPA2v2nWLPvFL8mpHLmXDFV3F15smdjxnVr5LwXN22l81Nw5DdYPgGC2xk9eCHsiAS6kzmYmsuKPSdZufcku5KzAKhVzZNBrevRK6ImnRoFytDKtXJxgcGfwPQusGA0PLTaGFsXwk5IoDu4khLN7mNZ/ByXwsq4kySk5AIQWd+fZ29tQo/wmjSvW02GVazFNwjumAlfDIT3wqFJNDQfDE37GSs8CmEi+Ql0UIXFJczd9Aczfj3Eyex8XBTcFFqD125rRu/mtanrLzMxbCasKzywEmK+hPglsPc7aHU33P6x2ZWJSk4C3QGtP5DKxKVxJJ7KpVOjAF6IbkqPpjWpfi07/IhrU/8m49HvXVj9GmycYqwT06SP2ZWJSkwC3YEcSMnhreX7WL3vFA0CvJk5KoqeETVlOMVMrm5wy8twYBUsfQoe2yzj6sI0EugO4HDaGT5df4j5W4/i4+HGi9HhPNAlFE83ubhpF9w84fapMLMX/PRPGPSR2RWJSkoC3U4ln85j6a4TLN11nLgT2bi5KEZ1DOWJno2vbfNkYVv12horNW74EEI6QORIY1aMEBVIAt2OnCkoYsmu43z/+zG2HskAoFV9f/7VP4IBLetS288O1xgXf+n+EhxeB4sfg83Tjc0zwvvLXaWiwkig2wGtNT/sPsH/LYsjJbuARkE+PN+nKQMj6zrW+uKVnbsXjFkFexYaqzb+724IH2AsFeAmv1UJ25NAN9nhtDO8vGgPvyWm0bxuNT4a2YaoBtXlQqejcnWDyOHQYihs+ghWvQoL74dhc8DV3ezqhJOTQDdJcYlm9m+HeXflfjxcXZg4sDn3dGiAq4sEuVNwdYMuT4GbF6x4ERY+AENnS6gLm5JAN8GeY1m8vHgPO49m0iuiFm8MbkEte9yDU1y/DuOMzal/egm+GQVDPgVPX7OrEk5KAr2CnCsqYfmeE3yx6Q92/HEaf293Jg1vxcDIujK84uw6Pmr0zJe/ALOjYeR88As2uyrhhCTQbaywuIQF25P5aM0BjmflExrgzb/6RzCsbX38vOXX70qj3UNQPRQW3A+f3gIj5kO9NmZXJZyMBLqNlJRoFu86xgc/H+BoRh6t6vvzxpAb6dY4CBcZJ6+cGt8KD/4MX94JX90F436DqrXMrko4EQl0Gyi92XLzutWYPTqKHk3lFn0B1IyAu7+BGT3gu4fg3u/BRe74FdYhgW5FZwqKePHb2As7An14lzFGLj1y8Tc1I6DfO7Dkcfjtfbj5ebMrEk5CAt1Kjqbn8dAX2zlwKodnbm3C2JsbykYS4vJa3wuHf4Vf3oSQThDa2eyKhBOQxSasYF1CKgOn/sbJ7Hw+f6AdT/RsLGEurkwpGPCBcaF02VNQUmJ2RcIJSKBfh6SMPB6Zt4P7Zm+lZlVPlozvTNfGQWaXJRyFZ1Xo/g9IS4ADP5ldjXACFgW6UipaKbVfKZWolJpwhfNuUkoVK6WGWq9E+1NUXMKHqxLo+f461u5P5dlbm7BkfBcaBPiYXZpwNM1vB7/6sGGy2ZUIJ1DuGLpSyhWYCtwKJAPblFJLtNZxZZz3NuDUXY1TOfk8/tVOthzO4LbIuvyjXzh1/GS7N3GNXN2hw6PGnaTJ2yE4yuyKhAOzpIfeDkjUWh/SWp8D5gODyjjvceBb4JQV67MrGw+m0X/yb+xKzuS9YZFMGdFawlxcvzajjF2ONkwyuxLh4CyZ5VIPSCr1PBloX/oEpVQ9YDBwC3DT5V5IKTUWGAsQEhJytbWaZv/JHP77035WxafQMNCHuWPaEV67mtllCWfh6QtRY+C3DyD9IAQ0Mrsi4aAs6aGXNYlaX/T8Q+BFrXXxlV5Iaz1Dax2ltY4KCrL/i4e5BUW8sHAX0ZN+ZcuhdJ7r3YRlT3SRMBfW1/5hY/hl3TugL/7nJYRlLOmhJwP1Sz0PBo5fdE4UMP/8nZCBQD+lVJHWepE1ijRDQkoOj8zbweG0MzzUtSGPdm+Ev7dsUiBspGpt6PCIMeyii2HgR8aGGUJcBUsCfRvQWCkVBhwDhgMjS5+gtQ7782Ol1BxgmaOGudaa73ce45/f78HH05V5D7anU6NAs8sSlUGvicZY+urX4fQfMPxL8K1pdlXCgZQb6FrrIqXUeIzZK67AbK31XqXUuPPHp9u4xgpzPPMsLy/aw+p9p2gXWoMpI1vLOuWi4igFXZ+FgMbw3ViY3hUGToYmfcyuTDgIpU0ar4uKitLbt2835b1L01qTlHGW5XtOMHn1AUo0PNu7CaM7heLmKvddCZOciDVCPTUeWt0D0W8avXdR6Smldmity5zfWmnXctl/MofPNx1h3f5UjmWeBeDmJkG8cXsL2ZhZmK9OS3h4Hax9CzZ8CElbYOwvxt2lQlxGpQv0jQfT+GTdIdYlpFLF3ZWujQMZe3NDOt8QQKMgX1niVtgPN0/o9SqEdYV5d8DSp+COmcbQjBBlqDSBHn8imzd/jGf9gTQCfT15rncT7m7fgOo+MnNF2LlGtxhrvvzyb2jYzbgRSYgyOH2g550r4vWlcfxvexLVvNz5V/8I7unQQFZDFI6l6zNwZD38+ALUi4JazcyuSNghp77ql5VXyD0zt/DN9iQe6BzGuue782BXWadcOCAXVxjyqTGGvmA0nDtjdkXCDjltoJ/KzueuGZvYcyybj+9uw8sDmsmNQcKxVa0Fd3xqLLf74wtmVyPskFMG+omsswz7ZBNHM/KYPfomolvUMbskIayjYXe4+TmImQe75ptdjbAzThfoWXmF3Dd7K+m55/jywfZ0aSx3eQon022CsW3dsmcg7YDZ1Qg74lSBnl9YzJjPt3EkLY8Zo9rSOqS62SUJYX2ubjB0ljGt8ZtRkJdhdkXCTjhNoBcVlzD+q53sOHqaD+5qJeuvCOdWrS4MnQ3piTB3MJzNNLsiYQecJtD/XK984sDm9G8pY+aiEmjUA+6aByl7JdQF4CSBviz2OJ/8eoh7OzRgVMdQs8sRouI06QN3zYWTu41Qz001uyJhIocP9P0nc3hhYSxtG1Tn5QFys4WohJr2NUL9VDx8egukxJX/NcIpOXSgZ+cXMm7eDnw83fj47jZ4uDl0c4S4dk37wv0/QvE5mNUbElaaXZEwgUMn4OtL4ziakcfHd7eRdcuFqNcGHloDNcJg/gg4HmN2RaKCOWygr45PYeGOZB7p1oibQmuYXY4Q9sGvHty3BLwDYPFjUHTO7IpEBXLIQM/MO8dL3+0mvHZVHu95g9nlCGFfqlSHAR9Ayh747X2zqxEVyCEDfeLSODLOnOPdYZF4uslCW0JcIrw/tBgKv/4XTu4xuxpRQRwu0FfuPcn3O48x/pYbaFFPtuQS4rL6vmP01hc/CsVFZlcjKoDDBXqLen6M7hTKYz1kqEWIK/IJMEL9xC7Y9bXZ1YgK4HCBXte/Cq8NbI67bOAsRPmaD4bgm+CXN6HwrNnVCBuTVBTCmSkFvSZCznHYMt3saoSNSaAL4exCO0OTaFj/gazM6OQk0IWoDHq+CgXZMo3RyUmgC1EZ1GoGrUbClhmQtM3saoSNSKALUVn0es24k3TeEEjebnY1wgYk0IWoLHxrwn3LjGUB5g6GYzvMrkhYmQS6EJWJXz0YvQy8a8AXgyEr2eyKhBVJoAtR2fgFw73fQ2EerJeLpM5EAl2IyqhGQ2h9D+ycC1nHzK5GWIlFga6UilZK7VdKJSqlJpRx/G6lVOz5x0alVKT1SxVCWFXXZ0Br2PCh2ZUIKyk30JVSrsBUoC/QDBihlLp4r7fDQDetdUvg/4AZ1i5UCGFl/iHGVMYdn0P2CbOrEVZgSQ+9HZCotT6ktT4HzAcGlT5Ba71Ra336/NPNQLB1yxRC2ETXZ0AXSy/dSVgS6PWApFLPk89/7nLGAMvLOqCUGquU2q6U2p6aKruTC2G66qEQORx2zIGck2ZXI66TJYGuyvicLvNEpXpgBPqLZR3XWs/QWkdpraOCgoIsr1IIYTtdnzU2l5bFuxyeJYGeDNQv9TwYOH7xSUqplsBMYJDWOt065QkhbK5GQ4i4DbbNhoIcs6sR18GSQN8GNFZKhSmlPIDhwJLSJyilQoDvgHu11gnWL1MIYVOdnoSCLPj9C7MrEdeh3EDXWhcB44GfgHjgG631XqXUOKXUuPOnvQIEAB8rpWKUUrJQhBCOJLgtNOgMmz6G4kKzqxHXSGld5nC4zUVFRent2yX3hbAb+1fA13fBkE+h5Z1mVyMuQym1Q2sdVdYxuVNUCGFo3BuCwmHDJOOGI+FwJNCFEAYXF+j0OKTsgc3TzK5GXAMJdCHEX1oOh/AB8NNLsPVTs6sRV0kCXQjxF1c3GPoZNO0HPz4H22aZXZG4ChLoQoi/c/OAYZ8bG0v/8AwkrDS7ImEhCXQhxKXcPODOLyCwKSx/Hgrzza5IWMDN7AJKKywsJDk5mfx8+eG5Fl5eXgQHB+Pu7m52KcIZuHlC37dh7u2wcQp0e97sikQ57CrQk5OTqVq1KqGhoShV1hIy4nK01qSnp5OcnExYWJjZ5Qhn0agHNBsE69+DyLuMJXeF3bKrIZf8/HwCAgIkzK+BUoqAgAD57UZYX+83jD9/+oe5dYhy2VWgAxLm10G+d8Im/OvDzc9C/FLYvdDsasQV2F2gCyHsUKcnIKQjfDcW4habXY24DAn0i7i6utKqVStatGjBsGHDyMvLu+7XfOWVV1i1atVlj0+fPp0vvpBV7oQdc/OEuxdAcBQsfMDorQu7Y1eLc8XHxxMREWFKPX/y9fUlNzcXgLvvvpu2bdvyzDPPXDheXFyMq6urWeWVyx6+h8KJ5WfDvDvg+O9GwDe6xeyKKp0rLc5lV7NcSpu4dC9xx7Ot+prN6lbj1duaW3x+165diY2NZe3atUycOJE6deoQExPD7t27mTBhAmvXrqWgoIDHHnuMhx9+GIB33nmHuXPn4uLiQt++fXnrrbcYPXo0AwYMYOjQoUyYMIElS5bg5uZG7969effdd3nttdfw9fXlueeeIyYmhnHjxpGXl0ejRo2YPXs21atXp3v37rRv355ffvmFzMxMZs2aRdeuXa36/RGiXF7V4J6FMDsavn0Ixv0G1eqYXZU4z24D3WxFRUUsX76c6OhoALZu3cqePXsICwtjxowZ+Pn5sW3bNgoKCujcuTO9e/dm3759LFq0iC1btuDt7U1GRsbfXjMjI4Pvv/+effv2oZQiMzPzkvcdNWoUU6ZMoVu3brzyyitMnDiRDz/88EJNW7du5ccff2TixIlXHMYRwma8/GDYHJjRHb57CEYtBhf7/a21MrHbQL+anrQ1nT17llatWgFGD33MmDFs3LiRdu3aXZjfvXLlSmJjY1m40Ljin5WVxYEDB1i1ahX3338/3t7eANSoUeNvr12tWjW8vLx48MEH6d+/PwMGDPjb8aysLDIzM+nWrRsA9913H8OGDbtwfMiQIQC0bduWI0eOWL3tQlgsqCn0fw8WPQLr3oEeL5ldkcCOA90sVapUISYm5pLP+/j4XPhYa82UKVPo06fP385ZsWLFFacOurm5sXXrVlavXs38+fP56KOPWLNmjcW1eXp6AsaF26KiIou/TgibaDUSDv8K696GKv4Q9YBx8VSYRma5XIM+ffowbdo0CguNrboSEhI4c+YMvXv3Zvbs2Rdmxlw85JKbm0tWVhb9+vXjww8/vOQ/Dj8/P6pXr8769esBmDt37oXeuhB2qd+7ENoFVkyAya1h20woOmd2VZWW9NCvwYMPPsiRI0do06YNWmuCgoJYtGgR0dHRxMTEEBUVhYeHB/369ePNN9+88HU5OTkMGjSI/Px8tNZ88MEHl7z2559/fuGiaMOGDfnss88qsmlCXB1PX7hvKRz6BX75D/zwLOz+Fu6aBz4BZldX6ci0RScj30NhGq1h9wJY8jj41oKR30DNcLOrcjqyp6gQwvaUMjaXHv0DFJ6FWbfCoXVmV1WpSKALIawrOAoeWgN+wfDVXRLqFUgCXQhhff71jbH1GmFGqB9eb3ZFlYIEuhDCNnwCYdQSqN4AvroTDsiNcLYmgS6EsB3fIKOnXj0UvrwDlj0NBblmV+W0JNCFELblW9MYU+84HrZ/BtM6QuwCCXYbkEC/SOnlc2+77bYy11u5HqGhoaSlpQHGyo5CVAruVaDPG/DACnD1gO8ehP82gv/dC8d2mF2d05BAv8ift/7v2bOHGjVqMHXqVLNLEsJ5hHSAx7bB/cuhzSj4YyN8MRhS95tdmVOw3ztFl0+Ak7ut+5q1b4S+b1l8eseOHYmNjQXg4MGDPPbYY6SmpuLt7c2nn35KeHg4KSkpjBs3jkOHDgEwbdo0OnXqxO23305SUhL5+fk8+eSTjB071rptEcJRubhAg07Go9MT8Okt8OUwY1jGJ9Ds6hya9NAvo7i4mNWrVzNw4EAAxo4dy5QpU9ixYwfvvvsujz76KABPPPEE3bp1Y9euXfz+++80b26sEjl79mx27NjB9u3bmTx5Munp6aa1RQi75V8fRsyH3BSYPxIKZZPz62G/PfSr6Elb05/L5x45coS2bdty6623kpuby8aNG/+2lG1BQQEAa9asubB9nKurK35+fgBMnjyZ77//HoCkpCQOHDhAQICsbSHEJYLbwuBPYMF98HEHqNsKgiKgfjsI7Qqu9htT9sai75RSKhqYBLgCM7XWb110XJ0/3g/IA0ZrrX+3cq0V4s8x9KysLAYMGMDUqVMZPXo0/v7+ZS6rW5a1a9eyatUqNm3ahLe3N927dyc/X3oeQlxW89uheCbs+RaO74S9iwAN3gEQMdBYqrd+O5OLtH/lDrkopVyBqUBfoBkwQinV7KLT+gKNzz/GAtOsXGeF8/PzY/Lkybz77rtUqVKFsLAwFixYABjroe/atQuAnj17Mm2a0dzi4mKys7PJysqievXqeHt7s2/fPjZv3mxaO4RwGC2Hwcj58OQueCnZWLGxYXeI/cZYF2buYEjaZnaVds2SHno7IFFrfQhAKTUfGATElTpnEPCFNpZu3KyU8ldK1dFan7B6xRWodevWREZGMn/+fL788kseeeQR/v3vf1NYWMjw4cOJjIxk0qRJjB07llmzZuHq6sq0adOIjo5m+vTptGzZkqZNm9KhQwezmyKEY/H0hYjbjMe5M7BtFmyYBLN6QY1G4OpudoXXp/W90Gm81V/WkkCvBySVep4MtLfgnHrA3wJdKTUWowdPSEjI1dZaIXJz/36zw9KlSy98vGLFikvOr1WrFosXL77k88uXLy/z9UtvHXfxewkhyuDhA52fMHZE2j4bjm0v/2vsnW9Nm7ysJYFe1p5qFy+ibsk5aK1nADPAWA/dgvcWQgiDp68R7OKyLJm2mAzUL/U8GDh+DecIIYSwIUsCfRvQWCkVppTyAIYDSy46ZwkwShk6AFnXOn5u1g5KzkC+d0JUbuUOuWiti5RS44GfMKYtztZa71VKjTt/fDrwI8aUxUSMaYv3X0sxXl5epKenExAQgDETUlhKa016ejpeXl5mlyKEMIld7SlaWFhIcnKyzNm+Rl5eXgQHB+Pu7uAzAIQQl3WlPUXt6hYsd3d3wsLCzC5DCCEckqzlIoQQTkICXQghnIQEuhBCOAnTLooqpVKBP67xywOBNCuW4wikzZWDtLlyuJ42N9BaB5V1wLRAvx5Kqe2Xu8rrrKTNlYO0uXKwVZtlyEUIIZyEBLoQQjgJRw30GWYXYAJpc+Ugba4cbNJmhxxDF0IIcSlH7aELIYS4iAS6EEI4CbsOdKVUtFJqv1IqUSk1oYzjSik1+fzxWKVUGzPqtCYL2nz3+bbGKqU2KqUizajTmsprc6nzblJKFSulhlZkfbZgSZuVUt2VUjFKqb1KqXUVXaO1WfCz7aeUWqqU2nW+zde0aqu9UErNVkqdUkrtucxx6+eX1touHxhL9R4EGgIewC6g2UXn9AOWY+yY1AHYYnbdFdDmTkD18x/3rQxtLnXeGoylmoeaXXcF/D37Y+zbG3L+eU2z666ANv8DePv8x0FABuBhdu3X0eabgTbAnssct3p+2XMP/cLm1Frrc8Cfm1OXdmFzaq31ZsBfKVWnogu1onLbrLXeqLU+ff7pZozdoRyZJX/PAI8D3wKnKrI4G7GkzSOB77TWRwG01o7ebkvarIGqytgMwRcj0Isqtkzr0Vr/itGGy7F6ftlzoF9u4+mrPceRXG17xmD8D+/Iym2zUqoeMBiYXoF12ZIlf89NgOpKqbVKqR1KqVEVVp1tWNLmj4AIjO0rdwNPaq1LKqY8U1g9v+xqPfSLWG1zagdicXuUUj0wAr2LTSuyPUva/CHwota62El2srKkzW5AW6AnUAXYpJTarLVOsHVxNmJJm/sAMcAtQCPgZ6XUeq11to1rM4vV88ueA70ybk5tUXuUUi2BmUBfrXV6BdVmK5a0OQqYfz7MA4F+SqkirfWiCqnQ+iz92U7TWp8BziilfgUiAUcNdEvafD/wljYGmBOVUoeBcGBrxZRY4ayeX/Y85FKhm1PbiXLbrJQKAb4D7nXg3lpp5bZZax2mtQ7VWocCC4FHHTjMwbKf7cVAV6WUm1LKG2gPxFdwndZkSZuPYvxGglKqFtAUOFShVVYsq+eX3fbQdQVuTm0vLGzzK0AA8PH5HmuRduCV6ixss1OxpM1a63il1AogFigBZmqty5z+5ggs/Hv+P2COUmo3xnDEi1prh11WVyn1NdAdCFRKJQOvAu5gu/ySW/+FEMJJ2POQixBCiKsggS6EEE5CAl0IIZyEBLoQQjgJCXQhhHASEuhCCOEkJNCFEMJJ/D8T1kaXa9W8KgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(df_scores_1['threshold'], df_scores_1['precision'], label='Precision')\n",
    "plt.plot(df_scores_1['threshold'], df_scores_1['recall'], label='Recall')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f56a9a14c10>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAuEElEQVR4nO3deXiU1d3/8fc3+74TCGsSNtkXA6iIYlFAXFDUKvrUpfWxtLj/+lTt1bq01se21lqXSqnSujyWWlesCm4girIERXaQTQggELKQfT2/P2bEgAECyWQmmc/ruubKzH3fmfkwCfnOOee+zzHnHCIiErxC/B1ARET8S4VARCTIqRCIiAQ5FQIRkSCnQiAiEuTC/B3geKWlpbnMzEx/xxARaVOWL1+e75zr0Ni+NlcIMjMzyc3N9XcMEZE2xcy+OtI+dQ2JiAQ5FQIRkSCnQiAiEuTa3BiBiBxdTU0NeXl5VFZW+juK+EFUVBRdu3YlPDy8yd+jQiDSzuTl5REfH09mZiZm5u840oqcc+zfv5+8vDyysrKa/H0+6xoys1lmttfMVh9hv5nZo2a2ycxWmtlwX2URCSaVlZWkpqaqCAQhMyM1NfW4W4O+HCP4BzDxKPvPBXp7bzcAT/owi0hQUREIXifys/dZ15BzbqGZZR7lkMnAs84zD/ZiM0syswzn3G5f5Nm4p4T/fLGLuKgw4iLDiY0MJd57Py4yzHOL8nyNCNMYuogED3+OEXQBdjR4nOfd9p1CYGY34Gk10L179xN6sY17Snj0g01NOjYyLISE6HASosK8X8O/8zg5JpykmHASoyNIjg0nKTqCpJhwosJDTyifSHsSGhrKoEGDqK2tpV+/fjzzzDPExMQ06znvvvtuzjjjDM4+++xG98+YMYOYmBiuvvrqZr1OVlYWc+fOpW/fvge33XrrrXTu3Jmf//znx/z+zMxM4uPjMTOSk5N59tln6dGjR7MyNTR27FgeeughcnJyDl5gm5aW1qzn9GchaKz90ugqOc65mcBMgJycnBNaSef8wZ2ZNDCD8po6SitrKa3y3g65X0NpVS0lVbUcqKjlQGUNBypqKKqoYUdBOQcqayiuqKGm7sgR4iLD6JUeR9+O8fTpFO/9GkeHuEg11yVoREdHs2LFCgCuuuoqZsyYwe23335wf11dHaGhx/eh6de//vVR90+bNu24czbmiiuuYPbs2dxzzz0A1NfX89JLL7Fo0aImP8f8+fNJS0vjnnvu4f777+dvf/tbi2TzFX8WgjygW4PHXYFdvnzBkBA72A10opxzVNTUUVReQ2F5NcXlNRSW11BUUU1ReQ17DlSycU8J767bw79yv23wpMVFcHqvNMb2TeeMPh1IiY1oiX+SSMAbM2YMK1euZMGCBdx3331kZGSwYsUKVq1axZ133smCBQuoqqpi+vTp/PjHPwbg97//Pc899xwhISGce+65PPjgg1x77bWcf/75XHrppdx5553MmTOHsLAwxo8fz0MPPcS9995LXFwcP/vZz1ixYgXTpk2jvLycnj17MmvWLJKTkxk7diyjRo1i/vz5FBUV8fTTTzNmzJhD8k6dOpXLL7/8YCFYuHAhmZmZ9OjRgzVr1nDddddRXV1NfX09L7/8Mr179z7iv/3UU0/l0UcfBWDfvn1MmzaN7du3A/DII48wevRoSktLuemmm8jNzcXMuOeee7jkkkv4yU9+wrJly6ioqODSSy/lvvvu88WPB/BvIZgD3Ghms4FRQLGvxgdakpkRExFGTEQYnZOij3psfmkVG/eUsPHrElbsKGLhl/m8tmIXZjCiRwoXDO3MeYMyVBTEZ+57Yw1rdx1o0efs3zmBey4Y0KRja2trefvtt5k40XPeyNKlS1m9ejVZWVnMnDmTxMREli1bRlVVFaNHj2b8+PGsX7+e1157jSVLlhATE0NBQcEhz1lQUMCrr77K+vXrMTOKioq+87pXX301jz32GGeeeSZ333039913H4888sjBTEuXLuWtt97ivvvu47333jvkewcPHkxISAhffPEFQ4YMYfbs2UydOhXwdD/dcsstXHXVVVRXV1NXV3fUf//cuXO56KKLALjlllu47bbbOP3009m+fTsTJkxg3bp1/OY3vyExMZFVq1YBUFhYCMBvf/tbUlJSqKurY9y4caxcuZLBgwc36X0/Xj4rBGb2T2AskGZmecA9QDiAc24G8BYwCdgElAPX+SqLv6TFRZIWF8lpPT39d/X1jpU7i5m/fi9vrtrNr15bzX1z1nDnuSdx/ZhsP6cVaTkVFRUMHToU8LQIfvSjH/HJJ58wcuTIg+e3v/POO6xcuZKXXnoJgOLiYr788kvee+89rrvuuoNjCikpKYc8d0JCAlFRUVx//fWcd955nH/++YfsLy4upqioiDPPPBOAa665hssuu+zg/ilTpgBw8skns23btkbzT506ldmzZzNgwABef/31g91Sp556Kr/97W/Jy8tjypQpR2wNnHXWWezZs4f09HTuv/9+AN577z3Wrl178JgDBw5QUlLCe++9x+zZsw9uT05OBuDFF19k5syZ1NbWsnv3btauXdv2CoFzbuox9jtguq9ePxCFhBhDuyUxtFsSt57dm3W7S/j9vPU8+PZ6RvdKo19Ggr8jSjvT1E/uLa3hGEFDsbGxB+8753jssceYMGHCIcfMnTv3qONpYWFhLF26lPfff5/Zs2fz+OOP88EHHzQ5W2RkJOAZ0K6trW30mKlTpzJ+/HjOPPNMBg8eTHp6OgBXXnklo0aN4s0332TChAk89dRTfO973/vO98+fP5/Y2FiuvfZa7r77bh5++GHq6+v59NNPiY4+tCfBOfedf+/WrVt56KGHWLZsGcnJyVx77bU+vVJc50n6iZnRv3MCf/r+UJJiwvmfl75g+VeF7D1QSX39CY2Hi7QpEyZM4Mknn6SmpgaAjRs3UlZWxvjx45k1axbl5eUA3+kaKi0tpbi4mEmTJvHII498p+AkJiaSnJzMRx99BMBzzz13sHXQVD179iQ1NZU777zzYLcQwJYtW8jOzubmm2/mwgsvZOXKlUd8jujoaB555BGeffZZCgoKGD9+PI8//vjB/d/kPnx7YWEhBw4cIDY2lsTERPbs2cPbb799XPmPl6aY8LPk2Ajuv2gQP/m/5Vzy5CcARISF0DUpmn4ZCQzumsjgrkkM6prYrEFukUBz/fXXs23bNoYPH45zjg4dOvDaa68xceJEVqxYQU5ODhEREUyaNIkHHnjg4PeVlJQwefJkKisrcc7xpz/96TvP/cwzzxwcLM7Ozubvf//7ceebOnUqd911FxdffPHBbf/61794/vnnCQ8Pp1OnTtx9991HfY6MjAymTp3KE088waOPPsr06dMZPHgwtbW1nHHGGcyYMYNf/vKXTJ8+nYEDBxIaGso999zDlClTGDZsGAMGDCA7O5vRo0cfd/7jYZ4emrYjJyfHtceFabbvL2fzvlLyCsvJK6rgq/xyVu8qJq+wAgAz6J0exyXDuzJ1VHcSopo+oZQEl3Xr1tGvXz9/xxA/aux3wMyWO+dyGjteHzEDRPfUGLqnfveCm/2lVazcWczKHcUs2pzP/769nkff/5LLR3TnvMEZZCRG0SE+kvBQ9fKJyIlRIQhwqXGRnNU3nbP6pnPL2b1ZvbOYpz7awrOfbmPWoq2Ap7WQGhtJx4RIYht0H4WaMXFgJy4f0U1XPIvIEakQtDEDuyTyyBXDuGtSP9bsKmbPgSq+Lq5kb0klXxdXUlHz7XnNheXV3DNnDU/M38QNZ2Rz1ageREeoIASDxs5EkeBwIt39KgRtVMeEKDomRB31GOcci7cU8NgHX3L/m+t4csFmrh+TzdWn9jik5SDtS1RUFPv379dU1EHom/UIoqKO/rfhcBosDhK52wp49INNLNy4j17pcfz92hF0S2neJGASmLRCWXA70gplRxssViEIMh9/mc/0Fz4jLMT42zU5DO+e7O9IItIKjlYIdKpJkDm9dxqv/PQ04qLCmDpzMW+uDPjpnUTEx1QIglDPDnG8+tPRDOqSyPQXPuPqWUv5ZHP+CQ0yiUjbp0IQpFJiI3j++lH8z4S+rN11gCv/toSLnljE0q0Fx/5mEWlXVAiCWFR4KNPP6sXHd5zFAxcPYn9ZNVfM/JS/LNik+Y5EgojOIRSiwkO5clR3LhzamTtfXsnv527g0837Oad/RzonRtMl2XPTtBYi7ZMKgRwUFxnGY1OHMSorhQffXs9HX+Yfsj8rLZbTeqYyulcap2ankqwFdUTaBZ0+Ko2qr3fkl1Wxq6iSnYUVbC8oJ3dbAUu2FlBaVUuIwQMXD+KKkd39HVVEmkCTzslxCwkx0uOjSI+PYmi3JO/WntTW1fNFXjEPv7uBu19fw4DOiQzqmujPqCLSTBosluMSFhrCyT2SeXzqcNLiIvjpC8sprqjxdywRaQYVAjkhybERPHblcHYXVfL/XlxBnc4yEmmzVAjkhJ3cI5lfnd+f99bt5YG31vk7joicII0RSLNcc1omW/PLePrjrWQkRvHD0VmEhGjGS5G2RC0CabZfnd+fs/ulc/+b6/jeHxcw6+OtlFRq3ECkrVAhkGYLDTFm/NfJPDZ1GCmxEfz6P2s55YH3+dey7f6OJiJNoK4haRFhoSFcMKQzFwzpzBc7ivj9vPXc8fIqSipruX5Mtr/jichRqEUgLW5ItyT+fu1Izh3YifvfXMej73+pmU1FApgKgfhERFgIj00dxpRhXXj43Y38bu4GFQORAKWuIfGZsNAQHrpsCNERocz4cDPbC8r43SWDidfkdSIBRS0C8amQEOP+iwbyi0knMW/NHiY/voiv9pf5O5aINKBCID5nZtxwRk9euH4UheXVXP7XxWzNVzEQCRQqBNJqRmWn8sJ/n0J1XT2X//VT8grL/R1JRFAhkFbWLyOB2TecwoHKGv4wb4O/44gIPi4EZjbRzDaY2SYzu7OR/Ylm9oaZfWFma8zsOl/mkcDQp2M8143OYs4Xu1i3+4C/44gEPZ8VAjMLBZ4AzgX6A1PNrP9hh00H1jrnhgBjgT+amZa9CgLTzuhJfGQYf3xHrQIRf/Nli2AksMk5t8U5Vw3MBiYfdowD4s3MgDigAKj1YSYJEIkx4fz4zJ68t24v89fv9XcckaDmy0LQBdjR4HGed1tDjwP9gF3AKuAW51z94U9kZjeYWa6Z5e7bt89XeaWV/ej0LE7qFM9tL65gZ1GFv+OIBC1fFoLG5iI+/NLSCcAKoDMwFHjczBK+803OzXTO5Tjncjp06NDSOcVPosJDefK/Tqa2zjHtueW8mLuDxVv2s7u4gnotdCPSanx5ZXEe0K3B4654Pvk3dB3woPPMPbDJzLYCJwFLfZhLAkhWWiwPXTaEW2Z/zs9fWnlwe4/UGF75yWmkxkX6MZ1IcPBlIVgG9DazLGAncAVw5WHHbAfGAR+ZWUegL7DFh5kkAE0c2InV901gV1EF2wvK2by3lN++tY6756zhiSuH+zueSLvns0LgnKs1sxuBeUAoMMs5t8bMpnn3zwB+A/zDzFbh6Uq6wzmX76tMErjCQ0PokRpLj9RYxvTuQFl1HX+Yt4FzB+7i/MGd/R1PpF2ztjYjZE5OjsvNzfV3DPGx2rp6LpnxKdv3l/HObWfSIV5dRCLNYWbLnXM5je3TlcUSkMJCQ/jjZYMpq67j+mdzWbtLF56J+IoKgQSsXunxPPz9IWzfX8b5j33EXa+sIr+0yt+xRNodFQIJaOcP7syCn53FNadl8u/cHZz1hwVs+LrE37FE2hUVAgl4iTHh3HPBAObeOoaKmjpeX7HT35FE2hUVAmkzeqXHM6RbEos27/d3FJF2RYVA2pTRPVNZlVdEcUWNv6OItBsqBNKmjOnTgXoHc9Q9JNJiVAikTcnpkcwp2Sn88d2NFJVX+zuOSLugQiBtiplx74UDKKmsZdrzyymt0qzlIs2lQiBtzkmdEvjjZUNYtq2Q8x/9iE825dPWrpAXCSS+nHROxGcuGtaFjglR3PHySq58agmdE6MYe1I6Z/VNZ0zvNKLCQ/0dUaTN0FxD0qaVV9cyZ8Uu5m/Yy8df5lNWXUfnxCh+eX5/zh3YCc/idyJytLmGVAik3aiurWfR5nx+P3cD63Yf4OHvD2HK8K7+jiUSEDTpnASFiLAQzuqbzhs3juakTvHMXLhFYwciTaBCIO1OWGgIPxydxfqvS5i/Ya+/44gEPBUCaZcuHNqZHqkxTHv+M55b/BV1WgNZ5IhUCKRdigoP5dWfjmZUVgq/em01Fz7+MYu3aI4ikcaoEEi7lRIbwbM/HMmjU4dRWFbNFTMX85Pnl7N9f7m/o4kEFBUCadfMjAuHdOaDn43l9nP6sGDDPs7504dszS/zdzSRgKFCIEEhKjyUm8f15tXpp1FVW8+nmspa5CAVAgkqfTvGExcZxrrdWgNZ5BsqBBJUzIyTOsWz/msVApFvqBBI0OmXkcAXecX88rVVzF29W9NZS9DTpHMSdK46pTt5heW88tlOnl+8HTMY2DmR03qlMr5/J07ukezviCKtSnMNSdCqqavnix1FfLwpn0827efzHYXU1DlOyU7hlnF9OLVnqr8jirQYTTon0gSlVbX8a9kOZny4mX0lVZw3OIMnrhzu71giLUKTzok0QVxkGD86PYuPfn4W6fGR7CjQhWcSHDRGIHKY6rp68kur2FtSxVkPLSCnRzIjMlMYP6AjSTER/o4n0uJUCEQOkxAVzps3j+GjL/exdGsh767bw7+X57F4Sxcevnyov+OJtDgVApFG9MtIoF9GAjecAfmlVeTc/x490+P8HUvEJ3w6RmBmE81sg5ltMrM7j3DMWDNbYWZrzOxDX+YRORFLthQA6Cwiabd81iIws1DgCeAcIA9YZmZznHNrGxyTBPwFmOic225m6b7KI3KiPt2Sj5mny0ikPfJli2AksMk5t8U5Vw3MBiYfdsyVwCvOue0AzjktJyUBJzM1lhAzzn74Q37w9BLmrv6a2rp6f8cSaTG+HCPoAuxo8DgPGHXYMX2AcDNbAMQDf3bOPXv4E5nZDcANAN27d/dJWJEjuX5MNhcM6cy/lu3gn0u3M+355XRMiOSKEd25YmQ3MhKj/R1RpFl8dkGZmV0GTHDOXe99/ANgpHPupgbHPA7kAOOAaOBT4Dzn3MYjPa8uKBN/qq2rZ/6GfTy/+CsWfrmPUDMuGtaFn47tSXYHDSZL4DraBWW+bBHkAd0aPO4K7GrkmHznXBlQZmYLgSHAEQuBiD+FhYZwTv+OnNO/I9v3lzNr0Vb+uXQ7L3+Wx6RBGUwf24v+nRP8HVPkuPhyjGAZ0NvMsswsArgCmHPYMa8DY8wszMxi8HQdrfNhJpEW0z01hnsvHMCiO7/HtDN78uGGfUx69COuf2YZ+0qq/B1PpMl8Vgicc7XAjcA8PH/cX3TOrTGzaWY2zXvMOmAusBJYCjzlnFvtq0wivpAWF8kdE09i0R3f4/Zz+rBwYz6Pf/Clv2OJNJkmnRNpYbe/uIJ5q7/m01+M0ymnEjA06ZxIK7rutCzKqut44M11vLt2D2t2FVNYVk1b+9AlwUNTTIi0sEFdEzm7X0dmL9vB7GXfnkGdHh/Jsz8ayUmdNJgsgUVdQyI+UF/vyC+rYldRJbuLKthZVMHMhVuIjghlzo2nkxitLiNpXSd8+qiZlQCNVQoDnHNOH21EGhESYqTHR5EeH8XQbkkADOuexBUzF3PBYx8zpncaOZnJ5PRIoWtyNGbm38AS1NQiEGlF76z5mucWf8Xn24soraoFoGNCJDmZKYzKSmHqyO6Eh2roTlpec1oEKUfb75wraE4wkWAzfkAnxg/oRF29Y8PXJeR+VUDutkKWf1XImyt3U1pVy0/H9vJ3TAkyxxosXo6na6ixdqsDsls8kUgQCA0x+ndOoH/nBK4+NROAi/+yiGc/+Yr4yDAmDcogNS7SvyElaKhrSCRALNmyn1+9vpqNe0oJDTHG9E7jkuFdOX9whsYQpNlaZK4hM0sGegNR32xzzi1sfjwRARiVncq8W89g/dclvL5iF698lseCDZ/Ts0Oc5i8Sn2pSITCz64Fb8EwctwI4Bc9Mod/zWTKRIGRmB5fJXLv7AKEhJfTpqFlNxbeaenrCLcAI4Cvn3FnAMGCfz1KJBLnt+8tZuHEfl4/oRpjOIhIfa+pvWKVzrhLAzCKdc+uBvr6LJRLc/rlsOyEGl4/oduyDRZqpqWMEed71hV8D3jWzQr67toCItJCPvtxHvYNrZi1lVFYqo7JTOKNPB01iJz5x3GcNmdmZQCIw17sWcavSWUMSDLbsK+Xt1V+zZGsBy7cVUFZdR3ZaLG/cdDqxkZoiTI7f0c4aalIhMLNTgDXOuRLv43igv3NuSYsmbQIVAgk2tXX1vL9+Lz95fjmXDO/KHy4b4u9I0ga1xDTUTwKlDR6XebeJiI+FhYYwYUAnpp/Vi38vz+ONL9QrKy2rqYXAXIOmg3OuHk1hLdKqbh7Xm2Hdk/jFq6vIKyz3dxxpR5paCLaY2c1mFu693QJs8WUwETlUeGgIf758GM7BrbNXUF5d6+9I0k40tRBMA04DdgJ5eBaZv8FXoUSkcd1TY/jfKYP4bHshFz/xCVvzy/wdSdqBJnXvOOf2Alf4OIuINMEFQzqTFBPOTf/8nHP/vJDzBnXm8hHdGJGZrDmJ5IQ0qUVgZn3M7H0zW+19PNjMfunbaCJyJGN6d+A/N53OxcO6MHf1br7/108Z98cP+euHm9lXUuXveNLGNPX00Q+B/wH+6pwb5t222jk30Mf5vkOnj4ocqqyqljdX7ebFZTvI/aqQsBBjXL90Lh/RjTN6d9AUFQK0zOyjMc65pYc1OzVSJRIAYiPD+H5ON76f041Ne0t5MXcHLy/PY96aPXRJiua/x2Rx+YjuREeE+juqBKimflTIN7OeeNcvNrNLgd0+SyUiJ6RXehy/mNSPT+8ax5NXDScjMYp731jL6b/7gCfmb+JAZY2/I0oAamrXUDYwE8+ZQ4XAVuAq59xXvo33XeoaEjk+S7bs54kFm1m4cR/xUWFce1omN32vNxFh6jIKJs3uGnLObQHONrNYPK2ICuByoNULgYgcn1HZqYzKTmVVXjEPvLWOxz7YxOm90hiVnervaBIgjvqRwMwSzOwuM3vczM4ByoFrgE3A91sjoIi0jEFdE+mXkUBEWAiDuyb5O44EkGO1CJ7D0xX0KfDfwM+BCOAi59wK30YTkZa2YMNeTs1O1cCxHOJYhSDbOTcIwMyeAvKB7t/MQioibce2/DK25Jdx9ak9/B1FAsyxRosOnmLgnKsDtqoIiLRN63YfADxdRCINHasQDDGzA95bCTD4m/tmdqA1AopIyxjWPRmApVsL/ZxEAs1RC4FzLtQ5l+C9xTvnwhrcTzjWk5vZRDPbYGabzOzOoxw3wszqvNcniIgPdEqMon9GAvPX7/V3FAkwPjuR2MxCgSeAc4H+wFQz63+E434HzPNVFhHxGNcvndyvCigqb/VVZiWA+fKKkpHAJufcFu/axrOByY0cdxPwMqCPKSI+dtZJ6dQ7+HDjPn9HkQDiy0LQBdjR4HGed9tBZtYFuBiYcbQnMrMbzCzXzHL37dMvsMiJGtI1idTYCD5Q95A04MtC0NjE6IfPZ/EIcIf3jKQjcs7NdM7lOOdyOnTo0FL5RIJOaIhxZt8OLNiwj/xSTVctHr4sBHlAtwaPuwKHr7qdA8w2s23ApcBfzOwiH2YSCXpXjOhORU0dEx/5SF1EAvi2ECwDeptZlplF4FnhbE7DA5xzWc65TOdcJvAS8FPn3Gs+zCQS9EZmpfD69NEkx4Rzzayl3P+ftVTVHrVRLu2czwqBc64WuBHP2UDrgBedc2vMbJqZTfPV64rIsfXLSOCNm07n6lN78NTHW5nyF61/HMyaNA11INE01CIt6921e/jZv7+gtq6eB6YMYvLQLsf+JmlzjjYNtSYkFwly5/TvyFu3jOGkjARumb2CW2d/TmGZrjMIJioEIkKXpGhm33AKt4zrzX9W7uacPy1k7uqv/R1LWokKgYgAEB4awm3n9OH1G0eTHh/JtOeXc+MLn7Ffp5m2eyoEInKIAZ0Tef3G0dx+Th/mrfma8X9ayJsrtUR5e6ZCICLfER4aws3jevPGTafTOSma6S98xl8/3OzvWOIjKgQickQndUrg1Z+exojMZF5anufvOOIjKgQiclRhoSFMHJjBl3tL2VFQ7u844gMqBCJyTGP7eub4WqApKdolFQIROabstFi6p8Twxhe7qK9vWxehyrGpEIjIMZkZ14/JYunWAv747gZ/x5EWFubvACLSNvzglB6s232AJ+ZvJjstjktO7urvSNJCVAhEpEnMjF9PHshX+8u565VVxESEcu6gDH/HkhagriERabLw0BCevOpk+mXE85P/+4xfvLqKimpNYd3WqRCIyHFJjAnn39NO48dnZvPCku1c+PjHrP/6gL9jSTOoEIjIcYsIC+Guc/vx3I9Gkl9axY0vfO7vSNIMKgQicsLG9O5AckwEPVJi/B1FmkGFQERO2L6SKrbklzEiK8XfUaQZVAhE5ITlbisAPOsgS9ulQiAiJ2zptgKiwkMY2DnR31GkGVQIROSEbd9fTnZaHBFh+lPSlumnJyInrKbeqQi0A/oJisgJq62rJzzU/B1DmkmFQEROWG2dIzREhaCtUyEQkRNWU19PeKj+jLR1+gmKyAmpr3cUldeoELQD+gmKyAl5aXkeW/PLmDigk7+jSDOpEIjIcdtfWsUDb69jZGYKl2pdgjZPhUBEjttjH2yiqLyG+yYPIESDxW2eCoGIHLdBXTxXEv9j0Tac0xrGbZ1WKBOR43bJyV3Zml/G4/M30SU5mpvH9fZ3JGkGn7YIzGyimW0ws01mdmcj+68ys5Xe2ydmNsSXeUSk5fy/8X2YMqwLD7+7kX/n7vB3HGkGn7UIzCwUeAI4B8gDlpnZHOfc2gaHbQXOdM4Vmtm5wExglK8yiUjLMTMevGQwe0oqueuVVXRMiOKMPh38HUtOgC9bBCOBTc65Lc65amA2MLnhAc65T5xzhd6HiwGdfiDShkSEhfDkf51Mr/Q4fvp/n7F2l5asbIt8WQi6AA3bi3nebUfyI+DtxnaY2Q1mlmtmufv27WvBiCLSXAlR4fz9uhHER4Vx7d+X8unm/f6OJMfJl4WgsXPKGj29wMzOwlMI7mhsv3NupnMuxzmX06GDmp4igSYjMZp/XDeSiLAQpv5tMTf983O+Lq70dyxpIl8WgjygW4PHXYFdhx9kZoOBp4DJzjl9lBBpo/p2iue928/k5nG9mbfma8b9cQEzF26mpq7e39HkGHxZCJYBvc0sy8wigCuAOQ0PMLPuwCvAD5xzG32YRURaQVR4KLef04d3bzuDU7JTeeCt9Zz754/4ZFO+v6PJUfisEDjnaoEbgXnAOuBF59waM5tmZtO8h90NpAJ/MbMVZpbrqzwi0np6pMby9LUjeOrqHKpq67jyqSXc+MJn6i4KUNbWrgrMyclxubmqFyJtRWVNHTM+3MyTCzYTGmLcPK43PxydpZXNWpmZLXfO5TS2Tz8JEfGpqPBQbj27D+/edian9UzlwbfXc+6fF7JI3UUBQ4VARFpF99QYnrpmBLOuzaGmznHVU0s479GPePrjrewrqfJ3vKCmriERaXWVNXXMXrqdlz/byaqdxYSGGGf0TmPK8K6c078jUeGh/o7Y7hyta0iFQET86ss9Jbzy+U5e+3wnu4sriY8MY9KgDKYM78KIzBRNc91CVAhEJODV1zsWb9nPK5/v5O1VuymrrmPqyG7875TB/o7WLmiwWEQCXkiIcVqvNB66bAjLfnk2w7snsXZ3ib9jBQUVAhEJODERYWSmxrK/VIPIrUGFQEQCUmpcBPtLq/0dIyioEIhIQEqNi6Sipo7y6lp/R2n3VAhEJCAlRYcDqFXQClQIRCQgfba9kLjIMDrER/o7SrunQiAiAaeqto63V3/N+AG6uKw1qBCISMB5Ycl2SiprmTz0aIsaSkvx2eL1IiLHq77e8Yd3NvDkgs2M7pXK6J6p/o4UFFQIRCQglFXVcuu/VvDu2j1cOao79104gLBQdVq0BhUCEfG7vMJyrn8ml417Srj3gv5cc1omZppjqLWoEIiIX+09UMlFTyyisLyGp68ZwVknpfs7UtBRu0tE/KreQWxkGHX1jl++tpq/LdxCaZUuImtNKgQi4ledEqP44P+N5a8/OJmuydH89q11/OaNtf6OFVRUCETE70JDjAkDOvGvH59Kzw6xahG0MhUCEQkoZsaW/DI+3LhPBaGVaLBYRALKqdmpvLB0O9fMWkpoiDGgcwIjM1MYmZXCiMwUkmMj/B2x3dEKZSIScMqqavlseyFLtxawZGsBK3YUUV1bD0DfjvGeopCVwqisFDomRPk5bdugpSpFpE2rrKlj1c7ig4Vh+bYCyqrrAOiRGsPIzBRO7ZnK6F5pKgxHoEIgIu1KbV09a3cfOFgYlm0roKi8BoCeHWI5rWcao3ulcmp2Gokx4X5OGxhUCESkXauvd6zdfYBPNuezaNN+lm4toKKmDjMY2DmRkVkppMVFEhcZSlxUGLERYcRFhREfGU6sd1tcZBjR4aHt9opmFQIRCSrVtfWs2FHEJ5vz+WTTfj7fUUhN3bH/1oWY5+K2+MgwYiPDDhaIb26xkWHER4WRFhdJl6RouiRH0zkpmsTowG91qBCISFBzzlFZU09JVQ1lVXWUVtZ+e7+qhlLvtrKqWkq/uVXWUlZdS0nD7ZW1lFbXcvifzfjIsINFoWGB6OK9pcdHEhLi35bG0QqBTh8VkXbPzIiOCCU6IhTim/dc9fWO/LIqdhZWsKuokp1F5ewsrGBnUSU7iyrI3VbAgcpDr38IDzUyEqPpnBRFl6QYuiRH06XB/YzEKL8uwKNCICJyHEJCjPT4KNLjoxjWvfFjSiprvi0SRZXeQlHBrqIKFm3KZ09J5XdaFZ7upihvkTi0ddHF2/3kq/ELFQIRkRYWHxVO307h9O3UePOjuraePQcqyWtQIHYWVrCruIL1u0t4f91eqrzXTXwjOjyUhy4bwnmDM1o8r08LgZlNBP4MhAJPOecePGy/efdPAsqBa51zn/kyk4iIv0WEhdAtJYZuKTGN7nfOsb+s+pCWxP1vrmPVzuK2VQjMLBR4AjgHyAOWmdkc51zDaQXPBXp7b6OAJ71fRUSClpmRFhdJWlwkQ7olAfD7eRt89nq+nHRuJLDJObfFOVcNzAYmH3bMZOBZ57EYSDKzli93IiJyRL4sBF2AHQ0e53m3He8xmNkNZpZrZrn79u1r8aAiIoFu4oBOnHSEMYfm8uUYQWPD24dftNCUY3DOzQRmguc6guZHExFpWx6dOsxnz+3LFkEe0K3B467ArhM4RkREfMiXhWAZ0NvMsswsArgCmHPYMXOAq83jFKDYObfbh5lEROQwPusacs7VmtmNwDw8p4/Ocs6tMbNp3v0zgLfwnDq6Cc/po9f5Ko+IiDTOp9cROOfewvPHvuG2GQ3uO2C6LzOIiMjRac1iEZEgp0IgIhLkVAhERIKcCoGISJBrcwvTmNk+4KtWeKk0IL8VXqe5lLNlKWfLUs6W1ZycPZxzHRrb0eYKQWsxs9wjreYTSJSzZSlny1LOluWrnOoaEhEJcioEIiJBToXgyGb6O0ATKWfLUs6WpZwtyyc5NUYgIhLk1CIQEQlyKgQiIkEu6AuBmU00sw1mtsnM7mxk/2QzW2lmK7yrpJ0eiDkbHDfCzOrM7NLWzNfg9Y/1fo41s2Lv+7nCzO4OxJzeY8Z6M64xsw9bO6M3w7Hez/9p8F6u9v7sUwIwZ6KZvWFmX3jfz1afabgJGZPN7FXv//elZjawtTN6c8wys71mtvoI+83MHvX+O1aa2fBmv6hzLmhveKbH3gxkAxHAF0D/w46J49uxlMHA+kDM2eC4D/DM+HppIOYExgL/aQM/9yRgLdDd+zg9EHMedvwFwAeBmBP4BfA77/0OQAEQEWAZ/wDc471/EvB+a7+X3tc+AxgOrD7C/knA23hWeDwFWNLc1wz2FsFIYJNzbotzrhqYDUxueIBzrtR5330glkaW0mwFx8zpdRPwMrC3NcM10NSc/taUnFcCrzjntgM45/zxnh7v+zkV+GerJDtUU3I6IN7MDM+HqwKgNsAy9gfeB3DOrQcyzaxjK2bE+9oL8bw/RzIZeNZ5LAaSzCyjOa8Z7IWgC7CjweM877ZDmNnFZrYeeBP4YStla+iYOc2sC3AxMAP/adL7CZzq7SJ428wGtE60QzQlZx8g2cwWmNlyM7u61dJ9q6nvJ2YWA0zE80GgtTUl5+NAPzxL0a4CbnHO1bdOPKBpGb8ApgCY2UigB57lcwNNk38vmirYC4E1su07n/idc686504CLgJ+4+tQjWhKzkeAO5xzdb6Pc0RNyfkZnjlPhgCPAa/5OlQjmpIzDDgZOA+YAPzKzPr4OthhmvT76XUBsMg5d7RPkr7SlJwTgBVAZ2Ao8LiZJfg21iGakvFBPMV/BZ7W9ee0bqulqY7n96JJfLpCWRuQB3Rr8Lgrnk8sjXLOLTSznmaW5pxrzQmqmpIzB5jtaXmTBkwys1rn3GutktDjmDmdcwca3H/LzP4SoO9nHpDvnCsDysxsITAE2Ng6EQ9maOrv5xX4p1sImpbzOuBBbzfrJjPbiqcffmnrRGzy7+Z14BmQBbZ6b4HmuP5uNYk/BkMC5YanEG4Bsvh2AGnAYcf04tvB4uHAzm8eB1LOw47/B/4ZLG7K+9mpwfs5EtgeiO8nnm6M973HxgCrgYGBltN7XCKePuXY1v6ZH8f7+SRwr/d+R+//o7QAy5iEdwAb+G88/fCt/n56Xz+TIw8Wn8ehg8VLm/t6Qd0icM7VmtmNwDw8ZxXMcs6tMbNp3v0zgEuAq82sBqgALnfen0aA5fS7Jua8FPiJmdXieT+vCMT30zm3zszmAiuBeuAp51yjp/P5M6f30IuBd5yn9dLqmpjzN8A/zGwVnj9gd7hWbAU2MWM/4Fkzq8NzxtiPWitfQ2b2Tzxn16WZWR5wDxDeIOdbeM4c2gSU423FNOs1W/n/oIiIBJhgHywWEQl6KgQiIkFOhUBEJMipEIiIBDkVAhGRIKdCIEHJO0vnNzN2/ts7RUNzn/PXZnb2UfZP89NUFSJHpdNHJSiZWalzLs57//+A5c65hxvsD3X+na5DpNWoRSACHwG9vOsPzDezF4BVZhZqZn8ws2Xeed9//M03mNnPzWyVd/K8B73b/mHedSDM7EEzW+v9voe82+41s5957w81s8Xe/a+aWbJ3+wIz+513PvyNZjamtd8MCT5BfWWxiJmFAecCc72bRuKZSmKrmd0AFDvnRphZJLDIzN7BM0fORcAo51y5HbYQjPfxxcBJzjlnZkmNvPSzwE3OuQ/N7Nd4rh691bsvzDk30swmebcfsbtJpCWoRSDBKto7y2QunvmOnvZuX+qc+2aisfF4phdZASwBUoHeeP4w/905Vw7gvjvj5wGgEnjKzKbgmQbgIDNLBJKcc9+sevYMnsVIvvGK9+tyPHPOiPiUWgQSrCqcc0MbbvDO3Npwvh7D86l93mHHTeQo0/5657UZCYzDMyvojcD3jiNblfdrHfo/Kq1ALQKRI5uHZ4K8cAAz62NmscA7wA+/OdOoka6hOCDROfcWnu6eoQ33O+eKgcIG/f8/APyyJrII6NOGyNE8hadr5jPv/PT7gIucc3PNbCiQa2bVeGaD/EWD74sHXjezKDytitsaee5rgBneYrKFFphBUuRE6fRREZEgp64hEZEgp0IgIhLkVAhERIKcCoGISJBTIRARCXIqBCIiQU6FQEQkyP1/4B+zd+NCMHUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plot of Precision (x-axis) Vs Recall (y-axis)\n",
    "plt.plot(df_scores_1['precision'], df_scores_1['recall'], label='Precision Vs Recall')\n",
    "plt.xlabel('Precision')\n",
    "plt.ylabel('Recall')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f56a997eaf0>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAuu0lEQVR4nO3deXxV1bXA8d/KPJKEDEwBEiABAgSQEEBAQCyTVsXhCQ4o6sO5tK/1SftaWqdX61OLM1KlivZJfc4DokVBUJFRiMyEOYyBJAwhEJKs98e9Yggh3MA9uUnu+n4++eSec/Y5d22Dd929zz57i6pijDHGfwX4OgBjjDG+ZYnAGGP8nCUCY4zxc5YIjDHGz1kiMMYYPxfk6wBqKyEhQVNSUnwdhjHGNCjLli3br6qJ1R1rcIkgJSWFpUuX+joMY4xpUERk25mOWdeQMcb4OUsExhjj5ywRGGOMn2tw9wiMMTU7ceIEeXl5HDt2zNehGB8ICwsjOTmZ4OBgj8+xRGBMI5OXl0d0dDQpKSmIiK/DMXVIVTlw4AB5eXmkpqZ6fJ5jXUMiMl1E9onIqjMcFxF5RkRyRSRHRC5wKhZj/MmxY8eIj4+3JOCHRIT4+PhatwadvEfwKjCihuMjgTT3zwTgRQdjMcavWBLwX+fyt3esa0hV54tISg1FrgBmqGse7O9EJFZEWqjqbifiWb/nMJ/k7Dptf0bLGEZ0be7EWxpjTIPgy1FDrYAdlbbz3PtOIyITRGSpiCzNz88/pzfL3XeEZ+fmnvLzzJe53PfmckpKy8/pmsaY6gUGBtKjRw+6du3Ktddey9GjR8/7mpMnT2bOnDlnPD516lRmzJhx3u+TmprK+vXrT9n3y1/+kscff9yj81NSUujWrRuZmZkMGjSIbdvO+BzXORk8ePDJh2pTUlLYv3//eV/Tl4mguvZLtavkqOo0Vc1S1azExGqfkD6rSzNbsOXPl57y89qt2ZwoV77bcuCcrmmMqV54eDgrVqxg1apVhISEMHXq1FOOl5fX/svXQw89xCWXXHLG43feeSfjxo2r9XWrGjNmDDNnzjy5XVFRwdtvv811113n8TXmzp1LTk4OgwcP5pFHHjnvmJzmy0SQB7SutJ0MnN5346A+qU0JDQrgq/Xn1sowxpzdwIEDyc3NZd68eQwZMoTrr7+ebt26UV5ezv3330/v3r3JzMzkpZdeOnnO448/Trdu3ejevTuTJk0C4JZbbuHtt98GYNKkSWRkZJCZmclvfvMbAP70pz/xxBNPALBixQr69u1LZmYmo0ePprCwEHB9m37ggQfIzs4mPT2dBQsWnBbv2LFjT0kE8+fPJyUlhbZt27J69Wqys7Pp0aMHmZmZbNy4sca69+vXj507dwKQn5/P1VdfTe/evenduzfffPMNAEeOHGH8+PEnWxHvvPMOAHfddRdZWVl06dKFP/7xj7X/D18Lvhw++iFwr4jMBPoAB526P3AmYcGB9Gsfz1cbLBGYxunBj1azZtchr14zo2UT/vjzLh6VLSsr49NPP2XECNe4kcWLF7Nq1SpSU1OZNm0aMTExLFmyhOPHj9O/f3+GDRvGunXreP/991m0aBEREREUFBSccs2CggLee+891q1bh4hQVFR02vuOGzeOZ599lkGDBjF58mQefPBBpkyZcjKmxYsXM2vWLB588MHTupsyMzMJCAhg5cqVdO/enZkzZzJ27FjA1f00ceJEbrjhBkpLS8/aspk9ezZXXnklABMnTuRXv/oVAwYMYPv27QwfPpy1a9fy8MMPExMTww8//ABwMmk9+uijNG3alPLycoYOHUpOTg6ZmZke/XevLccSgYi8CQwGEkQkD/gjEAygqlOBWcAoIBc4Cox3KpaaDE5P5E8frWHbgWLaxkf6IgRjGp2SkhJ69OgBuFoEt912G99++y3Z2dknx7d//vnn5OTknPyWf/DgQTZu3MicOXMYP348ERERADRt2vSUazdp0oSwsDBuv/12Lr30Ui677LJTjh88eJCioiIGDRoEwM0338y111578vhVV10FQK9evdi6dWu18f/YKujSpQsffPABDz30EOD6hv/oo4+Sl5fHVVddRVpaWrXnDxkyhL1795KUlHSya2jOnDmsWbPmZJlDhw5x+PBh5syZc0oLJC4uDoC33nqLadOmUVZWxu7du1mzZk3DSwSqOvYsxxW4x6n399Sgjknw0Rrmrc/n5gstEZjGxdNv7t724z2CqiIjf/p/TFV59tlnGT58+CllZs+eXeMQyKCgIBYvXswXX3zBzJkzee655/jyyy89ji00NBRw3dAuKyurtszYsWMZNmwYgwYNIjMzk6SkJACuv/56+vTpwyeffMLw4cN5+eWXufjii087f+7cuURGRnLLLbcwefJknnrqKSoqKli4cCHh4eGnlFXV0+q7ZcsWnnjiCZYsWUJcXBy33HKLo0+K+/1cQ6kJkbSNj7DuIWPq2PDhw3nxxRc5ceIEABs2bKC4uJhhw4Yxffr0kyONqnYNHTlyhIMHDzJq1CimTJlyWsKJiYkhLi7uZP//66+/frJ14Kn27dsTHx/PpEmTTnYLAWzevJl27drxi1/8gssvv5ycnJwzXiM8PJwpU6YwY8YMCgoKGDZsGM8999zJ4z/GXXV/YWEhhw4dIjIykpiYGPbu3cunn35aq/hry+8TAbi6h77dtJ9jJ2wYqTF15fbbbycjI4MLLriArl27cscdd1BWVsaIESO4/PLLycrKokePHidvAP/o8OHDXHbZZSeHZ/71r3897dqvvfYa999/P5mZmaxYsYLJkyfXOr6xY8eybt06Ro8efXLfP//5T7p27UqPHj1Yt27dWUcptWjRgrFjx/L888/zzDPPsHTpUjIzM8nIyDg5kur3v/89hYWFdO3ale7duzN37ly6d+9Oz5496dKlC7feeiv9+/evdfy1Ia4emoYjKytLvb0wzdx1+xj/6hJm3JrNRennNjzVmPpi7dq1dO7c2ddhGB+q7t+AiCxT1azqyluLAOjbLp6QoADrHjLG+CVLBEB4SCB9Upsyb/0+X4dijDF1zhKB2+COSWzKL2ZHwfk/Cm+MrzW0Ll/jPefyt7dE4DbIfW/AuodMQxcWFsaBAwcsGfihH9cjCAsLq9V5tjCNW/vESJLjwpm3Pp8b+7b1dTjGnLPk5GTy8vI41wkaTcP24wpltWGJwE1EGNwxkXeX7+R4WTmhQYG+DsmYcxIcHFyr1amMsa6hSganJ3G0tJxlWwt9HYoxxtQZSwSV9GsfT0hgAPPsPoExxo9YIqgkMjSI3qlxNozUGONXLBFUMTg9iQ17j7CrqMTXoRhjTJ2wRFDFoI6uYaRf557/8m/GGNMQWCKoon1iFMGBwtb9xb4OxRhj6oQlgioCA4SWseHkFVrXkDHGP1giqEZyXDg7Cm2qCWOMf7BEUI3k2AhrERhj/IajiUBERojIehHJFZFJ1RyPE5H3RCRHRBaLSFcn4/FUclw4+YeP20I1xhi/4FgiEJFA4HlgJJABjBWRjCrFfgesUNVMYBzwtFPx1EZyU9eaojttCKkxxg842SLIBnJVdbOqlgIzgSuqlMkAvgBQ1XVAiog0czAmjyTHRQBY95Axxi84mQhaATsqbee591W2ErgKQESygbbAadPmicgEEVkqIkvrYkbF5DhXiyDPbhgbY/yAk4lAqtlXdYL0x4A4EVkB3Ad8D5SddpLqNFXNUtWsxETn1xROig4jOFCsRWCM8QtOTkOdB7SutJ0M7KpcQFUPAeMBRESALe4fn7JnCYwx/sTJFsESIE1EUkUkBBgDfFi5gIjEuo8B3A7MdycHn0uOC7euIWOMX3AsEahqGXAv8BmwFnhLVVeLyJ0icqe7WGdgtYiswzW6aKJT8dSWPUtgjPEXjq5QpqqzgFlV9k2t9HohkOZkDOeqc4to/rl0Bws25jMwzfn7EsYY4yv2ZPEZjMluQ2pCJH94f5U9WGaMadQsEZxBWHAgD1/Rla0HjvLCvE2+DscYYxxjiaAGA9ISuLJHS6bO28Sm/CO+DscYYxxhieAs/uvSDEKDA/j9e6tQrfoYhDHGNHyWCM4iMTqUB0Z0YuHmA3ycs9vX4RhjjNdZIvDA2Ow2dG7RhMc+XWc3jo0xjY4lAg8EBgiTL8tgZ1EJf5u/2dfhGGOMV1ki8FC/9vGM6NKcF+ZtYs/BY74OxxhjvMYSQS38blRnyiuUx2ev83UoxhjjNZYIaqFNfAS3DUzl3e938v32Ql+HY4wxXuHoFBON0T1DOvD2sjwe+ngN7951Ia5JUz1z7EQ5Ww8UsyW/mC3u37sOlhAYEEB4cABhwYGEBQUSFhxAWIjrdau4cK65IJmAAM/fxxhjasMSQS1FhQZx//CO/OfbOfxj0XYGpf80D1FidChhwYGnnVN0tJS/LdjMq99spbj0p1FHSdGhJMeFU67l7DtUTsmJco6dKOfYiQpKTpRTWlYBwKZ9R/jtqM7OV84Y45csEZyDay5I5vWF2/j9+6tO2d8yJow3bu9Du8QoAA6WnOCVBZuZ/s1WikvLGNWtBSO6NCc1IZKUhEiiQmv+z19Rofzpo9W8NH8zyU0juKlvW8fqZIzxX9LQnpbNysrSpUuX+joMCopLmbtu38kl10rLKnjy8/WICC/ddAFfbzzAy19v5vCxMkZ1a87Eoel0bB5d6/cpK6/gjteXMXf9Pl65uTdDOiV5tyLGGL8gIstUNavaY5YIvCd332FueHkRew8dB2BYRjN+eUk6GS2bnNd1i4+X8W8vLWTL/mLeuqMfXVvFeCNcY4wfsURQh7YdKOa1b7cxumcruiV77wN776FjjH7+G8oqlPfv6U/L2HCvXdsY0/jVlAhs+KiXtY2PZPLPM7yaBACaNQlj+vjelJSWM/Zv35GTV+TV6xtj/JclggakU/MmvHprb0rLKrjqhW957suNlFc0rBadMab+sUTQwPRq25TZEy9iRNfmPPH5Bq57aSE7Co76OixjTAPmaCIQkREisl5EckVkUjXHY0TkIxFZKSKrRWS8k/E0FjERwTw7tidTruvB+j2HGfn0At5ZlmfrJRhjzoljiUBEAoHngZFABjBWRDKqFLsHWKOq3YHBwJMiEuJUTI2JiHBlz1bMmjiQjBZN+PX/reTeN7+n6Gipr0MzxjQwTrYIsoFcVd2sqqXATOCKKmUUiBbXPA1RQAFQ5mBMjU7rphG8OaEv/zmiI5+t2sOIKQv4Nne/r8MyxjQgTiaCVsCOStt57n2VPQd0BnYBPwATVbWi6oVEZIKILBWRpfn5+U7F22AFBgh3D+7Ae3f3JyI0kOtfXsSjn6zheJktomOMOTsnE0F1s6RV7cQeDqwAWgI9gOdE5LSnr1R1mqpmqWpWYmJi1cPGrVtyDJ/cN5Cb+rblbwu2cNkzX/P6d9s4ePSEr0MzxtRjTiaCPKB1pe1kXN/8KxsPvKsuucAWoJODMTV64SGBPHxlV6bfkkWACH94fxW9H53D3f9Yxhdr91JWflqDyyuOl5Wzo+Aoy7YV8P32QrtxbUwD4uSkc0uANBFJBXYCY4Drq5TZDgwFFohIM6AjYGtBesHFnZoxpGMSq3cd4u1leXy4chezfthDQlQIV/ZoxdW9kunc4tTGV0lpOUu2FvD99iKOlZVToYqqa/K7CsW97Xp9tLScfYePse/QcfYdPkZhlVbHwLQEJl+WQVqz2s+vZIypW45OMSEio4ApQCAwXVUfFZE7AVR1qoi0BF4FWuDqSnpMVd+o6Zr1fYqJ+qq0rIJ56/fxzvI8vly3jxPlSkaLJlx1QStKSsv5ZtN+lm8rorS8AhEIDghABAJECHD/FoGAACFAhNCgAJKahJEUHUqzJqE0iw4jqUkoSU3C2LTvCE9/sZGjpeWM69eWX16STkx4sK//Exjj12yuIXOKwuJSPly5i3eW55GTdxCALi2b0L9DAv07JNA7JY6IkPNrLB44cpwn/7WBNxdvJyo0iFFdW3BFz5b0TY23RXaM8QFLBOaMth84SlRYEE0jnXl8Y/Wug7zy9RY+W7WH4tJymjcJ4/IeLRnQIYHkuHBaxoZXu5iPMca7LBEYnyspLWfO2r18sGIn89bnU1ZpjqSEKNdKba3iwuncPJrx/VOJPMuiPcaY2rFEYOqVoqOlrNtzmJ2FJewsKvnpd1EJW/YX0yo2nEeu7GqL8BjjRTUlAvvaZepcbEQIfdvFV3ts2bYCJr3zA+NfXcLl3Vsy+ecZJESF1nGExvgXm33U1Cu92jbl418M4FeXpDN71R4ueeor/m/pDnsuwRgHWSIw9U5oUCATL0lj1sQBdEiM4v63c7jxlUXk7jvi69CMaZQsEZh6q0NSNG/d0Y9HruxKzo6DjJgynz99uNpmWDXGyywRmHotIEC4sW9b5t4/mH/r3ZoZC7cy6H/m8eo3WxybLsMYf2OJwDQICVGh/PfobsyaOJCurZrwp4/WcNmzX7Nw0wFfh2ZMg2fDR02Do6p8tnoPD3+8lp1FJVya2YJRXVuQHBdOclw4TSNDcC1xYYz5kQ0fNY2KiDCiawsGd0xi6lebeHHeJj7J2X3yeHhwIK3cScH1E0GrWNfrClXyCkvIq/IMQ2RIIE9d14P2iVE+rJkxvmEtAtPgFR8vY3vBUfcHvOv3zsIS8opcr4vOsB5D08gQWsWG0yo2nCVbC6hQ5eWbs+jVtmkd18AY51mLwDRqkaFBdG7R5LRptX90+NgJdhaVkFdQQmCAnJzOovLEetsOFHPz9MVc/7dFPD2mJyO6Nq+r8I3xOWsRGON24Mhxbp+xlBU7injo8i7c1C/F1yEZ4zU1tQhs1JAxbvFRofzv7X0Z2qkZf/hgNW8u3u7rkIypE5YIjKkkPCSQF264gMEdE/mv937g0x92n/0kYxo4SwTGVBESFMCLN/SiZ5s4Js5cwTe5+30dkjGOskRgTDXCQwKZfnNvUhMimTBjKSt3FPk6JGMc42giEJERIrJeRHJFZFI1x+8XkRXun1UiUi4iNnbP1AsxEcHMuC2bplEh3PL3xWzYe9jXIRnjCI8SgYj0F5F/icgGEdksIltEZPNZzgkEngdGAhnAWBHJqFxGVf9HVXuoag/gt8BXqlpwTjUxxgHNmoTx+q19CAwI4LJnvubBj1Zz4MhxX4dljFd5+hzBK8CvgGVAuYfnZAO5qroZQERmAlcAa85QfizwpofXNqbOpCRE8vF9A5gyZwOvfbuVfyzazoAOCQztnMTQTs1oHhPm6xCNOS8ePUcgIotUtU+tLixyDTBCVW93b98E9FHVe6spGwHkAR2qaxGIyARgAkCbNm16bdu2rTahGOM1ufuO8I9F25izdi87CkoA6NYqhks6N+OSjCQyWjSxeY5MvXTeaxaLyGNAIPAucLJdrKrLazjnWmB4lUSQrar3VVP2OuBGVf352WKxB8pMfaCqbNx3hH+t2csXa/fy/Y4iVKFdQiQzJ/QlqYm1Ekz94o0pJn5sDVS+iAIX13BOHtC60nYysOsMZcdg3UKmARER0ptFk94smnuGdCD/8HG+WLuXyR+s5i+z1/Pkv3U/pXxFhbKj8ChBgQG0ig33UdTGVM+jRKCqQ87h2kuANBFJBXbi+rC/vmohEYkBBgE3nsN7GFMvJEaHMia7DVsPHGXqV5u4oG0sR46VsWHvETbsPUzuviOUnCgnMEB4dXxvBqYl+jpkY07ydNRQjIg8JSJL3T9Puj/Az0hVy4B7gc+AtcBbqrpaRO4UkTsrFR0NfK6qxedaCWPqi3sv7kBSdCj/9d4q/vzpOhZszCc2Ipix2W147KpupCVFcfc/ltv6y6Ze8fQewTvAKuA1966bgO6qepWDsVXL7hGY+m77gaPsOXSM9GZRxEaEnHJsR8FRrnz+G6LCgvjdqM6kxEfSPjGSoEB7ttM4yxs3i1e4x/rXuK8uWCIwDd2ybQXc9Mpijpa6RmIP6JDA67dl22gj4yhvzD5aIiIDKl2wP1DijeCM8Te92jZl0e+G8uG9/bl7cHu+zt3PZ6v3+Dos48c8HTV0F/Ca+76AAAXALU4FZUxjFx0WTGZyLBktmvCvNXv5y+z1DO3cjGDrIjI+4NG/OlVdoardgUygm6r2VNWVzoZmTOMXFBjAb0d1Ysv+Yv53ka1/YHyjxhaBiNyoqm+IyH9U2Q+Aqj7lYGzG+IUhHZPo1y6ep/61geNl5Yzs2oLWTSN8HZbxI2drEUS6f0ef4ccYc55EhIev7EqbphH896x1DHx8Lle/+C37bXI7U0dszWJj6pHtB47y4cqdPPH5Bv748wzG90/1dUimkTjvUUMi8riINBGRYBH5QkT2i4g9CWyMl7WJj+Dei9NolxDJgo22MpqpG54OURimqoeAy3DNIZQO3O9YVMb4uQFpCXydu5+JM7/nxXmbmLt+H3sOHqOhteBNw+Dp8NFg9+9RwJuqWmAPvxjjnHH92rKrqISlWwv5YMVPczX2bBPLSzf1IinaZjc13uNpIvhIRNbheojsbhFJBI45F5Yx/q1DUjQv39wbgIMlJ1i/5zArdxTx1L82cNUL3/Lq+Gw6JEX5OErTWHh8s1hE4oBDqlruXkimiarW+eOQdrPY+LOcvCJufXUJJ8qVv43LIjvVlvg2njnnm8UicrH791XAEOAK9+sRwIXeDtQYU7PM5Fjeu7s/8VEh3PjKImb9sNvXIZlG4GxdQ4OAL4HqVg5TXCuWGWPqUOumEbx714Xc/tpS7nvzewIERnRt4euwTANmzxEY00AVHy9j3PTF5OQVMfXGXgzt3MzXIZl6zBvPEfy3iMRW2o4TkUe8FJ8x5hxEhgbx9/G96dyiCXe9sZz5G/JPK6OqlJZV+CA605B4uh7B96ras8q+5ap6gWORnYG1CIw5VdHRUsb+bRFb9h/hhj5tKSguZffBEvYcPMbug8eoUOWduy4kMznW16EaH/LGegSBIhJa6YLhQGgN5Y0xdSQ2IoQ3bnMNJ52xcCtLthZQXqF0S47l5gtTiAkP5pGP19rDaOaMPH2O4A3gCxH5O66bxLfy07KVZyQiI4CngUDgZVV9rJoyg4EpuB5a26+qgzyMyRjjFh8Vykf3DkAVAgJOfdizbXwE//XeKp7+YiMBIpScKOeOi9qdtoym8V+1eY5gBHAJroVpPlfVz85SPhDYAPwM17QUS4CxqrqmUplY4FtghKpuF5EkVd1X03Wta8iY2ikrr2DUMwvYsPcIAAECKfGR/H18b9rGR57lbNNY1NQ15GmLAGAtUKaqc0QkQkSiVfVwDeWzgVxV3ewOYiZwBbCmUpnrgXdVdTvA2ZKAMab2ggID+N9/78vuomO0T4pk1c5D3PH6Uq58/htm3NqHbskxvg7R+Jino4b+HXgbeMm9qxXw/llOawXsqLSd595XWToQJyLzRGSZiIw7w/tPEJGlIrI0P//0kRHGmJolRIXSLTmGiJAgslOb8t7d/QkPDuSX//ye42Xlvg7P+JinN4vvAfoDhwBUdSOQdJZzqpuVrmo/VBDQC7gUGA78QUTSTztJdZqqZqlqVmJioochG2POJCUhkkdHd2NTfjHTvtrs63CMj3maCI6raumPGyISxOkf6lXlAa0rbScDu6opM1tVi1V1PzAf6O5hTMaY8zCkUxKjujXn2bm5bN1f7OtwjA95eo/gKxH5HRAuIj8D7gY+Oss5S4A0EUkFdgJjcN0TqOwD4Dl3YgkB+gB/9TR4Y8z5+ePPuzB/w35ueHkRHZtHExcRQlxEMHGRITSNDKFlbDgDOyScNhLJNC6eJoIHgNuBH4A7gFnAyzWdoKplInIv8Bmu4aPTVXW1iNzpPj5VVdeKyGwgB6jANcR01blVxRhTW82ahPH0mB78/Zut7D10jPV7DlNQXErJiZ/uGwzpmMgT13YnPsoeHWqszjp8VEQCgBxV7Vo3IdXMho8a47xjJ8opPFrK56v38uistcRFBDPlup70ax/v69DMOTqvJ4tVtQJYKSJtvB6ZMaZeCgsOpEVMODdfmMJ7d19IZEgQ17/8HU99vp5jJ2yUUWPj6VxDXwK9gcXAybtKqnq5c6FVz1oExtS94uNl/OGDVby7fCctYsK4rndrurSMoXOLaFrFhmNL19Z/NbUIPE0E1U77oKpfnWdstWaJwBjf+XbTfp78fAPLthWe3NckLIgre7Zi0shORITU5hlVU5fO+cliEQkD7gQ64LpR/Iqqlnk/RGNMQ3Bh+wQuvCuB4uNlrNtzmDW7D7F8WyGvf7eNrzfu580JfWnWJMzXYZpaOts9gteALFxJYCTwpOMRGWPqvcjQIHq1jeOmvm3563U9+Mdtfdh1sIT/fDvHZjltgM6WCDJU9UZVfQm4BhhYBzEZYxqYCzsk8NuRnflqQz6vfbvV1+GYWjpbIjjx4wvrEjLG1OSmvm0ZlJ7Inz5aw8SZ35NXeNTXIRkP1XizWETK+WmUkADhwFH3a1XVJo5HWIXdLDam/iotq+DFeZt4bu5GKhSGdExiQId4+rSLp2OzaHtC2YfOe9RQfWKJwJj6b1dRCa8t3MrHK3ezs6gEgKjQIFrFhtM8JowWMWE0jwmjZUw4A9MTaBET7uOIGz9LBMYYn8krPMqizQWszCti98FjJ9dS3n/kOADRYUE8fnUmI7u18HGkjZslAmNMvVNaVsGm/CNMeieHlXkHubFvG35/aQZhwYG+Dq1RskRgjKm3SssqeOLz9Uybv5mEqFAGpScyuGMiA9MSbF1lL/LWUpXGGON1IUEB/G5UZwalJzJzyQ6+WLeXd5bnESDQs00cg9ITGdm1OWnNon0daqNlLQJjTL1SXqGs2FHEV+v3MW9DPjl5BwkJCmDebwbTMtZuKp+r85p91Bhj6lJggNCrbRz/MawjH947gE8nDqS0rILPVu/xdWiNliUCY0y91rlFE9KSonj9u23M+mE3R0vt2VZvs3sExph671c/S+f376/i7n8sJzQogEHpiQztnMQFbeJonxhlD6qdJ0sExph6b1S3FgzLaMaSrYXMXrWb2av38PmavYBrGuwebeLo2TqWXm3j6Nc+nuBA6+yoDUdvFovICOBpXGsWv6yqj1U5PhjXAvZb3LveVdWHarqm3Sw2xqgqm/cXs3xbIcu3F/H99kI27D1MhUKzJqHc1LctY7Pb2DrLlfjkOQIRCQQ2AD8D8oAlwFhVXVOpzGDgN6p6mafXtURgjKnOkeNlLNx0gBkLt7Jg435CggIY27s1vxiaZgkB3z1HkA3kqupmdxAzgSuANTWeZYwx5yAqNIifZTTjZxnNyN13hFe+3swbi7bzzvKd3D4wlZv6trWEcAZOdqS1AnZU2s5z76uqn4isFJFPRaRLdRcSkQkislRElubn5zsRqzGmEemQFMWfr8rks19exIXt45kyZyMXPvYlv303h417D/s6vHrHyRZBdbfxq/ZDLQfaquoRERkFvA+knXaS6jRgGri6hrwcpzGmkeqQFMW0cVls3HuY6d9s5d3leby5eAcXpSdy+4BUBqYlIGIjjpxsEeQBrSttJwO7KhdQ1UOqesT9ehYQLCIJDsZkjPFDac2i+fNV3Vj426H8+mfprN19iHHTF/OHD1ZRUWHfLZ1MBEuANBFJFZEQYAzwYeUCItJc3OlYRLLd8RxwMCZjjB9rGhnCfUPT+PqBIUy4qB1vfLed3733g98nA8e6hlS1TETuBT7DNXx0uqquFpE73cen4loH+S4RKQNKgDHa0CY/MsY0OKFBgfx2ZCfCggJ45stcth04yvV92vCzjGZ+OQ22TTpnjPFrf/9mCy99tZk9h44RHRrEyG7Nubx7K/q0a9qoHkyz9QiMMaYG5RXKos0HePf7ncxetYcjx8uICQ9maOckRnRpzkXpiQ2+pWCJwBhjPHTsRDnzN+Qze/Ue5qzZy6FjZUSEBPLc9T25uFMzX4d3ziwRGGPMOThRXsF3mw/wyMdrKSopZc5/DCI6LNjXYZ0TW4/AGGPOQXBgAAPTEvnLNZnsO3ycJz/f4OuQHGGJwBhjzqJH61jGZrfh9e+2cfDoCV+H43WWCIwxxgNXX9CK8gplQW7jm+bGEoExxnige3IsMeHBfLBiF6VlFb4Ox6ssERhjjAeCAgO4rndr/rVmL8P++hWzV+2hoQ22ORNLBMYY46HfjuzE32/pTXBgAHe+sYwbX1lE0dFSX4d13iwRGGOMh0SEIZ2S+HTiQB6+ogtLthTyby8tZPfBEl+Hdl4sERhjTC0FBQZwU78UXh3fm11Fx7j6hW+Zs2Zvg+0qskRgjDHn6MIOCcyc0JegwABun7GUy579ms9W72lws5laIjDGmPPQtVUMX/x6EP9zTSbFx8u44/VljHpmAZ+t3uPr0DxmicAYY85TcGAA12a1Zs5/DOKv13WntLyCO15fxp9nrW0QrQNLBMYY4yVBgQGM7pnM57+8iJv6tuWl+Zu553+Xc+xEua9Dq5ElAmOM8bKgwAAeuqILv7+0M7NX72HMtO/Yf+S4r8M6I0sExhjjABHh9oHtePGGXqzbc4hxryz2dUhnZInAGGMcNKJrc8b1SyF33xFfh3JGjiYCERkhIutFJFdEJtVQrreIlIvINU7GY4wxvhAYIL4OoUaOJQIRCQSeB0YCGcBYEck4Q7m/4Frk3hhjGp3osCBKyyvq7XQUTrYIsoFcVd2sqqXATOCKasrdB7wD7HMwFmOM8Zmstk0BWLylwMeRVM/JRNAK2FFpO8+97yQRaQWMBqY6GIcxxvhU99YxhAYFsMgPE0F1nWJVn6yYAjygqjUOshWRCSKyVESW5uc3vkUhjDGNW2hQIH3axfPPJTv4bvMBX4dzGicTQR7QutJ2MrCrSpksYKaIbAWuAV4QkSurXkhVp6lqlqpmJSYmOhSuMcY45/GrM2keE8bN0xfz5bq9vg7nFE4mgiVAmoikikgIMAb4sHIBVU1V1RRVTQHeBu5W1fcdjMkYY3yieUwYb93Rj/Rm0UyYsaxeDSd1LBGoahlwL67RQGuBt1R1tYjcKSJ3OvW+xhhTXzWNDOHv410L27wwN9fX4ZwU5OTFVXUWMKvKvmpvDKvqLU7GYowx9UFCVCg39m3DK19v4RdD00hJiPR1SPZksTHG1LV/v6gdwYEBvDR/k69DASwRGGNMnUuKDmNk1+Z8tnov5fVgmmpLBMYY4wMXd25GQXEpy7YV+joUZ+8RGGOMqd6gtETCgwMZM20hPVrHclF6IoPSE+nROhaRup2byFoExhjjAzERwbx/T3/uGdKB8grl6S82MvqFb3nkk7V1Hou1CIwxxkc6No+mY/OO/HpYRwqKS3ny8/W88vUW0pKiGJPdps7isBaBMcbUA00jQ3jw8i5clJ7IHz5Yxdrdh+rsvS0RGGNMPREUGMCU63oQEx7Mr99aSWlZRZ28ryUCY4ypR5pGhvDo6G6s2X2Ivy3YXCfvaYnAGGPqmeFdmnNReiIzFm6lrNz5VoElAmOMqYdu7NOGvYeO88U659fsskRgjDH10MWdkmjeJIx/LNru+HtZIjDGmHooKDCAMdmtmb8hn90HSxx9L0sExhhTT12U7lqIa9VOZ4eSWiIwxph6Ki0pCoANew87+j6WCIwxpp6KDgumVWy4JQJjjPFn6c2i2LDX2WUtLREYY0w9ltYsmk37jji6boElAmOMqcfaJURSWl7BriLnRg45mghEZISIrBeRXBGZVM3xK0QkR0RWiMhSERngZDzGGNPQpLrXNN68v9ix93AsEYhIIPA8MBLIAMaKSEaVYl8A3VW1B3Ar8LJT8RhjTEOUmuhKBFvynbtP4GSLIBvIVdXNqloKzASuqFxAVY+o6o8dX5GA7xfvNMaYeiQxKpSo0CC2NMQWAdAK2FFpO8+97xQiMlpE1gGf4GoVnEZEJri7jpbm5+c7EqwxxtRHIkJqQmTD7BoCqlt087Rv/Kr6nqp2Aq4EHq7uQqo6TVWzVDUrMTHRu1EaY0w916ZpBHmFDfNmcR7QutJ2MrDrTIVVdT7QXkQSHIzJGGManOSm4ewsLKHCoSGkTiaCJUCaiKSKSAgwBviwcgER6SAi4n59ARACHHAwJmOMaXCS4yIoLa9g3+HjjlzfscXrVbVMRO4FPgMCgemqulpE7nQfnwpcDYwTkRNACXBdpZvHxhhjgNZx4QDkFR6leUyY16/vWCIAUNVZwKwq+6ZWev0X4C9OxmCMMQ1dclwEADsKj5KV0tTr17cni40xpp5L/rFFUODMDWNLBMYYU8+FBQdyRY+WtG4a4cj1He0aMsYY4x1Pj+np2LWtRWCMMX7OEoExxvg5SwTGGOPnLBEYY4yfs0RgjDF+zhKBMcb4OUsExhjj5ywRGGOMn5OGNsebiOQD287x9ARgvxfDaQiszv7B6uwfzqfObVW12gVdGlwiOB8islRVs3wdR12yOvsHq7N/cKrO1jVkjDF+zhKBMcb4OX9LBNN8HYAPWJ39g9XZPzhSZ7+6R2CMMeZ0/tYiMMYYU4UlAmOM8XONMhGIyAgRWS8iuSIyqZrjIiLPuI/niMgFvojTmzyo8w3uuuaIyLci0t0XcXrT2epcqVxvESkXkWvqMj4neFJnERksIitEZLWIfFXXMXqbB/+2Y0TkIxFZ6a7zeF/E6S0iMl1E9onIqjMc9/7nl6o2qh8gENgEtANCgJVARpUyo4BPAQH6Aot8HXcd1PlCIM79eqQ/1LlSuS+BWcA1vo67Dv7OscAaoI17O8nXcddBnX8H/MX9OhEoAEJ8Hft51Pki4AJg1RmOe/3zqzG2CLKBXFXdrKqlwEzgiiplrgBmqMt3QKyItKjrQL3orHVW1W9VtdC9+R2QXMcxepsnf2eA+4B3gH11GZxDPKnz9cC7qrodQFUber09qbMC0SIiQBSuRFBWt2F6j6rOx1WHM/H651djTAStgB2VtvPc+2pbpiGpbX1uw/WNoiE7a51FpBUwGphah3E5yZO/czoQJyLzRGSZiIyrs+ic4UmdnwM6A7uAH4CJqlpRN+H5hNc/vxrj4vVSzb6qY2Q9KdOQeFwfERmCKxEMcDQi53lS5ynAA6pa7vqy2OB5UucgoBcwFAgHForId6q6wengHOJJnYcDK4CLgfbAv0Rkgaoecjg2X/H651djTAR5QOtK28m4vinUtkxD4lF9RCQTeBkYqaoH6ig2p3hS5yxgpjsJJACjRKRMVd+vkwi9z9N/2/tVtRgoFpH5QHegoSYCT+o8HnhMXR3ouSKyBegELK6bEOuc1z+/GmPX0BIgTURSRSQEGAN8WKXMh8A49933vsBBVd1d14F60VnrLCJtgHeBmxrwt8PKzlpnVU1V1RRVTQHeBu5uwEkAPPu3/QEwUESCRCQC6AOsreM4vcmTOm/H1QJCRJoBHYHNdRpl3fL651ejaxGoapmI3At8hmvEwXRVXS0id7qPT8U1gmQUkAscxfWNosHysM6TgXjgBfc35DJtwDM3eljnRsWTOqvqWhGZDeQAFcDLqlrtMMSGwMO/88PAqyLyA65ukwdUtcFOTy0ibwKDgQQRyQP+CASDc59fNsWEMcb4ucbYNWSMMaYWLBEYY4yfs0RgjDF+zhKBMcb4OUsExhjj5ywRGFMN92ylK0RklXtmy1gvX3+riCS4Xx/x5rWNqS1LBMZUr0RVe6hqV1wTgN3j64CMcYolAmPObiHuSb1EpL2IzHZP6LZARDq59zcTkffcc+KvFJEL3fvfd5ddLSITfFgHY86o0T1ZbIw3iUggrukLXnHvmgbcqaobRaQP8AKuyc6eAb5S1dHuc6Lc5W9V1QIRCQeWiMg7jWCeJ9PIWCIwpnrhIrICSAGW4ZrRMgrXAj//V2k201D374uBcQCqWg4cdO//hYiMdr9uDaQBlghMvWKJwJjqlahqDxGJAT7GdY/gVaBIVXt4cgERGQxcAvRT1aMiMg8IcyJYY86H3SMwpgaqehD4BfAboATYIiLXwsm1Y39c+/kL4C73/kARaQLEAIXuJNAJ17KCxtQ7lgiMOQtV/R7XWrljgBuA20RkJbCan5ZNnAgMcc+AuQzoAswGgkQkB9cMmd/VdezGeMJmHzXGGD9nLQJjjPFzlgiMMcbPWSIwxhg/Z4nAGGP8nCUCY4zxc5YIjDHGz1kiMMYYP/f/jx6jVs4YPLMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plot of Recall (x-axis) Vs Precision (y-axis)\n",
    "plt.plot(df_scores_1['recall'], df_scores_1['precision'], label='Precision Vs Recall')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#After filling NaNs in precision with 1, calculating auc with recall first and precision second, dont get the above error\n",
    "auc_score = auc(df_scores['recall'],df_scores['precision'])\n",
    "auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dropping NaNs in precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scores_2 = df_scores.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scores_2 = df_scores_2.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>tp</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tn</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>386</td>\n",
       "      <td>1023</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.273953</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.430084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.1</td>\n",
       "      <td>366</td>\n",
       "      <td>556</td>\n",
       "      <td>20</td>\n",
       "      <td>467</td>\n",
       "      <td>0.396963</td>\n",
       "      <td>0.948187</td>\n",
       "      <td>0.559633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.2</td>\n",
       "      <td>333</td>\n",
       "      <td>356</td>\n",
       "      <td>53</td>\n",
       "      <td>667</td>\n",
       "      <td>0.483309</td>\n",
       "      <td>0.862694</td>\n",
       "      <td>0.619535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.3</td>\n",
       "      <td>284</td>\n",
       "      <td>236</td>\n",
       "      <td>102</td>\n",
       "      <td>787</td>\n",
       "      <td>0.546154</td>\n",
       "      <td>0.735751</td>\n",
       "      <td>0.626932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.4</td>\n",
       "      <td>249</td>\n",
       "      <td>166</td>\n",
       "      <td>137</td>\n",
       "      <td>857</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.645078</td>\n",
       "      <td>0.621723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.5</td>\n",
       "      <td>210</td>\n",
       "      <td>101</td>\n",
       "      <td>176</td>\n",
       "      <td>922</td>\n",
       "      <td>0.675241</td>\n",
       "      <td>0.544041</td>\n",
       "      <td>0.602582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>150</td>\n",
       "      <td>53</td>\n",
       "      <td>236</td>\n",
       "      <td>970</td>\n",
       "      <td>0.738916</td>\n",
       "      <td>0.388601</td>\n",
       "      <td>0.509338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.7</td>\n",
       "      <td>76</td>\n",
       "      <td>20</td>\n",
       "      <td>310</td>\n",
       "      <td>1003</td>\n",
       "      <td>0.791667</td>\n",
       "      <td>0.196891</td>\n",
       "      <td>0.315353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.8</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>373</td>\n",
       "      <td>1022</td>\n",
       "      <td>0.928571</td>\n",
       "      <td>0.033679</td>\n",
       "      <td>0.065000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    threshold   tp    fp   fn    tn  precision    recall        f1\n",
       "0         0.0  386  1023    0     0   0.273953  1.000000  0.430084\n",
       "10        0.1  366   556   20   467   0.396963  0.948187  0.559633\n",
       "20        0.2  333   356   53   667   0.483309  0.862694  0.619535\n",
       "30        0.3  284   236  102   787   0.546154  0.735751  0.626932\n",
       "40        0.4  249   166  137   857   0.600000  0.645078  0.621723\n",
       "50        0.5  210   101  176   922   0.675241  0.544041  0.602582\n",
       "60        0.6  150    53  236   970   0.738916  0.388601  0.509338\n",
       "70        0.7   76    20  310  1003   0.791667  0.196891  0.315353\n",
       "80        0.8   13     1  373  1022   0.928571  0.033679  0.065000"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_scores_2[::10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f56a988fee0>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAzxklEQVR4nO3deVxVdf7H8deXXRYBWRRFBBG33EFNzbTct7RSyyyXccn2aX7NVDPTNu3LTKtpmuaSZZuWLe5blruG+waIAhoCCgLKer+/Pw4amspVgXOXz/PxOI+43MO9H07w9vA93/P9KK01Qggh7J+L2QUIIYSoHBLoQgjhICTQhRDCQUigCyGEg5BAF0IIB+Fm1hsHBwfryMhIs95eCCHs0rZt2zK11iGXes60QI+MjGTr1q1mvb0QQtglpdSRyz0nQy5CCOEgJNCFEMJBSKALIYSDkEAXQggHIYEuhBAOosJAV0rNVEqdUErtvszzSin1nlIqQSm1UynVrvLLFEIIURFrztBnAX2v8Hw/IKZsmwhMuf6yhBBCXK0K56FrrX9WSkVeYZfBwBxtrMO7USkVoJQK01ofr6wiL5BxEHZ9BbUaQlC08V/vIFCqSt5OCCGuRn5hCbPWJ1NYXHrZfeIia3Fz40veG3RdKuPGonpASrnHqWWf+1OgK6UmYpzFExERcW3vlr4L1r0F2vLH5zxrQq0oI9wv3nxrS9gLIarNoh3HeHPpAeDy0TOpW7TNBvqlSr5k1wyt9TRgGkBcXNy1ddZocSc0HQjZR+FkEmQlwqnDxsfHd8DeRaDL/cvoWROiboZGPaBRTwi4xn9IhBDCCusTs6hd05ONT/dAVfPJZGUEeipQv9zjcOBYJbzu5bl5QnCMsV2stBhyUoyAP3kY0ndDwirY/4PxfHBjI9gb9YAGXcC9RpWWKoRwHlprNiRm0jUmpNrDHCon0BcBDyul5gMdgZwqGz+3hqv7H8Mt52gNWQmQsAIOLYctM2Djh+DmBZE3lQV8TwhqJMMzQohrduhEHpl5RXSKDjLl/SsMdKXU50B3IFgplQo8B7gDaK2nAj8B/YEE4AwwtqqKvWZK/XFGf+MDUHQGjqyHhOWQsBKWPGXsV6sh9H/TCHchhLhK6xMyAehsq4GutR5RwfMaeKjSKqoOHt4Q09PYAE4lG8G+eRp8eid0nAQ9n5fhGCHEVVmfmEVELW/CA71NeX+5UxQgMBLaj4OJa6DjA7BpKkzrboR8broxZCOEEFdQatFsTMoy7ewcTFwP3Sa514B+r0FML/j2Qfj0DuPzHr7GtMjwDsZwTNTN4Olrbq1CCJuy99hpTheUmDZ+DhLol9aoBzy8GVK2lM2WSYTMQ7BjPmydAS7u0KAztBsFzW4DNw+zKxZCmGx9ojF+3qmhBLrt8fL/Y4z9nJJCSNlkzJbZ+x18M864cSl2LLQcZgzduMohFcIZrU/MolGoL6E1vUyrQdLnarh5GsMtUTdDj+eNYN88Dda+ZmwubsaNS4FRFw7JePpB1yeMYRshhMMpKrGwJfkkQ2PDTa1DAv1aubhA497GlpUIRzeUDc+U3dB0Ou2PfXNSYc+30O8NaHOPzHUXwsHsTM3mTFGpqRdEQQK9cgRFG9vlZB+FhQ/Adw/CwSUw6F3wrlV99QkhqtT6xCyUgo5R5ga6TFusDgERMHoR9HwBDiyGqV3h90suLy+EsEPrEzNpHlaTQB9zJ0hIoFcXF1e46a8wfrmxUuTMPnBwqdlVCSGu06n8IrYfyaZLo2CzS5FAr3Z128KEVca6MZ/fDRs+lBuXhLBjC35Lo6jUwh3t6pldigS6KWqGwdjF0HQALH0aZg8ylv0tLTG7MiHEVdBa8/nmo7SNCKBpnZpmlyOBbhoPbxg2B/q8CqeOwJf3wbutYd1/oTDP7OqEEFbYeuQUCSfyGNHBNvosSKCbycUFOj0Ij8XD3Z8bq0Gu/A+83w62zwHL5VtYCSHM9/mmo/h5ujGwVZjZpQAS6LbBxRWa9odR38K4FRDQABY9Ah/dDDu/NDoxFeaaXaUQopzsM0X8sOs4Q9rWw9vDNmaA20YV4g/128O4ZbBnASx/HhZM+OM5n1Co30Ha6QlhAxb+lkZRicVmhltAAt02KVXWO3UQZB74o3dqVgIc/vmPdnqhN8CQycbMGSFEtTl3MbR1/QCa1zX/Yug5Eui2zM0D6rQ0tnO0NlZ+TFhhtNH7pD/cMQ2aDTKvTiGczPajpziYnsfrd7aseOdqJGPo9kYpCGlsXEydsApCm8MX98Ivb8t8diGqQUFxKa8vOYCvpxsDW9U1u5wLSKDbM99QGPODMTyz4nlYeL9cPBWiChUUlzJx7ja2JJ/kP4NvwMfTtgY5JNDtnXsNuHMGdP+nMSNm6k2QstnsqoRwOIUlpTzw6TZ+PpjB63e04o525i6VeykS6I5AKej+JIz9CSxl68SsfgVKi82uTAiHUFRi4aF521l9IINXbm/J8Pb1zS7pkiTQHUmDzvDAL9DqLlj7OrzTCta+CXkZZlcmhF2bujaRFftO8J/BN3BPR9uZpngxCXRH4+UPt0+Fkd9AaFNY/RK83RwWToKz2WZXJ4TdOZp1hsmrExjQMoxRnSLNLueKbGtEX1SemJ7GlnkINk+HrTMh4wDctwBqBJpdnRB2QWvNc4t24+aieGZgc7PLqZCcoTu64Bjo/wbcNRfSd8OcwXDmpNlVCWEXlu1NZ/WBDP7aszF1/M1r/mwtOUN3Fk36wV3zjDnrc26DUYukDZ5wOBaLZuPhLDLzisgtKCavoARXF0WDIB8aBHkTUcsbL3dXq17rTFEJ//l+L01q+zGmS2TVFl5JJNCdSePeMOIz+Pweow1ehwnQbpQEu3AIWmv+uXAX87ekXHYfpaB/yzCeG9ScUL8Lz7iPZZ9lz7HT5x+v3JdOWvZZvprUCXdX+xjMkEB3No16Gv1NV70EK56DNa9Ci6HQ5THjDlQh7NSbSw8wf0sKE29uyPC4cHw93fHzcqOoxMKRk2c4kpXP7rQcZm84wrqDGfxrQDOGx9Vnz7HTTPs5iR93HafUcuHd1nfF1ad9pP2c8Cht0u3icXFxeuvWraa8tyhzYp9xwXTHfCgpgPbjoNtT4GNu53IhrtbH65J46cd9jOgQwSu3t0Apddl9EzPyeHrBLjYfPkm9gBqkZZ/F19ONezpG0K9FnfNn4y5K0bSOHy4ul38tMyiltmmt4y75nAS6ID/TOFPf+gl4+EK3f8CNDxoNOISwcQu2p/K3L3fQr0UdPrinHa5WBLDFovliawoLt6fRq3lt7upQn5pe7tVQ7fWTQBfWObEflv0bEpZDm5Fw2/tG8w0hbNSeYzkMmfwr7SNr8cnY9ni6Of7P65UC3apTMKVUX6XUAaVUglLqqUs876+U+l4ptUMptUcpNfZ6ixYmCG0K934N3Z+G+Hnw7YPSBk/YrILiUh7/Ip5Abw8m39POKcK8IhVeFFVKuQKTgV5AKrBFKbVIa7233G4PAXu11oOUUiHAAaXUPK11UZVULapW96dAucDql0GXwpCp4CrXz4VteWvpAQ6m5zFrbHsCfTzMLscmWPNb2gFI0FonASil5gODgfKBrgE/ZVyJ8AVOAiWVXKuoTt3+YQy3rPwPFJ+FwZOhRoDZVQkBwPrETD7+5TD33diA7k1CzS7HZlgz5FIPKD+xM7Xsc+V9ADQDjgG7gMe01paLX0gpNVEptVUptTUjQxaMsnld/w/6vgYHl8CULnB4ndkVCcHpgmKe+HIHDYN9eLp/U7PLsSnWBPqlLhlffCW1DxAP1AXaAB8opf7UaE9rPU1rHae1jgsJCbnKUoUpbnzAaFrt5gmzB8GyZyDvhHRHEqY4nnOW8bO3kp5byP/uaoO3hwwFlmfN0UgFyi/+G45xJl7eWOA1bUyZSVBKHQaaAtJpwRHUi4VJ62DpP2H9e8bm4Qu1oqB2S+jzstxtKqyWeuoMM345zPK96YT6eRIZ5ENksA+hfp64uijcXBWuLi5Eh/jQrE7N8/PAF+04xr8X7qLEonlrWCva1A8w9xuxQdYE+hYgRikVBaQBdwP3XLTPUaAHsE4pVRtoAiRVZqHCZB4+MOhdaHMvHNsOJ5OMbfc3kJVg3H3qXsPsKoWN0lqzKy2HGb8c5oedx1FA9yYh5BeWsiEpiwW/pV3y62p6udEhKggXZSyU1TYigLeHtyEy2Kd6vwE7UWGga61LlFIPA0sBV2Cm1nqPUmpS2fNTgReBWUqpXRhDNE9qrTOrsG5hlvrtje2cPd/CV2Pgm/EwfI7MWxfnaa3ZdzyXH3cd46ddv3M4Mx9fTzf+0iWSsV2iqBvwxwlAQXEpWflFWCyaUoumqNTCnmM5bEw8yabDWaRln+Xxno156JZo3OxkXRUzyI1F4vptnApLnoT246H/W8YKSMLpnC4o5ovNKRxMzyUpM5/EjDyyzxTjoqBTdBADWtZlQKsw/Gtc/R2ZpRZt1R2gzuBKNxbJFQVx/W6cBKfTjLF1L3/jxiRX+7iNWlSOhBN5TJy7laSMfEL8PIkO8aF/yzBa1PWn9w21Cfb1vK7XlzC3jgS6qBw9XzDWhFn3X9i7CHq/BI37yNm6E1i+N53Hv4jHy92FLybeSMeGsribWWQwSlQOFxcY8iGM+MJ4/PldRnekrERz6xJV5vecAl5bvJ8Jc7bSMMSHRQ/fJGFuMjlDF5VHKWjSFxr1MFZuXP0yfNIPRn8PIU3Mrk5UgrzCEn7adZxvf0tjQ1IWWsPQ2HBeGtLC6k5AourIRVFRdTIOGDcjaW1MawxtZnZF4hqdyC1g1q/JzN14hNyCEhoEeTOkTT0Gt6lLwxBfs8tzKnJRVJgjpAmM+RFmDTS20Yug9g1mVyUuUmrR5BWU4O994YXs/MIStiSfZMnu31mwPY1ii4V+Leow7qYo2kUEXrGJhDCHBLqoWsExMPanslAfAL1fhtYjpHmGjdiVmsPjX8aTcCKPQG93Gob40iDIm+TMfHam5lBi0Xi6uTAsLpwJXRvKDT02ToZcRPU4mWTcfJS2DcJaG8Ee1dXsqpxWSamFqWsTeWfFIYJ9Pbn3xgjSsgtIysgjOSufMP8adI4OolN0ELENAmXNFBsiQy7CfLUawrgVxlIBK56H2QMh6mZo0t9oXB3USKY4VpP00wU88Ok2th/NZlDrurw0uMWfhluEfZJAF9XHxQVaDYNmA2HjhxD/OSwpa4AVEGEEe6OeRtB7+plbq4PKyC1kxPSNpOcU8O7dbRjc5uKVsIU9kyEXYa5TyZCwEhJWwOGfoSgPXNwgopMx/bFRT6jdQs7eK0FWnhHmKSfPMmdcB9pHygqZ9kiaRAv7UFIEKZuMcE9YAem7jc/71ik7e+8BDbvLUr3X4FR+ESOmbyQ5K59PxnSgU7TcAGSvJNCFfTp9HBJXQcJySFwNBdlGr9N6cdDlUWg2yOwKbdqZohK2Jp9iQ1IWi3cd51hOATNGx9E1RprL2DMJdGH/LKXGDJmEFcaSvZkHoO29Ros8GW8/T2vNukOZzF6fzM+HMigu1bi5KFqF+/N4r8YS5g5AZrkI++fiCvU7GFvXJ2Dt6/DL/yD5F7h9GkR0NLtCUxUUl/LV1hRmrU8mMSOfYF8PxnaJokujYOIaBOLjKb/qzkD+Lwv74+YBPZ4xxtUXToSZvSH0BmOMPaYX1L/R2MfBlJRaKCix4FsunEstmgXbU3l7+UGO5RTQsp4//xvemgGtwvB0k7VVnI0MuQj7VnAatn0Ch5bD0Y1gKQb/+nD7VIi8yezqKoXWmh93Hee1xftJPXWWhiE+tKkfQOPafizYnsrB9Dxah/vzZN+mdIoOklvyHZyMoQvnUJgLSWtg+XPGnaldHoNb/mW3Z+sWi2ZHajYv/7iPrUdO0bSOH31b1GF32mniU7LJzCskKtiHv/dpQr8WdSTInYSMoQvn4OlnzHxpeAss+xf8+g4kroRhsyEo2uzqKpSRW8g/F+7iYHou2WeKyS0oxqIh2NeT1+5oybC4+uc792itycgrpJa3h/TYFOdJoAvH4+kLg96FmD7w3UMwdwiMWw5+dcyu7LIOpecydtYWMvMK6d28DgHe7vjXcCe0phe3t613wbg5gFKKUD8vk6oVtkoCXTiupv2h5gL4ZADMG2as+miDUxx/OZTJA/O24eXuyhcTO9G6foDZJQk7JYEuHFvdtjB8Nnx2F3w5ymiRZ/KYutaalJNniU/NZlvySeZtOkp0iC8zxsQRHuhtam3CvkmgC8cX0wtue88Yfln0sDEc416j2ssoKC5lxi+HmfnLYbLyiwDwdHOhzw11ePXOltT0khUPxfWRQBfOoe29cPqY0ef00DJoex+0HweBkVX+1lprlu5J5+Wf9pJy8iy3Ng2lZ7PatAr3p0kdP9zloqaoJBLownl0+wc06AKbp8GGybD+fWjcFzpMMGbGVFIXpYLiUnal5XAwPZeDv+cSn5LNjtQcmtT2Y974jnRpFFwp7yPExSTQhXOJ7GJsOWnGDUnbZsGni40GG+0nQJsR4OV/zS+/7cgp/vZlPEeyzgDg6+lGTG1fXhx8AyM6RMgUQ1Gl5MYi4dxKCmHvd8ZZe+oWcPeB1ncbZ+2hzax+maISC++uPMiUNYnUDajB0/2a0SYigLr+XnLDj6hUcqeoENZI2w5bPoZdX0NpIUR2haYDKmyRdzgzn4fmbWfv8dMMjwvnmYHN8ZMLnKKKSKALcTXys+C3OfDbPMg6ZHwuoIFxIfXmJy4I9jUHTvDo57/h6qJ4/c5W9L7Bdm9eEo5Bbv0X4mr4BMFNjxtbWYs8vf9H1OqXIPc49H8LrRTTfk7i9SX7aVzbj+mj4qhfS+aQC3NJoAtxBTqgAWv8BvHyiWjuKPHhwa0zWLIrlY/8HuG31NMMaBnGm8Na4e0hv0rCfFb9FCql+gLvAq7Ax1rr1y6xT3fgHcAdyNRad6u0KoWoYlprdqTmUFhcSqCPB4HeHpzILeC1xftZdyiTqGAfirr9m1WHAul7YjYuupSkPq9yf/cYuegpbEaFga6UcgUmA72AVGCLUmqR1npvuX0CgA+Bvlrro0qp0CqqV4hKdSq/iG+2p/LZ5qMkZeT/6Xn/Gu48O7A5997YAA83F+j1HqypS+81r0KagvwPwVd+3IVtsOYMvQOQoLVOAlBKzQcGA3vL7XMPsEBrfRRAa32isgsVojIVlVh4a9kBZq1PpqjEQruIAN4c2oow/xqcPFNE9pkiSko1d7SrR4D3RWu/dH8KvINg2b/hw05w2/vGQmBCmMyaQK8HpJR7nApc3MCxMeCulFoD+AHvaq3nXPxCSqmJwESAiIiIa6lXiOuWln2Wh+ZtJz4lm6Gx4YzvGkXTOjWv7kU6TDCmNS4YD/NHQLvR0PdV8PCpmqKFsII1gX6pAcKL5zq6AbFAD6AGsEEptVFrffCCL9J6GjANjGmLV1+uENdOa83qAyf425c7KCnVfDiyHf1bhl37C4Y2hfGrjPVhfn3XaFh9x3QIj628ooW4CtYEeipQv9zjcODYJfbJ1FrnA/lKqZ+B1sBBhDDJseyzTF6dwMH0XE7kFnLidCFni0tpFlaTKSPbERlcCWfTbh7Q6wVjRccF98OMXtD9aWPKo6vMfBHVy5qfuC1AjFIqCkgD7sYYMy/vO+ADpZQb4IExJPN2ZRYqhLUKikv5aG0SU9YmoDW0qR9Aq/AAQv08iajlzV3t6+Pl7lq5bxp5EzzwK/z0BKx+CZJWw/A54CMLcYnqU2Gga61LlFIPA0sxpi3O1FrvUUpNKnt+qtZ6n1JqCbATsGBMbdxdlYULcY7Fokk5dYaD6XkcTM/ls01HScs+y4CWYTzVr2n13fBTIwDu/Bga9YLvH4Xpt8I9XxpDM0JUA7n1X9gNi0WTlJlPfEo2CSfySM7MJznL2AqKLef3a1nPn3/2b0an6CDzik3dBp/fDSUFMGwWNOphXi3Cocit/8JuFRSX8s32VJbtSSc+JZucs8UAuLsq6tfyJirIh5saBRNT25eY2n40CvW1jc4/4bEwYZUR6vOGGYFeKxpqNYSQJsYMmUpaf12IcyTQhU06lV/E3I1HmL0+maz8IqJDfOjXog7tIgJpGxFAVLCP7a8tHlAf/rIElj8LKVsg+VcoLrt5qW5b6PMqNOhkbo3CoUigC9MVlVhIOXWGvcdOsysth52p2cSnZFNQbOHWpqFMvLkhHaNq2ect9p5+MLBsfoDWkHcCElbAqpfgk77QfDD0eA6Cos2tUzgEGUMX1cJi0fx+uoCkjHwSM/JIzMjjcNkYeNqps1jKfgw93FxoFlaTtvUDuKdjBI1r+5lbeFUpOgMbPoBf3obis9Ckn3GzUlR3GYoRVyTroYtql5iRx9wNR9j/+2mOZRdwPOcsxaV//Kz5eboRFeJDZJAPkUHeRAT50LSOH41r+xlrpjiL3N+NphrbZkF+BgTFGMHeegR4XeXdq8IpSKCLavPb0VNMXZvIsr3peLi60KKeP/UCalAvsAbhgTVoGOxLdKgPIb6e9jmEUlXOtcLb9BGkbQUPX6MVXvsJMu1RXEACXVSpzLxCfthxjIXxx9iRko1/DXdGdWrA6M6RBPt6ml2e/UnbDpunw+5vjFZ4UTdDh4nQuJ/cfSok0EXl01qzPjGLj9cl8fOhTEotmuZhNRkaG87w9vXx9ZTguW75mbB9NmyZCadToWY4dPs7xI4xuzJhIpmHLiqN1ppfEjJ5d8Uhth45RaifJxNvbsiQNvVoUsdBL2CaxScYuv4fdH4MDi4xLqJ+/5jxnIS6uAQJdFGhs0WlbD1yko1JWaw9mMHutNOE+Xvx4uAbGBZXBeuiiAu5ukGzgdC4D3w+An54HHzrQJO+ZlcmbIwEuviTvMISth05xebDWWxKOsmO1GyKSzWuLopW4f68NKQFw+LC8XSTIK9Wru7GMgKzBsBXY2DMDxB+yb+8hZOSQBcUl1rYkZLNukOZ/JKQSXxKNqUWI8Bb1vPnLzdF0alhEHGRtWRs3GyevjDyK2OZ3s+Gw+0fGcsIuHuZXZmwAfLb6YQsFs3+33NZn5jJhsQsNh0+SV5hCUpBq3r+3H9zQzpFB9EuIhAfCXDb4xsK9y6AmX1h3lBwqwFRXY2bk9qNBhf5y8lZyW+rEykptfD9zmN8sCqBxLKGyFHBPtzWpi43NQqmc3TQn/tnCtsUFA2P7TC6JCWsMLYfHoecVOjxrNnVCZNIoDuBohILC39L5cM1iRzJOkPTOn68cWcrbooJpm5ADbPLE9fKwxsa9zY2gEWPwrr/Ggt/NRtkbm3CFBLoDiy3oJjPNx9lxi+HST9dSMt6/ky7L5aezWrj4iJ3aTqc/m9C+m5Y+AAEN4GQxmZXJKqZBLqDKS61sO3IKZbvTefLLSnkFpbQOTqIN4a25uaYYLnd3pG5eRpt7z7qBl+MNNZj95R7A5yJBLoDOJFbwJr9Gazaf4JfEzLJLSzBzUXR54Y63N+tIa3CA8wuUVQX/3AY9gnMGQKf3QW3/AsadAb5h9wpSKDbqYzcQr7YcpTl+06wIyUbgDB/Lwa2DqNb41C6NArCzxY694jqF3UzDHoXlv0bZvWH0BvKVnC8G9zlmokjk7Vc7ExWXiHTfk5i9oZkCoottK4fQM+mofRoVptmYX4ypCL+UHQGdn8Nm6ZB+i4IawMj5kPNMLMrE9dBFudyAKcLivlobSKf/JpMQXEpg9vU45FbG9EwxNfs0oSt0xr2/wAL7gcvfxjxOdRtY3ZV4hrJ4lx2rLjUwmebjvLuykOczC9iYKsw/tozhkahcrFLWEkpYxrjuEj47G74pB/cMd1YH0Y4FAl0G3W6oJglu39nyppEDmfm06lhEP8a0IwW9fzNLk3YqzotjZkv80fAF/dC54fh1meM2THCIUig25CiEgsr96XzXfwxVh04QVGJhZhQX2aOieOWJqEyPi6un19tGPMjLP0nrH8fElcbZ+u1m5tdmagEEug2ICO3kM82HeXTTUfIyC0k2NeTezpEcFuburStHyBBLiqXew0Y+DbE9IHvHoJp3aHvq9B+nNmVieskgW6i/b+fZtrPSXy/4xjFpZruTUIY3SmSrjHBuLk6UaNkYY4mfeHBDfDtA/Dj34ygb3OP2VWJ6yCBXs201mw6fJKP1iay+kAG3h6ujOzYgFGdGsiMFVH9fEPh7s+NVRsXPWI8btTT7KrENZJAr0Zp2Wd56pudrDuUSZCPB//XqzH3dWogKxwKc7l5wF2fwif94YtRMPYnmdZopyTQq4HWmq+3pfKf7/dSqjXPDGzOyI4R0rpN2A6vmn80zpg3DMavgMAGZlclrpIM1Fax5Mx8JszZyt+/3kmzujVZ8tjNjLspSsJc2J6aYXDvN1BSaIypC7sjZ+hV5GB6LpNXJ/D9jmO4u7rwzMDmjO0cKcvWCtsW0gS6/d1YByZxNUTfYnZF4ipYdYaulOqrlDqglEpQSj11hf3aK6VKlVJDK69E+3I85ywPzdtO77d/ZvnedMZ3bci6f9zCuJuiJMyFfWg/AfwjYPmzYLGYXY24ChWeoSulXIHJQC8gFdiilFqktd57if1eB5ZWRaG2zmLRzNt0hNeXHKDEYuGRWxvxly5RBPrIBU9hZ9y9oMczsGCCsbhXq+FmVySsZM2QSwcgQWudBKCUmg8MBvZetN8jwDdA+0qt0A7sTsvhuUV72HbkFF1jgnl5SEsigrzNLkuIa9diqHEn6coXodltRsgLm2dNoNcDUso9TgU6lt9BKVUPuB24lSsEulJqIjARICIi4mprtSkWi2bV/hPM+OUwG5KyCPB257/DWnNHu3pyZ6ewfy4u0PtFmDMYtkyHzo+YXZGwgjWBfql0unjN3XeAJ7XWpVcKM631NGAaGMvnWlmjzVm+N51Xf9pHUmY+Yf5ePNWvKSPaR+DvLQ0lhANp2N24yejnN6FRLwhtanZFogLWBHoqUL/c43Dg2EX7xAHzy8I8GOivlCrRWn9bGUXaiozcQp7/fg8/7jxO49q+vDeiLf1a1MFdbtMXjqr3y0bXo2ndoNeLRucj+QvUZlkT6FuAGKVUFJAG3A1csOCD1jrq3MdKqVnAD44U5qUWzdfbUnjlp/2cLSrlid6NmXhzNB5uEuTCwYU2hQc2GIt4Lf47HFoKgyeDXx2zKxOXUGGga61LlFIPY8xecQVmaq33KKUmlT0/tYprNI3WmpX7TvDm0gMcSM+lQ2QtXrmjJY1CZc0V4UT8aht3kW752Jif/kEHY656h4mylrqNkRZ0F9FacyyngG1HTjFnfTJbj5wiMsibJ/o0oX+LMJlLLpxb5iFY8jQkLIfASOj1H2MWjAzDVBtpQVeBguJSlu75ncW7fmf70VOcyC0EINTPk5dvb8HwuPoyTi4EQHAM3Ps1JKyApf+GL0dB7BgY+I6Eug1w6kDflZrD/C1HWbTjGLkFJYT5e9GlUTBtIwJoWz+QpmF+EuRCXEqjnhDVHVa+AOvfA7+60P1Js6tyek4X6Fpr1h7MYOraRDYmncTTzYV+LeowPK4+NzYMkiEVIazl6mYMueRnwppXoGZdaHef2VU5NacK9LUHM3ht8X72HT9NnZpe/HtAM4a3r09NL5k/LsQ1UQpuew/yfofvHwPf2tC4t9lVOS2nCPRSi+bt5Qf5YHUCDYN9eGNoK4a0qSfTDoWoDK7uMHyO0SDjq9FGE+p67cyuyik5fKKdzC9izCeb+WB1AsNiw/npsa4Mj6svYS5EZfL0g5Ffg08wfDYcTiaZXZFTcuhU23vsNAPfW8emwyd57Y6WvDmstTSWEKKq+NWGexeApQQ+vdMYWxfVymEDfUvySe6atgGLhm8mdebuDva9GJgQdiE4BkZ8AaePGWfqRflmV+RUHDLQV+8/wX0zNhHi68k3D3amZbi/2SUJ4TwiOsKdM+DYb/DFfVCYZ3ZFTsPhAv27+DQmzNlKo1BfvpzUiXoBNcwuSQjn02wgDHoXklbDzL6Qk2p2RU7BoQJ90Y5j/PWLeGIbBPL5hBsJ9pV1JoQwTbtRcM9XkH0Ept8KadvMrsjhOUygr9ibzt++iKd9g1rMGtsBP5lbLoT5YnrCuGXGIl6f9If4z8Gk9aOcgUME+vqETB78bDvN69Zkxpg4anjITBYhbEZoMxi/CurFwreT4Ou/wNlTZlflkOw+0H87eorxc7YSGeTNbDkzF8I2+YbA6O+hx7OwbxFM6QJJa82uyuHYdaAfzznLhDlbCfb15NNxHQn08TC7JCHE5bi4Qtf/g3HLwb2G0a90/09mV+VQ7DbQC4pLuX/uNgqKLcwcE0doTelKLoRdqNcO7v8ZwlrDwvshM8HsihyGXQa61pp/LdzNztQc3r6rDY1C/cwuSQhxNTx84K654OIGX4yUueqVxC4Dfdb6ZL7ZnsrjPRvTq3lts8sRQlyLgAgYOhMyDxo9S2X2y3Wzu0Bfn5jJSz/uo3fz2jxyayOzyxFCXI/oW6Dn87D3W1j/vtnV2D27C/QQX09uaRLK/+5qI80ohHAEnR81+pKu/A+k7zW7Grtmd4EeU9uPj0fH4evpFEu5C+H4lDJ6knrVhEWPgKXU7Irslt0FuhDCAfkEQb83IG0rbPrI7GrslgS6EMI2tLgTYvrAqhfhVLLZ1dglCXQhhG1QCgb+D5Sr0Z9UZr1cNQl0IYTt8A+HXs9D0hr49V2zq7E7EuhCCNsS+xdoNghWPAc/PgGlJWZXZDck0IUQtsXFBYbNhs6PwJbpRiu7ghyzq7ILEuhCCNvj4gq9X4JB78HhtTCjN5w5aXZVNk8CXQhhu2JHw8ivjeUBVr9sdjU2TwJdCGHbom+B9hNg60w4vtPsamyaVYGulOqrlDqglEpQSj11iedHKqV2lm3rlVKtK79UIYTTuuVpqBEIP/1dpjNeQYWBrpRyBSYD/YDmwAilVPOLdjsMdNNatwJeBKZVdqFCCCdWI9BYxCtlI+z80uxqbJY1Z+gdgAStdZLWugiYDwwuv4PWer3W+lyTwI1AeOWWKYRwem3uhbrtYPkzUHDa7GpskjWBXg9IKfc4texzlzMOWHw9RQkhxJ+4uED/tyAvHX5+w+xqbJI1gX6pNWovOYillLoFI9CfvMzzE5VSW5VSWzMyMqyvUgghAMJjoe29sHEqnDxsdjU2x5pATwXql3scDhy7eCelVCvgY2Cw1jrrUi+ktZ6mtY7TWseFhIRcS71CCGd3y7+N1nWrXjS7EptjTaBvAWKUUlFKKQ/gbmBR+R2UUhHAAuA+rfXByi9TCCHK1AyDzg/D7m8gbZvZ1diUCgNda10CPAwsBfYBX2qt9yilJimlJpXt9iwQBHyolIpXSm2tsoqFEKLzo+AdDMuelWmM5Sht0sGIi4vTW7dK7gshrtHm6fDTE3DPl9C4j9nVVBul1DatddylnpM7RYUQ9il2DNSKhuXPSdu6MhLoQgj75OoOPZ+DjH3wy//MrsYmSKALIexXs9uMbdVLsnY6EuhCCHumFAybVW7t9GFOvXa6BLoQwr5dsHb6z8ba6fmXvBXG4bmZXUB5xcXFpKamUlBQYHYpdsnLy4vw8HDc3d3NLkWI6hc7GgIjYd5QWPQI3D3POIN3IjYV6Kmpqfj5+REZGYlysv8R10trTVZWFqmpqURFRZldjhDmaNgNejwHy/4F22cbM2GciE0NuRQUFBAUFCRhfg2UUgQFBclfN0Lc+CA07A5LnobMQ2ZXU61sKtABCfPrIMdOCIxVGYdMBTdP+GY8lBSZXVG1sblAF0KI61YzDG57H47Hw8oXnGZ5AAn0i7i6utKmTRtatGjBsGHDOHPmzHW/5rPPPsuKFSsu+/zUqVOZM2fOdb+PEKKcZoOg/XjY8AF8/xiUFptdUZWzqbVc9u3bR7NmzUyp5xxfX1/y8vIAGDlyJLGxsfztb387/3xpaSmurq5mlVchWziGQtgMiwVWvwzr3oKom2H4HKOdnR270louNjXLpbwXvt/D3mOV22aqed2aPDfoBqv379q1Kzt37mTNmjW88MILhIWFER8fz65du3jqqadYs2YNhYWFPPTQQ9x///0AvPHGG8ydOxcXFxf69evHa6+9xpgxYxg4cCBDhw7lqaeeYtGiRbi5udG7d2/eeustnn/+eXx9fXniiSeIj49n0qRJnDlzhujoaGbOnElgYCDdu3enY8eOrF69muzsbGbMmEHXrl0r9fgI4XBcXKDHMxDUyJjK+HFPGPkV1GpodmVVwmYD3WwlJSUsXryYvn37ArB582Z2795NVFQU06ZNw9/fny1btlBYWEiXLl3o3bs3+/fv59tvv2XTpk14e3tz8uTJC17z5MmTLFy4kP3796OUIjs7+0/vO2rUKN5//326devGs88+ywsvvMA777xzvqbNmzfz008/8cILL1xxGEcIUU6bERDYAObfA3OGwLjl4Ffb7Koqnc0G+tWcSVems2fP0qZNG8A4Qx83bhzr16+nQ4cO5+d3L1u2jJ07d/L1118DkJOTw6FDh1ixYgVjx47F29sbgFq1al3w2jVr1sTLy4vx48czYMAABg4ceMHzOTk5ZGdn061bNwBGjx7NsGHDzj9/xx13ABAbG0tycnKlf+9COLQGnWHkNzB7IHw2HMb8CJ6+ZldVqWw20M1So0YN4uPj//R5Hx+f8x9rrXn//ffp0+fCNZiXLFlyxamDbm5ubN68mZUrVzJ//nw++OADVq1aZXVtnp6egHHhtqTEuRchEuKahMcaa798PgK+Gg0j5hurNjoImeVyDfr06cOUKVMoLjaumh88eJD8/Hx69+7NzJkzz8+MuXjIJS8vj5ycHPr3788777zzp384/P39CQwMZN26dQDMnTv3/Nm6EKKSNO4DA9+GhBWw6FEoKTS7okojZ+jXYPz48SQnJ9OuXTu01oSEhPDtt9/St29f4uPjiYuLw8PDg/79+/PKK6+c/7rc3FwGDx5MQUEBWmvefvvtP7327Nmzz18UbdiwIZ988kl1fmtCOIfY0ZB7HNa8CoeWGUsExI0F/3CzK7suMm3RwcgxFMJKWkPSaqOV3YHFoFyg5TDo+yp416r4601il9MWhRCiSikF0bca26lkI9g3TYXDa2HIFIi+xewKr5qMoQshRGAk9HkZxq8ETz+YOwSW/guK7WuxOwl0IYQ4p24bmLgW2k8wlgyYPRDyTphdldUk0IUQojwPbxjwFgyfC7/vhum3Qvoes6uyigS6EEJcSvPb4C+LwVJitLU7uNTsiiokgS6EEJdTty1MWAVB0fDZXfDdw5D7u9lVXZYE+kXKL587aNCgS663cj0iIyPJzMwEjJUdhRA2rmZdGLsYOj0EO+bDe+1g7ZtQdP1La1c2CfSLnLv1f/fu3dSqVYvJkyebXZIQwmwePsYsmIc3Q6MesPol+G9T+HIUbJ8DOWlmVwjY8jz0xU/B77sq9zXrtIR+r1m9e6dOndi5cycAiYmJPPTQQ2RkZODt7c306dNp2rQp6enpTJo0iaSkJACmTJlC586dGTJkCCkpKRQUFPDYY48xceLEyv1ehBDVr1ZDuGsuHNkA8fMgYSXs/c54rsP90Pc1Y8lek9huoJustLSUlStXMm7cOAAmTpzI1KlTiYmJYdOmTTz44IOsWrWKRx99lG7durFw4UJKS0vPN8eYOXMmtWrV4uzZs7Rv354777yToKAgM78lIURladDJ2LSGE/tgy3TY/BGUFsKAt00LddsN9Ks4k65M55bPTU5OJjY2ll69epGXl8f69esvWMq2sNBY0GfVqlXn28e5urri7+8PwHvvvcfChQsBSElJ4dChQxLoQjgapaB2cxjwP6MT0rr/GrNiBr1vSqjbbqCb5NwYek5ODgMHDmTy5MmMGTOGgICASy6reylr1qxhxYoVbNiwAW9vb7p3705BgX3dcSaEuApKwa3PgIsbrH3dWMExdqwxRONXx3i+Glj1T4hSqq9S6oBSKkEp9dQlnldKqffKnt+plGpX+aVWL39/f9577z3eeustatSoQVRUFF999RVgrIe+Y8cOAHr06MGUKVMAY5jm9OnT5OTkEBgYiLe3N/v372fjxo2mfR9CiGqiFNzyT7jlX7DrK5jVH/7XFF4Og+k9YPUrkLIZSquul0GFga6UcgUmA/2A5sAIpVTzi3brB8SUbROBKZVcpynatm1L69atmT9/PvPmzWPGjBm0bt2aG264ge++My6EvPvuu6xevZqWLVsSGxvLnj176Nu3LyUlJbRq1YpnnnmGG2+80eTvRAhRbbr9A/66G+5bCP3fMpbldXGFn9+EGb3gzWhY/0GVvLU1Qy4dgAStdRKAUmo+MBjYW26fwcAcbazFu1EpFaCUCtNaH6/0iqvYuYua53z//ffnP16yZMmf9q9du/b5cC9v8eLFl3z98q3jLn4vIYSDCKhvbNG3/vG5s6cgaY3RWKNmWJW8rTWBXg9IKfc4FehoxT71gAsCXSk1EeMMnoiIiKutVQgh7FeNQLjhdmOrItaMoV9qNP/irhjW7IPWeprWOk5rHRcSEmJNfUIIIaxkTaCnAvXLPQ4Hjl3DPlYxq4OSI5BjJ4RzsybQtwAxSqkopZQHcDew6KJ9FgGjyma73AjkXMv4uZeXF1lZWRJM10BrTVZWFl5eXmaXIoQwSYVj6FrrEqXUw8BSwBWYqbXeo5SaVPb8VOAnoD+QAJwBxl5LMeHh4aSmppKRkXEtX+70vLy8CA+37ya3QohrZ1NNooUQQlzZlZpEy2qLQgjhICTQhRDCQUigCyGEgzBtDF0plQEcucYvDwYyK7EcRyXHqWJyjComx8g61XWcGmitL3kjj2mBfj2UUlsvd1FA/EGOU8XkGFVMjpF1bOE4yZCLEEI4CAl0IYRwEPYa6NPMLsBOyHGqmByjiskxso7px8kux9CFEEL8mb2eoQshhLiIBLoQQjgImw50Z+xlerWsOEYjy47NTqXUeqVUazPqNFtFx6ncfu2VUqVKqaHVWZ8tsOYYKaW6K6XilVJ7lFJrq7tGs1nx++avlPpeKbWj7Bhd00KF10xrbZMbxsqOiUBDwAPYATS/aJ/+wGKMBhs3ApvMrtsGj1FnILDs437OdoysPU7l9luFsXroULPrtrVjBARgtJ6MKHscanbdNniM/gm8XvZxCHAS8KiuGm35DP18L1OtdRFwrpdpeed7mWqtNwIBSqmqadZnmyo8Rlrr9VrrU2UPN2I0H3E21vwsATwCfAOcqM7ibIQ1x+geYIHW+iiA1trZjpM1x0gDfkopBfhiBHpJdRVoy4F+uT6lV7uPI7va738cxl80zqbC46SUqgfcDkytxrpsiTU/S42BQKXUGqXUNqXUqGqrzjZYc4w+AJphdGzbBTymtbZUT3nWNYk2S6X1MnVgVn//SqlbMAL9piqtyDZZc5zeAZ7UWpcaJ1dOx5pj5AbEAj2AGsAGpdRGrfXBqi7ORlhzjPoA8cCtQDSwXCm1Tmt9uoprA2w70Ku1l6mdsur7V0q1Aj4G+mmts6qpNltizXGKA+aXhXkw0F8pVaK1/rZaKjSftb9vmVrrfCBfKfUz0BpwlkC35hiNBV7TxiB6glLqMNAU2FwdBdrykEu19TK1YxUeI6VUBLAAuM+JzqQuVuFx0lpHaa0jtdaRwNfAg04U5mDd79t3QFellJtSyhvoCOyr5jrNZM0xOorxFwxKqdpAEyCpugq02TN0XY29TO2VlcfoWSAI+LDs7LNEO9nKeVYeJ6dmzTHSWu9TSi0BdgIW4GOt9W7zqq5eVv4cvQjMUkrtwhiieVJrXW1LD8ut/0II4SBsechFCCHEVZBAF0IIByGBLoQQDkICXQghHIQEuhBCOAgJdCGEcBAS6EII4SD+H1ma1r9NGMERAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(df_scores_2['threshold'], df_scores_2['precision'], label='Precision')\n",
    "plt.plot(df_scores_2['threshold'], df_scores_2['recall'], label='Recall')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f56aa3d04c0>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAt0klEQVR4nO3dd3iUZb7/8fc3vfcESAIkVGmhhSaiqCtFUVBxNboH5SwiHtsWf8o2sezusbArirocZF270dVFWUVwERAVBYKE3kIPnXRIQtr9+2MGDCEJQ5InzyTzfV1Xrkx5ZubDJOQzT7tvMcaglFLKc3nZHUAppZS9tAiUUsrDaREopZSH0yJQSikPp0WglFIezsfuABcrJibGJCUl2R1DKaValLVr154wxsTWdl+LK4KkpCQyMjLsjqGUUi2KiOyr6z7dNKSUUh5Oi0AppTycFoFSSnm4FrePQClVv/LycrKzsyktLbU7irJBQEAAiYmJ+Pr6uvwYLQKlWpns7GxCQ0NJSkpCROyOo5qRMYacnByys7NJTk52+XGWbRoSkddE5JiIbKrjfhGRF0UkS0Q2iMgAq7Io5UlKS0uJjo7WEvBAIkJ0dPRFrw1auY/gdWBMPfePBbo6v6YCf7Mwi1IeRUvAczXkZ2/ZpiFjzAoRSapnkfHAm8YxDvb3IhIhIu2MMYetyLPjaBGfrj9ESIAPIf6+hAT4EOrvQ7C/DyH+PoQGOL4H+/vg56P70JVSnsPOfQQJwIFq17Odt51XBCIyFcdaAx06dGjQi+04WsSLS7NcWjbA14uwAF/CAn0JC/BxfvclPNCXsEAfwgJ8iQzyIzzI8T0iyNfxFeinJaIU4O3tTZ8+faioqKBHjx688cYbBAUFNeo5H3vsMS6//HJ+8pOf1Hr/nDlzCAoKYtKkSY16neTkZBYtWkT37t3P3vaLX/yC+Ph4HnnkkQs+PikpidDQUESEyMhI3nzzTTp27NioTNWNHDmSmTNnkpqaevYE25iYmEY9p51FUNv6S62z5Bhj5gJzAVJTUxs0k864lHiu7d2O4vJKTpZWcPJ0OUWlFZw6XVntcgVFpRWcPF1BYWk5BSXlFJZUkHuqjL0nTlFYWkFBSTmVVXVHCA/0pWtcCN3ahtK9TSjd2oTSvW0oUcF+DYmtVIsUGBhIZmYmAHfccQdz5szhV7/61dn7Kysr8fb2vqjnfPLJJ+u9f9q0aRedsza33XYb6enpzJgxA4Cqqio+/PBDvv32W5efY9myZcTExDBjxgz++Mc/8uqrrzZJNqvYWQTZQPtq1xOBQ1a+oJeXEOLcFAQBDXoOYwynyirJLy4jv7icPOf3M9ePFJay8+hJPttwmHdL9p99XNuwAEZ0jeHKS+K4rGsMYQGuH9qlVEs2YsQINmzYwPLly3niiSdo164dmZmZbNy4kenTp7N8+XJOnz7Nfffdxz333APAs88+y1tvvYWXlxdjx47l6aef5q677mLcuHFMnDiR6dOns2DBAnx8fBg1ahQzZ87k8ccfJyQkhIcffpjMzEymTZtGcXExnTt35rXXXiMyMpKRI0cyZMgQli1bRn5+Pn//+98ZMWLEOXnT0tK49dZbzxbBihUrSEpKomPHjmzevJnJkydTVlZGVVUVH330EV27dq3z3z5s2DBefPFFAI4fP860adPYv9/xd2HWrFkMHz6ckydP8sADD5CRkYGIMGPGDG6++Wbuvfde1qxZQ0lJCRMnTuSJJ56w4scD2FsEC4D7RSQdGAIUWLV/oCmJ/FgmiZF1L2eM4VjRabYfKWLH0SLWHchn8eYj/HNtNt5ewrBO0dzQL54xvdtqKSjLPPHvzWw5VNikz9kzPowZ1/dyadmKigo+//xzxoxxHDeyevVqNm3aRHJyMnPnziU8PJw1a9Zw+vRphg8fzqhRo9i2bRsff/wxq1atIigoiNzc3HOeMzc3l/nz57Nt2zZEhPz8/PNed9KkScyePZsrrriCxx57jCeeeIJZs2adzbR69WoWLlzIE088wZIlS855bEpKCl5eXqxfv56+ffuSnp5OWloa4Nj89NBDD3HHHXdQVlZGZWVlvf/+RYsWMWHCBAAeeughfvnLX3LZZZexf/9+Ro8ezdatW3nqqacIDw9n48aNAOTl5QHwpz/9iaioKCorK7n66qvZsGEDKSkpLr3vF8uyIhCR94CRQIyIZAMzAF8AY8wcYCFwLZAFFAOTrcpiBxGhTVgAbcICuLybY8C/isoq1h3IZ+m2Y3y24TCPfLiB33+8iT+M68l/DW26bYhK2a2kpIR+/foBjjWCn//856xcuZLBgwefPb79iy++YMOGDXz44YcAFBQUsHPnTpYsWcLkyZPP7lOIioo657nDwsIICAhgypQpXHfddYwbN+6c+wsKCsjPz+eKK64A4M477+SWW245e/9NN90EwMCBA9m7d2+t+dPS0khPT6dXr1588sknZzdLDRs2jD/96U9kZ2dz00031bk2cOWVV3L06FHi4uL44x//CMCSJUvYsmXL2WUKCwspKipiyZIlpKenn709MtLxCfODDz5g7ty5VFRUcPjwYbZs2dLyisAYk3aB+w1wn1Wv7458vL0YlBTFoKQoHhndncwD+cz8YjtPfbqFYZ2i6BIXandE1cq4+sm9qVXfR1BdcHDw2cvGGGbPns3o0aPPWWbRokX1HgLp4+PD6tWr+fLLL0lPT+ell15i6dKlLmfz9/cHHDu0Kyoqal0mLS2NUaNGccUVV5CSkkJcXBwAt99+O0OGDOGzzz5j9OjRzJs3j6uuuuq8xy9btozg4GDuuusuHnvsMf76179SVVXFd999R2Bg4DnLGmPO+/fu2bOHmTNnsmbNGiIjI7nrrrssPVNcD3GxiYjQv0Mks27tT5CfN498uIGdR4soLqv9F1Op1mb06NH87W9/o7y8HIAdO3Zw6tQpRo0axWuvvUZxcTHAeZuGTp48SUFBAddeey2zZs06r3DCw8OJjIzk66+/BuCtt946u3bgqs6dOxMdHc306dPPbhYC2L17N506deLBBx/khhtuYMOGDXU+R2BgILNmzeLNN98kNzeXUaNG8dJLL529/0zumrfn5eVRWFhIcHAw4eHhHD16lM8///yi8l8sHWLCZrGh/jx+fS9+8X4m1zy/AoDoYD8So4Lo2S6UvokRpCRG0K1NCD7e2tuq9ZgyZQp79+5lwIABGGOIjY3l448/ZsyYMWRmZpKamoqfnx/XXnstf/7zn88+rqioiPHjx1NaWooxhueff/68537jjTfO7izu1KkT//jHPy46X1paGr/5zW+48cYbz972/vvv8/bbb+Pr60vbtm157LHH6n2Odu3akZaWxssvv8yLL77IfffdR0pKChUVFVx++eXMmTOH3//+99x333307t0bb29vZsyYwU033UT//v3p1asXnTp1Yvjw4Red/2KIYwtNy5Gammpa48Q0Ww4VsvNYEdl5JWTnlbAv5xSbDhZQWOpYQwjw9aJPQjgTByYyvl8CAb4Xd+id8hxbt26lR48edsdQNqrtd0BE1hpjUmtbXtcI3ETP+DB6xoedc5sxhn05xazPzmf9gQJW7jrBox9t5LnFO7jr0o5clxJPu/AALQWlVKNoEbgxESEpJpikmGDG90vAGMPKXTnMXbGbmV/sYOYXOwDHSWxtwwJoEx5AiP+PpeAlwpjebbm2dzu8vHTsGaVU7bQIWhARYXiXGIZ3iWHn0SIyD+RztLCUo4WnOVJYytHCUg7n/3hcc2FpOZ9uOEzXuJ3cf1UXxqXE462F4BFqOxJFeYaGbO7XImihurYJpWub+g83rawyLNx4mNlLd/JQeiYvLHEUwg1943XHcysWEBBATk6ODkXtgc7MRxAQcHEjJ+jOYg9QVWVYvPkIL3y5k21HikjtGMncSak6/lErpTOUeba6Ziirb2exFoEHqaoyzF93kN/M30h8eAD/mDyY5JjgCz9QKdXi1VcEun3Ag3h5CTcPTOS9u4dQWFrBTa98y5q9uRd+oFKqVdMi8EADO0Yx/38uJTLIjzteXcXvP97I/pxiu2MppWyiReChOkYH89G9l3LzwAQ+WJPNyJnLeOC9dew9ccruaEqpZqb7CBRHC0t57Zs9vLPKMU76cxNTGNunnc2plFJNSXcWK5cczC/hf975gfUH8rlzWEdG9WpLQkQg7SIC8PfRs5eVasm0CJTLyiqq+PPCrby+cu85t8eF+pOSGMFlXaIZ3iWGLnEheoy6Ui2IFoG6aIfyS9iXU8zB/BIO5pWwP7eYjH257HPuVG4T5s/MW/oyomuszUmVUq7QQefURYuPCCQ+IvC82w/kFrNy1wle/XoPD7y3joUPjqh1OaVUy6FHDamL0j4qiFsHdeDVSamUV1TxwHvrKK+ssjuWUqoRtAhUgyTHBPO/N6ewdl8ez3y+ze44SqlG0E1DqsFu6BvP2r25zPtmD13bhHDroA52R1JKNYCuEahG+cO4nozoGsPv5m9iZdYJu+MopRpAi0A1io+3Fy/fMYDkmGAmvbaaB99bx7r9eXbHUkpdBC0C1WhhAb68e/dQ7rw0iWXbjnHjKyuZ8PK3ZB7ItzuaUsoFWgSqScSG+vOHcT357rdX8/j1PTlWWModr37P97tz7I6mlLoALQLVpEL8fbhreDLz7xtOu4hA7vrHar7acdzuWEqpemgRKEu0CQvg/alDSY4J4e43MvjPlqN2R1JK1UGLQFkmOsSf9LuH0iM+jHvfXssbK/c2aGJtpZS1tAiUpcKDfHn754O5vFssMxZs5pfvZ1JaXml3LKVUNVoEynKhAb7Mm5TKr6/pxifrD3H3mxlaBkq5ES0C1Sy8vIQHru7KMzen8E3WCaa8oWWglLvQIlDN6qep7XluYl++yTrBnK922R1HKYXFRSAiY0Rku4hkicj0Wu4PF5F/i8h6EdksIpOtzKPcw8SBiYzu1YZ5X+8h71SZ3XGU8niWFYGIeAMvA2OBnkCaiPSssdh9wBZjTF9gJPAXEfGzKpNyH78e1Z1TZRXMWaFrBUrZzco1gsFAljFmtzGmDEgHxtdYxgCh4pjzMATIBSoszKTcRLc2oUzol8AbK/eSnVdsdxylPJqVRZAAHKh2Pdt5W3UvAT2AQ8BG4CFjzHmznIjIVBHJEJGM48f1LNXW4lfXdMPHy4sH3ltHWYVObqOUXawsgtpmNq95NtFoIBOIB/oBL4lI2HkPMmauMSbVGJMaG6tz5LYW7aOCePrmPqzbn8+fF27l5GldGVTKDlZOTJMNtK92PRHHJ//qJgNPG8fpplkisge4BFhtYS7lRsalxLNmTy6vr9zL6yv3Eh3sR4foIAYnR/Ho6Evw8qrt84RSqilZWQRrgK4ikgwcBG4Dbq+xzH7gauBrEWkDdAd2W5hJuaE/jOvJpV1i2HX8JAdyi8k6dpL/+2o3bUID+O/Lku2Op1SrZ1kRGGMqROR+YDHgDbxmjNksItOc988BngJeF5GNODYlPWqM0WmuPIyPtxeje7U9e90Yw8/fyODZxdu48pI4kmOCbUynVOsnLW0QsNTUVJORkWF3DGWxIwWljHr+K7q1CeX9e4bhrZuIlGoUEVlrjEmt7T49s1i5pbbhAcy4vhcZ+/J48cudVFW1rA8sSrUkWgTKbd00IIHrUtrxwpc7mfDKt6zdl2t3JKVaJS0C5bZEhJfS+jPr1n4cKzzNzX/7jgffW0dxmR5mqlRT0iJQbk1EmNA/gaUPX8H9V3ZhwfpDfLDmwIUfqJRymRaBahGC/Hx4eHR32kcFsnJXjt1xlGpVtAhUi3Jppxi+351Dpe48VqrJaBGoFuXSLtEUllaQeSDP7ihKtRpaBKpFGdk9jsggX55dtJ2Wdg6MUu5Ki0C1KOGBvvx6VHdW7cklXXcaK9UktAhUi5M2uAPDu0Tz2/kbeWV5FhWVOoS1Uo2hRaBaHG8v4e93DmJs77Y8u2g741/+lnlf72b38ZN2R1OqRdKxhlSLZYzh3xsO8+KXO8k65iiBpOggbkltz5QRyfj7eNucUCn3Ud9YQ1oEqlU4kFvMsu3HWLz5CN9m5dApJpjXJw+mQ3SQ3dGUcgs66Jxq9dpHBTFpWBLvTBnK65MHcbiglBeX7rQ7llItghaBanVGdo/j5oEJLMg8xImTp+2Oo5Tb0yJQrdJdlyZTVlnFox9uoKi03O44Srk1LQLVKnWJC+HJ8b1YvuM4Y1/4ms83HtYT0JSqgxaBarUmDUvi/alDCfH34d53fuC2ud+z5VCh3bGUcjtaBKpVS02K4tMHLuOPE3qz42gR42Z/ze/mb9ST0JSqRotAtXo+3l78bGhHlj98JWmDO/DOqv0s337c7lhKuQ0tAuUxwoN8+e21PQDYelg3ESl1hhaB8ijB/j50jA5i6xEtAqXO0CJQHqdH2zA2HSykUA8rVQoAH7sDKNXc+neIYNHmI/R74gv6to9geOcYhneJYUDHCB2fSHkkHWtIeZyqKsPqvbmszDrBN1knWJ9dQGWVIcDXiyu7x3HflV3onRBud0ylmpQOOqdUPQpLy1m1O5dvdh5n/rqDFJZW8JMebfj1qG70aBdmdzylmoQWgVIuKiwt5/Vv9zLnq11EBfvxzaNX2R1JqSaho48q5aKwAF+mXt4JLxH8fbz4asdxHatItXq6s1ipWvRJCGf13lzufG01XgI948O4/8oujOndzu5oSjU5LQKlagjw9ea9qUM5dbqCdfvzWb0nhxeXZrF02zEtAtUq6aYhpeoQ7O/DZV1jGNvH8cd/UFKUzYmUsoalRSAiY0Rku4hkicj0OpYZKSKZIrJZRL6yMo9SDfHdrhwAhnWOtjmJUtawrAhExBt4GRgL9ATSRKRnjWUigFeAG4wxvYBbrMqjVEOtz85HBN5dtZ/svGK74yjV5KzcRzAYyDLG7AYQkXRgPLCl2jK3A/8yxuwHMMYcszCPUg0y9fJOnDpdyZyvdjHnq11cdUkcPxvakcu7xuLlJXbHU6rRrCyCBOBAtevZwJAay3QDfEVkORAKvGCMebPmE4nIVGAqQIcOHSwJq1RdesWHM+/OVA7ml/Deqv2kr9nPkq3H6BAVxM+GdiBtcAdCA3ztjqlUg1m5j6C2j0o1z17zAQYC1wGjgT+ISLfzHmTMXGNMqjEmNTY2tumTKuWChIhAHh7dnZXTr2Z2Wn/ahQfw54XbuOyZZcxasoP84jK7IyrVIFauEWQD7atdTwQO1bLMCWPMKeCUiKwA+gI7LMylVKP4+Xhxfd94ru8bz4bsfF5amsWsJTuZ9/Uefja0I1NGJBMT4m93TKVcZuUawRqgq4gki4gfcBuwoMYynwAjRMRHRIJwbDraamEmpZpUSmIEcyelsugXI7jykjj+b8UuLntmKat259gdTSmXWVYExpgK4H5gMY4/7h8YYzaLyDQRmeZcZiuwCNgArAbmGWM2WZVJKatc0jaM2Wn9WfKrKwgP9OWlZVl2R1LKZZaeWWyMWQgsrHHbnBrXnwOeszKHUs2lc2wIPxvSkb/8ZwdZx07SJS7E7khKXZAOMaFUE0sb0oHZS7OY9vZa+iZGEB8RQHxEIN3bhjKgQ6Td8ZQ6jxaBUk0sJsSfJ8b34sO12Xy36wRHCkupch4v9+zNKfx0UPv6n0CpZlZvEYhIEecf8gmOQ0ONMUZn7VCqFmmDHecXAFRUVnG06DSPfLie33+yiU6xwQzsGImInoym3INOTKNUM8k5eZrrZ3/DoYJS2oYFkJoUSWrHSFKToujRLgxvPUtZWai+iWkutEZQ73CLxpjcxgRTypNEh/jz8X3DWbz5CGv25rF2Xx6fbjgMQIi/D6lJkfzpxj4kRATanFR5mnrXCERkD45NQ7WeJWyM6WRVsLroGoFqTQ7ml5CxN5eMvXm8n3GA2wd34PEbetkdS7VCDV4jMMYkWxNJKQWOYSsS+iUwvl8CJ06e5u3v95GdV8z4fgn8pEcbAv287Y6oPIDLRw2JSCTQFQg4c5sxZoUVoZTyRE+O701iZCAL1h9iydZjBPt5c11KO2Zc34tgfz3AT1nHpTOLRWQKsALHWcJPOL8/bl0spTxPbKg/v7uuJyunX827U4bQtU0oH2Rkc6Sw1O5oqpVzdYiJh4BBwD5jzJVAf+C4ZamU8mDeXsKwztGUlFWSkhhO51g9O1lZy9UiKDXGlAKIiL8xZhvQ3bpYSnm2H/bnsf1oEbcP1vk3lPVc3fCY7ZxW8mPgPyKSx/lDSiulmsi7qw7g4yWM6KbzbyjruVQExpgbnRcfF5FlQDiOUUOVUhY4XFBCRZXhsmeW0r1NKEM7RTNxYCK9E8LtjqZaIZfOLBaRocBmY0yR83oo0NMYs8rifOfR8wiUJzhdUcn6AwWs2p3D93tyWLsvD18vLxY+NIL2UUF2x1MtUH3nEbhaBOuAAca5sIh4ARnGmAFNmtQFWgTKEx3ILWbsC19zSdtQ0qcOxcfbyjmlVGtUXxG4+tskplpjGGOq0JFLlWo27aOCeGpCLzL25fHK8l12x1GtjKtFsFtEHhQRX+fXQ8BuK4Mppc51Y/9ExveL54Uvd/LD/jy746hWxNUimAZcChzEMeH8EGCqVaGUUrV7akJv2oYF8FD6Og7kFtsdR7USLhWBMeaYMeY2Y0ycMaaNMeZ2Y8wxq8Mppc4VFuDLS7f3p6C4nHGzv2HZdv1vqBrP1SEmuonIlyKyyXk9RUR+b200pVRt+neIZMH9l9EuPIDJ/1jDuNlf89b3+ygsLbc7mmqhXD1q6Cvg/wH/Z4zp77xtkzGmt8X5zqNHDSnlUFJWyQcZB0hfc4CthwsJ8PXiuj7x3Da4Pak6A5qqocHDUFcTZIxZXeMXq6LRyZRSDRbo582dlyYxaVhHNh4sIH3NARZkHuKjH7LpFBvMbYPac/OARKJD/O2Oqtycq0VwQkQ645y/WEQmAoctS6WUcpmIkJIYQUpiBL+/rgefbTjM+2sO8OeF25j5xQ5uG9See67orDOfqTq5ummoEzAXx5FDecAe4A5jzD5r451PNw0p5ZqdR4v4+zd7+OiHbIyBG/sncO/IznTS0Uw9UqPPLK72RME4djCXALcaY95pmoiu0yJQ6uIcyi9h7ordvLd6P+WVVVyXEs+fb+xNaICv3dFUM2rwmcUiEiYivxGRl0TkGqAYuBPIAn7a9FGVUk0tPiKQx2/oxTePXsXYPu349/pD7DhaZHcs5UYutI/gLRybgr4D7gYeAfyACcaYTGujKaWaUmyoP+GBvgT7edMnIcLuOMqNXKgIOhlj+gCIyDzgBNDhzCikSqmWwxjD8m3HuKxrDH4+Omid+tGFfhvOnqFijKkE9mgJKNUy7Th6kkMFpVzZPc7uKMrNXGiNoK+IFDovCxDovC6AMcaEWZpOKdVk9pw4CUDnOD1qSJ2r3iIwxng3VxCllLUGJUUhAiuzchiUFGV3HOVGLN1QKCJjRGS7iGSJyPR6lhskIpXOE9WUUhaIDvGnb2IES3WgOlWDZUUgIt7Ay8BYoCeQJiI961juGWCxVVmUUg5XXRLHhux8Tpw8bXcU5UasXCMYDGQZY3YbY8qAdGB8Lcs9AHwE6McUpSx21SVxGAPLtx+3O4pyI1YWQQJwoNr1bOdtZ4lIAnAjMKe+JxKRqSKSISIZx4/rL7BSDdUrPoy4UH+dx0Cdw8oiqG0M3JrjWcwCHnUemlonY8xcY0yqMSY1Nja2qfIp5XFEhJHdY1mx4ziHC0rsjqPchJVFkA20r3Y9EThUY5lUIF1E9gITgVdEZIKFmZTyeGmDO1BZZRj7wtcs3nzE7jjKDVhZBGuAriKSLCJ+wG3AguoLGGOSjTFJxpgk4EPgf4wxH1uYSSmP179DJJ89OIL2kUHc89Zafjd/I6Xl9a6Uq1bOsiIwxlQA9+M4Gmgr8IExZrOITBORaVa9rlLqwpJjgvno3ku55/JOvLNqP+Nf+pasYzpogKe6qGGo3YEOQ61U01qx4zi/fD+T4rJK/jihNzcPTLQ7krJAg4ehVkq1fpd3i2XhQyNISQzn1/9cz/3v/kDuqTK7Y6lmpEWglKJNWADvTBnCr6/pxuLNRxj1/Fcs2qSz0XoKLQKlFAA+3l48cHVXFtx/GW3CApj29g888N46XTvwAFoESqlz9GgXxsf3DedX13Rj0abDunbgAbQIlFLn8fX24sEaawevrthtdyxlES0CpVSdzqwdpHaM5IOMAxd+gGqRtAiUUvXy9fZiTO+27Dx2koP5OixFa6RFoJS6oJHdHWN8LdfB6lolLQKl1AV1jg0hISKQhRsP09JOQlUXpkWglLogEeGuS5P4NiuHV5bvsjuOamJaBEopl0wZkcz4fvE8t3g7izbpqKWtiRaBUsolIsIzN6fQr30Ev3w/k5VZJ+yOpJqIFoFSymUBvt7MnTSQ+IgA7vj7Kv7yxXYqKqvsjqUaSYtAKXVR4kID+PcDl3HLwERmL83i1rnfk51XbHcs1QhaBEqpixbk58OzE/vywm392H6kiClv6NDwLZkWgVKqwcb3SyAhIpC4sAC7o6hG0CJQSjVYQXE5248WMahjpN1RVCNoESilGixjXy4Ag5KjbE6iGkOLQCnVYKv35uLrLfRrH2F3FNUIWgRKqQbbdewknWNDCPD1tjuKagQtAqVUg5VVGvy1BFo8LQKlVINVVFbh6yV2x1CNpEWglGqwiiqDj7cWQUunRaCUarCKyip8vPTPSEunP0GlVIOVlFfpGkEroEWglGqQFTuOs/VwIYOS9ByClk6LQCl10UrLK/nDJ5voFBPMlBHJdsdRjeRjdwClVMszd8Vu9uUU8+6UIfj76OGjLZ2uESilLlqVc97i7UeLbE6imoKuESilLtoDV3Vl6+FCnvx0C/ERgYzu1dbuSKoRLF0jEJExIrJdRLJEZHot998hIhucXytFpK+VeZRSTcPbS5h1a3/6JkbwUPo61u3PszuSagTLikBEvIGXgbFATyBNRHrWWGwPcIUxJgV4CphrVR6lVNMK9PNm3p2pxIUGMOWNDPblnLI7kmogK9cIBgNZxpjdxpgyIB0YX30BY8xKY8yZjxLfA4kW5lFKNbGYEH9enzyISmOY/I815J0qszuSagAriyABOFDterbztrr8HPjcwjxKKQt0ig1h3qRUsvNLuPvNDHK1DFocK4ugttMNTa0LilyJowgereP+qSKSISIZx48fb8KISqmmkJoUxfM/7UfmgXyu+sty3l21n6qqWv+7KzdkZRFkA+2rXU8EDtVcSERSgHnAeGNMTm1PZIyZa4xJNcakxsbGWhJWKdU416W047MHR9C9TSi/nb+RG/+2kg3Z+XbHUi6wsgjWAF1FJFlE/IDbgAXVFxCRDsC/gP8yxuywMItSqhl0bxtK+tShzLq1HwfzShj/8rf8bv5G8ot1c5E7s+w8AmNMhYjcDywGvIHXjDGbRWSa8/45wGNANPCKiABUGGNSrcqklLKeiDChfwJX9Yjj+f/s4I2Ve/l80xGmj72EiQMS8dL5C9yOGNOytuOlpqaajIwMu2MopVy05VAhj32yiYx9eQzoEMFTE3rTKz7c7lgeR0TW1vVBW4eYUEpZqmd8GB/cM4yZt/RlX04x18/+hscXbKagpNzuaMpJh5hQSlnOy0uYODCRa3q04S//2c6b3+3l0w2HuH1wB24ckEhyTLDdET2abhpSSjW7TQcLeHbxdr7eeRxjYGDHSG4akMC4PvGEB/naHa9Vqm/TkBaBUso2RwpK+TjzIB+tzWbnsZP4+XhxTY823DQggcu7xeLrrVuvm4oWgVLKrRlj2HSwkI9+yGbB+kPkniojJsSPR8dcwi2p7S/8BOqCdGexUsqtiQh9EsN5/IZerPrt1bw6KRV/H2/+mZFtdzSPoEWglHIrvt5eXNOzDf3aR3Di1Gm743gELQKllFuKDvEj56SekdwctAiUUm4pOtifgpJyyiur7I7S6mkRKKXcUnSIH4DOcdAMtAiUUm4p52QZIujYRM1Ai0Ap5XaMMfx7wyEGJUURE+Jvd5xWT4tAKeV2thwuJOvYSa7vG293FI+gRaCUcitbDhVy9xsZBPt5c23vtnbH8QhaBEopt/GfLUeZOGclVQbev2cY0bpZqFno6KNKKdsZY5i7YjdPL9pGn4RwXp2USpuwALtjeQwtAqWUrSoqq/jNvzbyz7XZXJfSjpkT+xLo5213LI+im4aUUrbafrSIf67NxsdL6NkujNMVlXZH8jhaBEopW/WKD+f1yYMY2ima5xZvZ9j/LuWDjAN2x/IoWgRKKduN7B7H21OG8PlDIwj29+ar7cftjuRRtAiUUm6jR7swIoL8MLSseVJaOt1ZrJRyK/4+XizceITRz69gcHIUg5OjGJIcRZweRWQZLQKllFt5Ma0/izYdYdWeXP71QzZvfb8PgOSYYAYnRTGkk6McEiODbE7aeuhUlUopt1VRWcXmQ4Ws3pPLqj05rN6TS2FpBQAJEYEMObPG0CmapOggRHSAurronMVKqVahqsqw7UgRq/fksGpPLqv35JLjHKa6bVgAw7vEMLxLNMO7xOgJaTVoESilWiVjDLuOn+L73Tms3HWClbtyyC8uB6BLXAjDOztKYUhyNGGBPh69xqBFoJTyCFVVhi2HC/k26wTf7sph9Z4cSssdM5z5egsh/j4E+/sQ4u9DaECNy34+hAQ4rof413052N8HX++Wd8ClFoFSyiOdrqhk3f581u3Pp7C0nJOlFZw87fxyXj51uoIi5/WSctfOao4M8iU+IpCEiEASIh3fEyMDSYgIIiEykMggX7db+6ivCPSoIaVUq+Xv483QTtEM7RTt0vIVlVWcKqvklLMsikorzl4+UxxFpRUcKyrlYH4Je06c4pusExSXnVsggb7eZwsi/mxJ/FgabcIC8Hajmde0CJRSysnH24vwQC/CA31dfowxhvzicg7ml5CdV8LB/BIO5pVwKN9xeePBAnJrzLvs4yW0DQ84Ww6JzsKoXh4Bvs038J4WgVJKNYKIEBnsR2SwH70TwmtdprisgkN1FMX3u3I4UlhKVY2t9DEh/mdLIiHSsVZxXZ92lszRYGkRiMgY4AXAG5hnjHm6xv3ivP9aoBi4yxjzg5WZlFKquQX5+dAlLpQucaG13l9eWcWRgtKzJXG2LApK2Hq4kCVbj3K6oooTRaf51ajuTZ7PsiIQEW/gZeAaIBtYIyILjDFbqi02Fujq/BoC/M35XSmlPIavtxfto4JoH1X72dLGGJJ/s5CySmsO7rHyGKjBQJYxZrcxpgxIB8bXWGY88KZx+B6IEJF2FmZSSqkWR0S4oW88l7StfY2isazcNJQAVB9UPJvzP+3XtkwCcLj6QiIyFZgK0KFDhyYPqpRS7u7FtP6WPbeVawS1HRtVc73GlWUwxsw1xqQaY1JjY2ObJJxSSikHK4sgG2hf7XoicKgByyillLKQlUWwBugqIski4gfcBiyoscwCYJI4DAUKjDGHaz6RUkop61i2j8AYUyEi9wOLcRw++poxZrOITHPePwdYiOPQ0Swch49OtiqPUkqp2ll6HoExZiGOP/bVb5tT7bIB7rMyg1JKqfq1vCH0lFJKNSktAqWU8nBaBEop5eFa3HwEInIc2NcMLxUDnGiG12kszdm0NGfT0pxNqzE5Oxpjaj0Rq8UVQXMRkYy6JnFwJ5qzaWnOpqU5m5ZVOXXTkFJKeTgtAqWU8nBaBHWba3cAF2nOpqU5m5bmbFqW5NR9BEop5eF0jUAppTycFoFSSnk4jy8CERkjIttFJEtEptdy/3gR2SAimSKSISKXuWPOassNEpFKEZnYnPmqvf6F3s+RIlLgfD8zReQxd8zpXGakM+NmEfmquTM6M1zo/fx/1d7LTc6ffZQb5gwXkX+LyHrn+9nsA0y6kDFSROY7/7+vFpHezZ3RmeM1ETkmIpvquF9E5EXnv2ODiAxo9IsaYzz2C8eoqLuAToAfsB7oWWOZEH7cl5ICbHPHnNWWW4pjoL+J7pgTGAl82gJ+7hHAFqCD83qcO+assfz1wFJ3zAn8FnjGeTkWyAX83Czjc8AM5+VLgC+b+710vvblwABgUx33Xwt8jmNir6HAqsa+pqevEVxwXmVjzEnjfPeBYGqZQa0ZuDL/M8ADwEfAseYMV42rOe3mSs7bgX8ZY/YDGGPseE8v9v1MA95rlmTnciWnAUJFRHB8uMoFKtwsY0/gSwBjzDYgSUTaNGNGnK+9Asf7U5cmn+vd04ugrjmTzyEiN4rINuAz4L+bKVt1F8wpIgnAjcAc7OPS+wkMc24i+FxEejVPtHO4krMbECkiy0VkrYhMarZ0P3L1/UREgoAxOD4INDdXcr4E9MAxA+FG4CFjTFXzxANcy7geuAlARAYDHXHMmuhuXP69cJWnF4GrcybPN8ZcAkwAnrI6VC1cyTkLeNQYU2l9nDq5kvMHHGOe9AVmAx9bHaoWruT0AQYC1wGjgT+ISDerg9Xg0u+n0/XAt8aY+j5JWsWVnKOBTCAe6Ae8JCJh1sY6hysZn8ZR/pk41q7X0bxrLa66mN8Ll1g6MU0LcFFzJhtjVohIZxGJMcY05wBVruRMBdIda97EANeKSIUx5uNmSehwwZzGmMJqlxeKyCtu+n5mAyeMMaeAUyKyAugL7GieiGczuPr7eRv2bBYC13JOBp52bmbNEpE9OLbDr26eiC7/bk4Gxw5ZYI/zy900/VzvduwMcZcvHEW4G0jmxx1IvWos04UfdxYPAA6eue5OOWss/zr27Cx25f1sW+39HAzsd8f3E8dmjC+dywYBm4De7pbTuVw4jm3Kwc39M7+I9/NvwOPOy22c/49i3CxjBM4d2MDdOLbDN/v76Xz9JOreWXwd5+4sXt3Y1/PoNQLj2rzKNwOTRKQcKAFuNc6fhpvltJ2LOScC94pIBY738zZ3fD+NMVtFZBGwAagC5hljaj2cz86czkVvBL4wjrWXZudizqeA10VkI44/YI+aZlwLdDFjD+BNEanEccTYz5srX3Ui8h6Oo+tiRCQbmAH4VsvZ5HO96xATSinl4Tx9Z7FSSnk8LQKllPJwWgRKKeXhtAiUUsrDaREopZSH0yJQHsk5SueZETv/6RyiobHP+aSI/KSe+6fZNFSFUvXSw0eVRxKRk8aYEOfld4C1xpi/Vrvf29g7XIdSzUbXCJSCr4EuzvkHlonIu8BGEfEWkedEZI1z3Pd7zjxARB4RkY3OwfOedt72ujjngRCRp0Vki/NxM523PS4iDzsv9xOR7533zxeRSOfty0XkGed4+DtEZERzvxnK83j0mcVKiYgPMBZY5LxpMI6hJPaIyFSgwBgzSET8gW9F5AscY+RMAIYYY4qlxkQwzus3ApcYY4yIRNTy0m8CDxhjvhKRJ3GcPfoL530+xpjBInKt8/Y6Nzcp1RR0jUB5qkDnKJMZOMY7+rvz9tXGmDMDjY3CMbxIJrAKiAa64vjD/A9jTDGAOX/Ez0KgFJgnIjfhGAbgLBEJByKMMWdmPXsDx2QkZ/zL+X0tjjFnlLKUrhEoT1VijOlX/QbnyK3Vx+sRHJ/aF9dYbgz1DPvrHNdmMHA1jlFB7weuuohsp53fK9H/o6oZ6BqBUnVbjGOAPF8AEekmIsHAF8B/nznSqJZNQyFAuDFmIY7NPf2q32+MKQDyqm3//y/AljmRlQL9tKFUfebh2DTzg3N8+uPABGPMIhHpB2SISBmO0SB/W+1xocAnIhKAY63il7U8953AHGeZ7KYJRpBUqqH08FGllPJwumlIKaU8nBaBUkp5OC0CpZTycFoESinl4bQIlFLKw2kRKKWUh9MiUEopD/f/ASJsqg1yvPCmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plot of Precision (x-axis) Vs Recall (y-axis)\n",
    "plt.plot(df_scores_2['precision'], df_scores_2['recall'], label='Precision Vs Recall')\n",
    "plt.xlabel('Precision')\n",
    "plt.ylabel('Recall')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f56a9853850>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAu9UlEQVR4nO3deXhV5bX48e/KRAYyQhKGBMIsAcIUUEEFHABHxOFKrKKoVZyq3quVTli1vbXWtqhoEZUq+rtF62xVVBAUFWSQeY5EICAkYchEQqb1++McYwghOYHsnJyc9XmePDl773fvs16DZ52930lUFWOMMf4rwNsBGGOM8S5LBMYY4+csERhjjJ+zRGCMMX7OEoExxvi5IG8H0Fjt27fXlJQUb4dhjDE+ZdWqVXmqGl/XMZ9LBCkpKaxcudLbYRhjjE8RkZ0nOmaPhowxxs9ZIjDGGD9nicAYY/ycz7URGGPqV15eTnZ2NqWlpd4OxXhBaGgoSUlJBAcHe3yOJQJjWpns7GwiIyNJSUlBRLwdjmlGqsqBAwfIzs6mW7duHp/n2KMhEZkjIjkisuEEx0VEnhKRTBFZJyJDnIrFGH9SWlpKu3btLAn4IRGhXbt2jb4bdLKN4CVgfD3HLwR6uX9uBf7hYCzG+BVLAv7rZP72jj0aUtUvRCSlniITgLnqmgd7mYjEiEhHVf3BiXi27ivkg3V7XRsiTBzcmW7tI5x4K2OM8Sne7DXUGdhdYzvbve84InKriKwUkZW5ubkn9WaZOUU8vSiTpxdl8tTC7fz1k60ndR1jTMMCAwMZNGgQ/fv35+qrr+bIkSOnfM3p06ezYMGCEx6fNWsWc+fOPeX36datG1u3Hvv5cO+99/L44497dH5KSgoDBgwgLS2NUaNGsXPnCcdxnZTRo0dXD6pNSUkhLy/vlK/pzURQ1/1LnavkqOpsVU1X1fT4+DpHSDfo4rSOZP3pYrL+dDFXDU1iyfY8KiqrTupaxpj6hYWFsWbNGjZs2EBISAizZs065nhlZWWjr/nII49w/vnnn/D41KlTmTx5cqOvW9ukSZOYN29e9XZVVRVvvPEG11xzjcfXWLRoEevWrWP06NH84Q9/OOWYnObNRJANJNfYTgL2Nscbj+4TT35JOWuzDzfH2xnj184++2wyMzNZvHgxY8aM4dprr2XAgAFUVlbywAMPMGzYMNLS0njuueeqz3n88ccZMGAAAwcOZNq0aQDceOONvPHGGwBMmzaN1NRU0tLSuP/++wH4/e9/zxNPPAHAmjVrOOOMM0hLS2PixIkcOnQIcH2bfvDBBxk+fDi9e/dmyZIlx8WbkZFxTCL44osvSElJoWvXrmzcuJHhw4czaNAg0tLS2L59e711P/PMM9mzZw8Aubm5XHnllQwbNoxhw4bx1VdfAVBUVMSUKVOq7yLefPNNAG6//XbS09Pp168fDz30UOP/wzeCN7uPvgfcJSLzgNOBfKfaB2o7u2c8gQHC4q25DO0a1xxvaYxXPPz+RjbtLWjSa6Z2iuKhS/t5VLaiooKPPvqI8eNd/UaWL1/Ohg0b6NatG7NnzyY6OpoVK1Zw9OhRRo4cydixY9myZQvvvPMO33zzDeHh4Rw8ePCYax48eJC3336bLVu2ICIcPnz4uPedPHkyTz/9NKNGjWL69Ok8/PDDzJgxozqm5cuX8+GHH/Lwww8f97gpLS2NgIAA1q5dy8CBA5k3bx4ZGRmA6/HTPffcw89+9jPKysoavLOZP38+l19+OQD33HMP9913H2eddRa7du1i3LhxbN68mUcffZTo6GjWr18PUJ20/vjHPxIXF0dlZSXnnXce69atIy0tzaP/7o3lWCIQkX8Bo4H2IpINPAQEA6jqLOBD4CIgEzgCTHEqltqiw4MZ0iWGRVtz+J+xfZrrbY3xGyUlJQwaNAhw3RHcfPPNfP311wwfPry6f/snn3zCunXrqr/l5+fns337dhYsWMCUKVMIDw8HIC7u2C9rUVFRhIaGcsstt3DxxRdzySWXHHM8Pz+fw4cPM2rUKABuuOEGrr766urjV1xxBQBDhw7l+++/rzP+H+8K+vXrx7vvvssjjzwCuL7h//GPfyQ7O5srrriCXr161Xn+mDFj2L9/PwkJCdWPhhYsWMCmTZuqyxQUFFBYWMiCBQuOuQOJjY0F4PXXX2f27NlUVFTwww8/sGnTJt9LBKqa0cBxBe506v0bMrpPAn/5eCs5haUkRIZ6KwxjHOXpN/em9mMbQW0RET/11FNVnn76acaNG3dMmfnz59fbBTIoKIjly5ezcOFC5s2bx8yZM/nss888jq1NmzaAq0G7oqKizjIZGRmMHTuWUaNGkZaWRkJCAgDXXnstp59+Oh988AHjxo3jhRde4Nxzzz3u/EWLFhEREcGNN97I9OnT+dvf/kZVVRVLly4lLCzsmLKqelx9s7KyeOKJJ1ixYgWxsbHceOONjo4U99u5hkb1djU6f7Ht1FvcjTGNN27cOP7xj39QXl4OwLZt2yguLmbs2LHMmTOnuqdR7UdDRUVF5Ofnc9FFFzFjxozjEk50dDSxsbHVz/9feeWV6rsDT/Xo0YN27doxbdq06sdCADt27KB79+784he/4LLLLmPdunUnvEZYWBgzZsxg7ty5HDx4kLFjxzJz5szq4z/GXXv/oUOHKCgoICIigujoaPbv389HH33UqPgby28TQb9OUcRHtmHx1hxvh2KMX7rllltITU1lyJAh9O/fn9tuu42KigrGjx/PZZddRnp6OoMGDapuAP5RYWEhl1xySXX3zL///e/HXfvll1/mgQceIC0tjTVr1jB9+vRGx5eRkcGWLVuYOHFi9b7XXnuN/v37M2jQILZs2dJgL6WOHTuSkZHBM888w1NPPcXKlStJS0sjNTW1uifVb3/7Ww4dOkT//v0ZOHAgixYtYuDAgQwePJh+/fpx0003MXLkyEbH3xjiekLjO9LT07WpFqZ54N9r+WTTflb99nyCAv02J5pWZvPmzfTt29fbYRgvquvfgIisUtX0usr79aff6D4J5JeUs2b3YW+HYowxXuPXieCsXu2ru5EaY4y/8utEEB3m6ka6eJu1E5jWxdce+ZqmczJ/e79OBOB6PLRhTwE5hbaIh2kdQkNDOXDggCUDP/TjegShoY3rEu/3C9OM7hPPXz7eyudbc7k6PbnhE4xp4ZKSksjOzuZkJ2g0vu3HFcoaw+8TQWrHKBIi27B4myUC0zoEBwc3anUqY/z+0ZCIMKp3PEu25dpspMYYv+T3iQBgzGkJFJRWWDdSY4xfskQAjOzp6ka6yEYZG2P8kCUCXN1Ih3aJtfEExhi/ZInAbVSfeDbuLeBgcZm3QzHGmGZlicAttWMUAFl5xV6OxBhjmpclArekWNcc4dmHTn2RbWOM8SWWCNw6VyeCEi9HYowxzcsSgVt4SBDt24bYHYExxu84mghEZLyIbBWRTBGZVsfxWBF5W0TWichyEenvZDwN6RwbbncExhi/41giEJFA4BngQiAVyBCR1FrFfg2sUdU0YDLwpFPxeCIpNozdB+2OwBjjX5y8IxgOZKrqDlUtA+YBE2qVSQUWAqjqFiBFRBIdjKleybHh7DlcQlWVzdpojPEfTiaCzsDuGtvZ7n01rQWuABCR4UBX4Lhp80TkVhFZKSIrnZxRMSk2jPJKZb9NSW2M8SNOJgKpY1/tr9qPAbEisga4G1gNVBx3kupsVU1X1fT4+PgmD/RHSdZzyBjjh5ychjobqDmvcxKwt2YBVS0ApgCIiABZ7h+vSI4LB1xjCYalxHkrDGOMaVZO3hGsAHqJSDcRCQEmAe/VLCAiMe5jALcAX7iTg1d0jnHdEew+aHcExhj/4dgdgapWiMhdwMdAIDBHVTeKyFT38VlAX2CuiFQCm4CbnYrHE6HBgcRHtrGxBMYYv+LoCmWq+iHwYa19s2q8Xgr0cjKGxkrtGMXCzTkcPlJGTHhIwycYY4yPs5HFtfxyfB8Ol5Tz2EdbvB2KMcY0C0sEtfTrFM1NI1OYt2I3K74/6O1wjDHGcZYI6nDv+b3pHBPGr99aT1mFrWNsjGndLBHUIaJNEI9M6Mf2nCKeX7LD2+EYY4yjLBGcwHl9ExnfrwNPf7adPYetO6kxpvWyRFCP31zcF1Ws4dgY06pZIqhHclw4t53TnffX7mV5ljUcG2NaJ0sEDZg6ugcdokJ5+P2NVNqspMaYVsgSQQPCQ4L41UWnsXFvAW+s2t3wCcYY42MsEXjgsoGdGNo1lr98vJWC0nJvh2OMMU3K0SkmWgsR4aFLU7ls5lfM/CyTX1/Ut1HnFx+tICuvuPrn+7xi9hWUEhwYQGhwAKHBgYQFBxIaHEib4ABCgwLpEhfOFUM645qU1RhjnGOJwENpSTFcPTSJf36VxdjURBKjQquPdYoJIzDg+A/svYdLeHZxJq+t2E155U/tC52iQ+kYE0bx0QpKy6soraikpKyS0vJKSiuqqgexfX+gmP8Z28f5yhlj/JolgkZ4YHwfPtqwj6tmLT1m/9Cuscy5cRjRYcEA/JBfwrOLvuO1FbtRlKuGJjOqd3tS2kfQNS6CsJDAet+nskr5zdvrefqzTDrHhDFpeBfH6mSMMZYIGiEhMpR37hzBmt351fsOFB3liU+2kjF7GX+5Oo3XVuxm3nJXArg6PZk7RvcgKTa8Ue8TGCA8enl/9uaX8pt3NtAxJoxRvZ1bmc0Y499E1be6RKanp+vKlSu9HcYxFm/N4bZXVnG0ooqgAOHq9GTuHNP4BFBb0dEK/mvWUnYeKOb1qWfSr1N0E0VsjPE3IrJKVdPrPGaJoGms2nmQTzbu57ozulYvedkU9heUMvGZr6hU5e07RtLJvYqaMcY0Rn2JwLqPNpGhXeP41UV9mzQJACRGhfLPKcM5crSSjOeXsWb34Sa9vjHGWCLwAX06RPLSTcMpr6jiyn98zYwF26iotOmxjTFNwxKBjxjaNZaP7j2HS9M6MmPBdq6atZSsvGJvh2WMaQUcTQQiMl5EtopIpohMq+N4tIi8LyJrRWSjiExxMh5fFx0WzIxJg3k6YzA7cou46Mkl/N83u/C1dh5jTMviWCIQkUDgGeBCIBXIEJHUWsXuBDap6kBgNPBXEbEV4xtw6cBOfHzfOQzpGsOv317PLS+vJLfwqLfDMsb4KCfvCIYDmaq6Q1XLgHnAhFplFIgU1zwKbYGDQIWDMbUaHaPDeOWm05l+SSpLMvMYP+MLPt2039thGWN8kJOJoDNQc7rObPe+mmYCfYG9wHrgHlU9rhVURG4VkZUisjI3N9epeH1OQIBw01nd+M/dZ5EYFcrP565k2pvrKD5qudQY4zknE0Fds6XVfpg9DlgDdAIGATNFJOq4k1Rnq2q6qqbHx9sI29p6J0by9p0jmDqqB6+t3M34J7/ghSU7yCks9XZoxhgf4GQiyAaSa2wn4frmX9MU4C11yQSygNMcjKnVahMUyLQLT2Pez88gLjyEP3ywmTP/9BlT/rmc99fupbS80pH3raxScgpL2bAnn68y8xx7H2OMc5yca2gF0EtEugF7gEnAtbXK7ALOA5aISCLQB9jhYEyt3und2/HuXWeRmVPIm9/u4e1v93D3v1YTGRrEJWkduWJIEuldY4+Z3rq8sop12fkszzpIYWk5VQqqSpUqVYrrd5XrdXllFXlFR8kpPMr+glLyisqOWbmtc0wYv7roNC4e0NGm0DbGRzg6xYSIXATMAAKBOar6RxGZCqCqs0SkE/AS0BHXo6THVPXV+q7ZUqeYaKkqq5RlOw7w5qpsPtqwj5Lyyuq1Dtq2CeLr7w6wPOsgRe52hZDAAEQgQIQA928RV3tEgAiBAUK7iBASo0JJiGzj+h3VhoRI17TcTy7czuYfChieEsf0S1Pp39nmRzKmJbC5hgzgWiDnow37eOvbbJbuOIAqdGsfwYge7RjZsz1ndm9HbMSp9d6trFJeW7GbJz7ZyqEjZYzo0Y4JAzszrn+H6mm6jTHNzxKBOc7+glIqq9SxSezyS8r551dZvL16DzsPHCEkKIBz+yRw4YAO9IhvS+eYMGLCg+3xkTHNxBKB8RpVZV12Pu+s2cP7a38gr+ingW/hIYF0igmjc0wYSbFhXDU0icFdYr0YrTGtlyUC0yJUVimbfygg+9AR9hwuZc+hEvYcPsKewyV8n3eE4rIKJp/RlfvH9SEy1B4jGdOU6ksEtkKZaTaBAUL/ztF1NiAXlpbz10+28fLS7/l4434emdCPsf06eCFKY/yPzT5qWoTI0GB+f1k/3rp9BDHhwdz6yipuf3UV+wtsUJwxTrNEYFqUwV1ief/us/jl+D58tiWH8//6OXOXfm/rLxjjIEsEpsUJDgzgjtE9+eS+cxiYHMP0dzdy4ZNLWLw1x9uhGdMqWSIwLVbXdhG8cvNwnrt+KOWVVdz4zxVMnrOczJxCb4dmTKtiicC0aCLCuH4d+OS+Ufz24r6s3nWI8TOW8Mj7m8gvKfd2eMa0CtZ91PiUA0VHeeKTbcxbsYu48BBuG9WdPh2iSIp1jUcIDQ70dojGtEg2jsC0Ohv25PPw+xtZ8f2hY/a3b9vGlRRiXYPUkmLCSIoNJz6yDQeLy9hzuMQ9fsH1e29+CWd2b8ejl/e3JGJaNUsEplVSVX7IL2XP4RKyDx0h+2CJ+/VPH/RldfQ2CgwQOkSF0jkmjOjwYD7dtJ9hKbE8PzmdmHBbKdW0TjagzLRKIkKnmDA6xYQxLCXuuONVVUpe0VF2Hyohp6CUuIgQOseG0SEqlKDAn5rH3l+7l/95fS1X/ONrXp4ynOS48OashjFeZ4nAtFoBAUJCVCgJUaH1lrt0YKfqpT4nPvsVc24cRlpSTPMEaUwLYL2GjAGGd4vjzdtHEBocyPUvLmfrPuuiavyHJQJj3HomtOVfPz+D0OAArn/xG3YdOOLtkIxpFpYIjKkhOS6cV24+nbLKKq578RtybK4j4wcsERhTS+/ESP554zDyio4yec5y8o/YwDXTujmaCERkvIhsFZFMEZlWx/EHRGSN+2eDiFSKyPHdP4xpZoO7xDL7+nR25BYz5aXlFJZaMjCtl0eJQERGisinIrJNRHaISJaI7GjgnEDgGeBCIBXIEJHUmmVU9S+qOkhVBwG/Aj5X1YMnVRNjmthZvdrzVMYg1uw+zOi/LOafX2VxtKLS22EZ0+Q87T76InAfsArw9P+E4UCmqu4AEJF5wARg0wnKZwD/8vDaxjSL8f078vYdI/nz/C08/P4mnlq4nTF9EjivbyLn9G5vK6mZVsHTRJCvqh818tqdgd01trOB0+sqKCLhwHjgrhMcvxW4FaBLly6NDMOYUzMwOYb/d8vpfJV5gLe+zWbR1hzeWr2H4EDh9G7tOK9vAuf3TbSBaMZneTTFhIg8BgQCbwHVq4+r6rf1nHM1ME5Vb3FvXw8MV9W76yh7DXCdql7aUCw2xYTxtsoq5dtdh1iweT8LN+eQmVMEwIge7Xj5puEEB1ofDNPyNMUUEz9+k695EQXOreecbCC5xnYSsPcEZSdhj4WMjwgMEIalxDEsJY5fXdiXnQeKeevbPTy5cDsvf/09t5zd/ZjyFZVVfH/gCNFhwcRHtvFS1MacmEeJQFXHnMS1VwC9RKQbsAfXh/21tQuJSDQwCrjuJN7DGK/r2i6C+y7ozbrsw8xYsJ2EqFB25hWzLaeI7fsL2ZFbTFllFeEhgbwxdQSpnaK8HbIxx/C011C0iPxNRFa6f/7q/gA/IVWtwPXM/2NgM/C6qm4UkakiMrVG0YnAJ6pafLKVMKYl+N0lqRytqOQX/1rNXz/dxupdh+gUE8aUs1J4/Mo0okKDueXlFeQU2iA107J42kbwJrABeNm963pgoKpe4WBsdbI2AtOSbdpbQFllFb0S2hLR5tgb7g178rl61lJ6J7blF+f1omu7CLq3jyAgQLwUrfEnp7wegYiscff1r3dfc7BEYHzZJxv3cef/fUt5pev/u8sGduKpjMFejsr4g/oSgafdG0pE5KwaFxwJlDRFcMb4k7H9OrDiN+fz1h0juO6MLry3di/f7Djg7bCMn/M0EdwOPCMi34vITmAmMLWBc4wxdYgJD2FIl1h+c1EqHaJC+d8PN1NV5VsrBZrWxaNEoKprVHUgkAYMUNXBqrrW2dCMad3CQgK5f1wf1mbn8/66E/WsNsZ59XYfFZHrVPVVEfnvWvsBUNW/ORibMa3exMGdmfNlFn/4YDMHisq4cEAHOkaHeTss42cauiOIcP+OPMGPMeYUBAYIf74yjXYRITzyn02c+afPyJi9zKa+Ns3Ko15DLYn1GjKt1Xe5Rby7eg9PfZbJY1cMYNJwm1fLNJ1T7jUkIo+LSJSIBIvIQhHJExEbCWxME+oR35b7LuhNh6hQlmzP83Y4xo942mtorKoWAJfgmkOoN/CAY1EZ46dEhLN7tWfhlv3cO281sz7/jsVbc9iXX4qv3b0b3+HppHM/Trp+EfAvVT34Y4OxMaZpTRnZjQPFZSzPOsg7a37qTTQoOYbZk4eSEBnqxehMa+RpInhfRLbgGkR2h4jEAzZhijEOSO0UxZwbhwGQf6ScLfsKWJt9mL9/up2Jz3zNS1OG0SvR+mqYpuNxY7GIxAIFqlrpXkgmSlX3ORpdHayx2Pir9dn53PTyCkrLK3nu+qGM6NHe2yEZH3LSjcUicq779xXAGGCC+/V4YERTB2qMObEBSdG8fccIOkSFcsOc5by7Zo+3QzKtREOPhkYBnwF1rRymuFYsM8Y0k6TYcN64fQS3zl3Jva+tQRUuH9zZ22EZH1dvIlDVh9y/pzRPOMaYhkSHBfPSlOFMeWk5//36GoIDA7g4raO3wzI+zNNxBP8rIjE1tmNF5A+ORWWMqVdYSCAv3jCMIV1i+cW81czfcHxznapSVlHlheiMr/F0PYLVqjq41r5vVXWIY5GdgDUWG/OTwtJyrn9xORv35vOz07tSUFrOvvxS109BKSXllcy5YRhjTkvwdqjGy5piPYJAEaledVtEwgBbhdsYL4sMDeblm4YzKDmGV5ft5JsdByktr6RvxygyhnehS1w4j/xnk90ZmHp5Oo7gVWChiPwTVyPxTfy0bOUJich44EkgEHhBVR+ro8xoYAauQWt5qjrKw5iMMbjaDF6/7UxUOW7Zy5E923HTSyv500ebSYgM5UDRUW4cmUJSbLiXojUtUWPGEYwHzgcE12LzHzdQPhDYBlyAa1qKFUCGqm6qUSYG+BoYr6q7RCRBVXPqu649GjLGc6rK5DnLq+cuCgwQYsODmT05nSFdYr0cnWlO9T0a8vSOAGAzUKGqC0QkXEQiVbWwnvLDgUxV3eEOYh4wAdhUo8y1wFuqugugoSRgjGkcEWHmtUPIyiume3wEOQVHuemlFUyavYyZGYMZ26+Dt0M0LYCnvYZ+DrwBPOfe1Rl4p4HTOgO7a2xnu/fV1BuIFZHFIrJKRCaf4P1vFZGVIrIyNzfXk5CNMW7RYcEMSo4hKjSYnglteefOkfRJjOT+f68lt/Cot8MzLYCnjcV3AiOBAgBV3Q401A2hrlnpaj+HCgKGAhcD44DfiUjv405Sna2q6aqaHh8f72HIxpi6xEWE8PdrBlFaXsUfPtjU8Amm1fM0ERxV1bIfN0QkiOM/1GvLBpJrbCcBtRdmzQbmq2qxquYBXwADPYzJGHOSeia05fbRPXh3zV6+2GZ32f7O0zaCz0Xk10CYiFwA3AG838A5K4BeItIN2ANMwtUmUNO7wEx3YgkBTgf+7mnwxpiTd/voHry/di/3zFvNgKQY4sKDiY0IITY8hNiIEOLbhnBO73jCQxrTlGh8kad/4QeBW4D1wG3Ah8AL9Z2gqhUichfwMa7uo3NUdaOITHUfn6Wqm0VkPrAOqMLVxXTDyVXFGNMYocGBPJUxmL9/uo28oqNk5RVxuLicwqMV1WV6xEcw89oh9O0Y5cVIjdMa7D4qIgHAOlXt3zwh1c+6jxrjrLKKKg6XlLFhTz7T3lzP4ZJyfntxX64/oyu2IJXvOqWRxapaBawVEVtJ2xg/EBIUQEJkKOeelshH95zNyB7tmP7uRm57ZRU5hbYeVWvk6VxDnwHDgOVA8Y/7VfUy50Krm90RGNO8qqqUOV9l8ef5WwgMEK5JT2Zwl1j6doyie3wEwYGe9jkx3lTfHYGniaDOaR9U9fNTjK3RLBEY4x3f5xXzt0+3MX/DPsoqXXMXhQQGMKxbLI9O6E/3+LZejtDU56QTgYiEAlOBnrgail9U1YoTntAMLBEY413llVXsyC1m8w8FbPqhgNdW7KasooqnMwZzfmqit8MzJ3AqieA1oBxYAlwI7FTVexyJ0kOWCIxpWfbll3LL3BXsPljCJ/edQ2JUqLdDMnU4lcbiVFW9TlWfA64Czm7y6IwxPq1DdChPTRrM0YpK7v/3Wo5WVHo7JNNIDSWC8h9fePuRkDGm5eoe35bfXZLKku15XPzUl3y5PQ9PZzY23tfQgLKBIlLgfi24RhYXuF+rqtooE2MMAD87vSudosP4zdvrue7FbzitQyTn903k9O5xDO0aayOUWzCP1yNoKayNwJiWrbS8knfX7GHeit2sy86nskoJChA6xoTSMSqMDtGhdIwOpUN0KKd1iOLMHu28HbJfOOXuoy2JJQJjfEfR0QpW7TzEiqyD7D50hB9+XE85v7S6C+oVgzvzyOX9advG7hic1FQL0xhjTKO0bRPEqN7xjOp97PTxqsrB4jLmLt3J059tZ/Xuwzw1aTADkqK9FKl/szsCY4xXLdtxgHvnrSG36ChDu8Yyqnc8o/vEk9oxyuY2akL2aMgY06IdKi7j+SU7WLw1l00/uPqnxEe2qb6bGNevAyFBNpXFqbBEYIzxGTkFpXy+LZfPt+WyZHse+SXl3Dgihd9f1s/bofm0U5p91BhjmlNCVChXpycz89ohfPu7Cxibmsh/1u2lssq3vrT6EksExpgWKzBAuHRgJ/KKynh8/ha27S+0gWoOsF5DxpgW7by+CZzTO57nvtjBc1/soHv7CMb268AZ3eMYnBxLdHiwt0P0edZGYIzxCTkFpXyyaT8fb9zH0u8OUOF+VNQ9PoIhXWIZ3CWGET3a0619hJcjbZm81lgsIuOBJ3GtWfyCqj5W6/hoXAvYZ7l3vaWqj9R3TUsExpiioxWs232Y1bsPs3rXIb7ddZiDxWUAnNWzPTeMSOHc0xIIDLDupz/yyoAyEQkEngEuALKBFSLynqpuqlV0iape4lQcxpjWp22bIEb0bM+Inu0B1wC1nQeO8MH6H3h12U5+PnclXeLC+e8LenPZwE4EWEKol5ONxcOBTFXdoaplwDxggoPvZ4zxUyJCSvsI7hzTkyW/HMOzPxtCZGgQ9762hktnfsn8Dfus11E9nEwEnYHdNbaz3ftqO1NE1orIRyJSZ0dhEblVRFaKyMrc3FwnYjXGtBJBgQFcNKAj7991FjOuGURBaTlTX13F6CcW8eKXWRSWljd8ET/jWBuBiFwNjFPVW9zb1wPDVfXuGmWigCpVLRKRi4AnVbVXfde1NgJjTGNUVFbx6ab9vPhlFit3HqJtmyD+Kz2ZKSNTSI4L93Z4zcZbA8qygeQa20nA3poFVLVAVYvcrz8EgkWkvYMxGWP8TFBgABcO6Mgbt4/g3TtHcn7fBOYu/Z7z//Y5i7bmeDu8FsHJRLAC6CUi3UQkBJgEvFezgIh0EPesUiIy3B3PAQdjMsb4sYHJMcyYNJglD46hZ0Jbbpu7ik827vN2WF7nWCJwL215F/AxsBl4XVU3ishUEZnqLnYVsEFE1gJPAZPU1wY2GGN8TsfoMP7vljPo2ymKO/7ft/z2nfWs2nnIb0ct24AyY4zfKiwt56F3N/LB+h84WlFFl7hwLh/cmYsHdKR3YttWNQ22zT5qjDH1KCwt5+ON+3ln9R6+/i6PKoWUduGM69eBsf0SGZwc6/NjESwRGGOMh3IKS1mwKYf5G/ex9Ls8yiuVpNgwXrvtTDrHhHk7vJNmicAYY05Cfkk5n23Zz6/f2sDInu14fnK6zz4usvUIjDHmJESHBTNxcBL3XdCLBZtz+Hjjfm+H5AhLBMYY04ApI7vRO7EtMxdt93YojrBEYIwxDQgODGDCoM5s2FNATkGpt8NpcpYIjDHGA6P7xAPw5rd7Wt14A0sExhjjgdSOUQxKjuHP87eQ8fwy1mfnezukJmOJwBhjPCAi/HvqmTwyoR/b9xdx6cwvmf7uBioqq7wd2imzRGCMMR4KDgxg8pkpLH5gNDeN7MbcpTuZ+uq3lJRVeju0U2KJwBhjGikyNJjpl6by6IR+LNyyn4znl7F292Fvh3XSLBEYY8xJuv7MFJ69dghZecVMeOYrbpiznFU7D3o7rEazRGCMMafgwgEd+fLBMfxyfB/W78nnyn8sJWP2Mp9KCJYIjDHmFEWGBnPH6J58+eAYfntxX7bnFPFfzy3jlWU7vR2aRywRGGNMEwkPCeKWs7uz+IHRnNOrPb97ZwN/+M8mKqta9rgDSwTGGNPE2rYJ4vnJ6dxwZlde+DKLqa+u4khZhbfDOiFLBMYY44CgwAAentCfhy5NZeHm/dz32hpvh3RCQd4OwBhjWrMpI7uxPOsgW/cXejuUE3L0jkBExovIVhHJFJFp9ZQbJiKVInKVk/EYY4w3BLbw1c0cSwQiEgg8A1wIpAIZIpJ6gnJ/xrXIvTHGtDqRocHkFR6lqoU2Gjt5RzAcyFTVHapaBswDJtRR7m7gTSDHwViMMcZrhqXEUlBawZZ9LfPxkJOJoDOwu8Z2tntfNRHpDEwEZjkYhzHGeNXp3dsB8E3WAS9HUjcnE0FdD8Vq3xfNAB5U1XpnbBKRW0VkpYiszM3Nbar4jDGmWXSOCaN7fATPLv6OrS3wrsDJRJANJNfYTgL21iqTDswTke+Bq4BnReTy2hdS1dmqmq6q6fHx8Q6Fa4wxzpl13VACBP7ruaV8u+uQt8M5hpOJYAXQS0S6iUgIMAl4r2YBVe2mqimqmgK8Adyhqu84GJMxxnhF78RI3pg6gpjwYCa/uJy8oqPeDqmaY4lAVSuAu3D1BtoMvK6qG0VkqohMdep9jTGmpUqOC+fFG4ZRXFbB80t2eDucao4OKFPVD4EPa+2rs2FYVW90MhZjjGkJeia05dK0TryydCe3ndODuIgQb4dkU0wYY0xzu/vcnpSUV/LSV1neDgWwRGCMMc2uV2IkI3q048MN+7wdCmCJwBhjvOK80xLJzCniu9wib4diicAYY7zh/L6JBAUI42d8wTXPLeWZRZls2JPvlVgsERhjjBd0aRfOm7eP4OazulNYWsFfPt7KJU9/yd8+3dbssdg01MYY4yUDk2MYmBzDtAtPI7fwKH/6cDNPLdxOz4S2XDawU7PFYXcExhjTAsRHtuGxK9MY3i2OX76xtlnbDiwRGGNMCxESFMDMjMG0CQrk/n+vpaKyqlne1xKBMca0IAlRoTx6eX9W7zrMy0t3Nst7WiIwxpgW5tK0jpzRPY45X2ZR2QyL2VgiMMaYFkZEuP6MFPYcLuGLbc5PvW+JwBhjWqALUhNp37YNry5z/vGQJQJjjGmBQoICmDQsmc+25pBTWOroe1kiMMaYFuqc3vGowsa9BY6+jyUCY4xpoXontgVgm8PLW1oiMMaYFiomPITEqDZs3W+JwBhj/FbvxEi2WSIwxhj/1Tsxku37i6hycDyBJQJjjGnBesS35WhFFT8UONdzyNFEICLjRWSriGSKyLQ6jk8QkXUiskZEVorIWU7GY4wxvqZb+wgAsnKLHXsPxxKBiAQCzwAXAqlAhoik1iq2EBioqoOAm4AXnIrHGGN8Ufd4VyLYkefcbKRO3hEMBzJVdYeqlgHzgAk1C6hqkar++OArAnB+Ug1jjPEhCZFtiAgJZIcv3hEAnYHdNbaz3fuOISITRWQL8AGuu4LjiMit7kdHK3NznZ93wxhjWgoRoVt8BDvyfDMRSB37jvvGr6pvq+ppwOXAo3VdSFVnq2q6qqbHx8c3bZTGGNPCdYkLJ/vgEceu72QiyAaSa2wnAXtPVFhVvwB6iEh7B2MyxhifkxwbTvbhEse6kDqZCFYAvUSkm4iEAJOA92oWEJGeIiLu10OAEOCAgzEZY4zPSYoLp6yiityio45c37HF61W1QkTuAj4GAoE5qrpRRKa6j88CrgQmi0g5UAJcU6Px2BhjDJAUGwbA7oNHSIwKbfLrO5YIAFT1Q+DDWvtm1Xj9Z+DPTsZgjDG+Ljk2HIDsQyWkpzT99W1ksTHGtHA17wicYInAGGNauNDgQCYM6kRyXLgj13f00ZAxxpim8eSkwY5d2+4IjDHGz1kiMMYYP2eJwBhj/JwlAmOM8XOWCIwxxs9ZIjDGGD9nicAYY/ycJQJjjPFz4mtzvIlILrCzkae1B/IcCKcl88c6g3/W2+rsP06l3l1Vtc4FXXwuEZwMEVmpqunejqM5+WOdwT/rbXX2H07V2x4NGWOMn7NEYIwxfs5fEsFsbwfgBf5YZ/DPelud/Ycj9faLNgJjjDEn5i93BMYYY07AEoExxvi5VpUIRGS8iGwVkUwRmVbHcRGRp9zH14nIEG/E2ZQ8qPPP3HVdJyJfi8hAb8TZlBqqc41yw0SkUkSuas74nOJJvUVktIisEZGNIvJ5c8fY1Dz49x0tIu+LyFp3nad4I86mJCJzRCRHRDac4HjTf46paqv4AQKB74DuQAiwFkitVeYi4CNAgDOAb7wddzPUeQQQ6359oT/UuUa5z4APgau8HXcz/a1jgE1AF/d2grfjboY6/xr4s/t1PHAQCPF27KdY73OAIcCGExxv8s+x1nRHMBzIVNUdqloGzAMm1CozAZirLsuAGBHp2NyBNqEG66yqX6vqIffmMiCpmWNsap78nQHuBt4EcpozOAd5Uu9rgbdUdReAqvp63T2pswKRIiJAW1yJoKJ5w2xaqvoFrnqcSJN/jrWmRNAZ2F1jO9u9r7FlfElj63Mzrm8SvqzBOotIZ2AiMKsZ43KaJ3/r3kCsiCwWkVUiMrnZonOGJ3WeCfQF9gLrgXtUtap5wvOaJv8ca02L10sd+2r3jfWkjC/xuD4iMgZXIjjL0Yic50mdZwAPqmql64tiq+BJvYOAocB5QBiwVESWqeo2p4NziCd1HgesAc4FegCfisgSVS1wODZvavLPsdaUCLKB5BrbSbi+JTS2jC/xqD4ikga8AFyoqgeaKTaneFLndGCeOwm0By4SkQpVfadZInSGp/++81S1GCgWkS+AgYCvJgJP6jwFeExdD88zRSQLOA1Y3jwhekWTf461pkdDK4BeItJNREKAScB7tcq8B0x2t7qfAeSr6g/NHWgTarDOItIFeAu43oe/GdbUYJ1VtZuqpqhqCvAGcIePJwHw7N/3u8DZIhIkIuHA6cDmZo6zKXlS51247oAQkUSgD7CjWaNsfk3+OdZq7ghUtUJE7gI+xtXbYI6qbhSRqe7js3D1ILkIyASO4Po24bM8rPN0oB3wrPsbcoX68KyNHta51fGk3qq6WUTmA+uAKuAFVa2zC6Iv8PBv/Sjwkoisx/XI5EFV9enpqUXkX8BooL2IZAMPAcHg3OeYTTFhjDF+rjU9GjLGGHMSLBEYY4yfs0RgjDF+zhKBMcb4OUsExhjj5ywRGFMH96yla0Rkg3t2y5gmvv73ItLe/bqoKa9tTGNZIjCmbiWqOkhV++OaAOxObwdkjFMsERjTsKW4J/USkR4iMt89qdsSETnNvT9RRN52z4u/VkRGuPe/4y67UURu9WIdjDmhVjOy2BgniEggrikMXnTvmg1MVdXtInI68CyuCc+eAj5X1Ynuc9q6y9+kqgdFJAxYISJvtoL5nkwrY4nAmLqFicgaIAVYhWtWy7a4Fvr5d41ZTdu4f58LTAZQ1Uog373/FyIy0f06GegFWCIwLYolAmPqVqKqg0QkGvgPrjaCl4DDqjrIkwuIyGjgfOBMVT0iIouBUCeCNeZUWBuBMfVQ1XzgF8D9QAmQJSJXQ/XasT+uAb0QuN29P1BEooBo4JA7CZyGa1lBY1ocSwTGNEBVV+NaL3cS8DPgZhFZC2zkp6UT7wHGuGfBXAX0A+YDQSKyDtcsmcuaO3ZjPGGzjxpjjJ+zOwJjjPFzlgiMMcbPWSIwxhg/Z4nAGGP8nCUCY4zxc5YIjDHGz1kiMMYYP/f/AZ0ryrn/7OVtAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plot of Recall (x-axis) Vs Precision (y-axis)\n",
    "plt.plot(df_scores_2['recall'], df_scores_2['precision'], label='Precision Vs Recall')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6590294271576884"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#After dropping NaNs in precision, calculating auc with recall first and precision second, dont get the above error\n",
    "auc_score = auc(df_scores_2['recall'],df_scores_2['precision'])\n",
    "auc_score"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}