algorithms.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "85cfb551",
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing necessary packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split,GridSearchCV,KFold,cross_val_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score,auc,classification_report,roc_curve,roc_auc_score\n",
    "from sklearn.decomposition import PCA\n",
    "from imblearn.over_sampling import SMOTE\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rcParams\n",
    "import seaborn as sns\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.naive_bayes import GaussianNB  \n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from scipy.stats import chi2_contingency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "6679d3c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11055, 31)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>having_IP_Address</th>\n",
       "      <th>URL_Length</th>\n",
       "      <th>Shortining_Service</th>\n",
       "      <th>having_At_Symbol</th>\n",
       "      <th>double_slash_redirecting</th>\n",
       "      <th>Prefix_Suffix</th>\n",
       "      <th>having_Sub_Domain</th>\n",
       "      <th>SSLfinal_State</th>\n",
       "      <th>Domain_registeration_length</th>\n",
       "      <th>Favicon</th>\n",
       "      <th>...</th>\n",
       "      <th>popUpWidnow</th>\n",
       "      <th>Iframe</th>\n",
       "      <th>age_of_domain</th>\n",
       "      <th>DNSRecord</th>\n",
       "      <th>web_traffic</th>\n",
       "      <th>Page_Rank</th>\n",
       "      <th>Google_Index</th>\n",
       "      <th>Links_pointing_to_page</th>\n",
       "      <th>Statistical_report</th>\n",
       "      <th>Result</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   having_IP_Address  URL_Length  Shortining_Service  having_At_Symbol  \\\n",
       "0                 -1           1                   1                 1   \n",
       "1                  1           1                   1                 1   \n",
       "2                  1           0                   1                 1   \n",
       "3                  1           0                   1                 1   \n",
       "4                  1           0                  -1                 1   \n",
       "\n",
       "   double_slash_redirecting  Prefix_Suffix  having_Sub_Domain  SSLfinal_State  \\\n",
       "0                        -1             -1                 -1              -1   \n",
       "1                         1             -1                  0               1   \n",
       "2                         1             -1                 -1              -1   \n",
       "3                         1             -1                 -1              -1   \n",
       "4                         1             -1                  1               1   \n",
       "\n",
       "   Domain_registeration_length  Favicon  ...  popUpWidnow  Iframe  \\\n",
       "0                           -1        1  ...            1       1   \n",
       "1                           -1        1  ...            1       1   \n",
       "2                           -1        1  ...            1       1   \n",
       "3                            1        1  ...            1       1   \n",
       "4                           -1        1  ...           -1       1   \n",
       "\n",
       "   age_of_domain  DNSRecord  web_traffic  Page_Rank  Google_Index  \\\n",
       "0             -1         -1           -1         -1             1   \n",
       "1             -1         -1            0         -1             1   \n",
       "2              1         -1            1         -1             1   \n",
       "3             -1         -1            1         -1             1   \n",
       "4             -1         -1            0         -1             1   \n",
       "\n",
       "   Links_pointing_to_page  Statistical_report  Result  \n",
       "0                       1                  -1      -1  \n",
       "1                       1                   1      -1  \n",
       "2                       0                  -1      -1  \n",
       "3                      -1                   1      -1  \n",
       "4                       1                   1       1  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df= pd.read_csv('phising.csv')\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "64eaaa6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "having_IP_Address              0\n",
      "URL_Length                     0\n",
      "Shortining_Service             0\n",
      "having_At_Symbol               0\n",
      "double_slash_redirecting       0\n",
      "Prefix_Suffix                  0\n",
      "having_Sub_Domain              0\n",
      "SSLfinal_State                 0\n",
      "Domain_registeration_length    0\n",
      "Favicon                        0\n",
      "port                           0\n",
      "HTTPS_token                    0\n",
      "Request_URL                    0\n",
      "URL_of_Anchor                  0\n",
      "Links_in_tags                  0\n",
      "SFH                            0\n",
      "Submitting_to_email            0\n",
      "Abnormal_URL                   0\n",
      "Redirect                       0\n",
      "on_mouseover                   0\n",
      "RightClick                     0\n",
      "popUpWidnow                    0\n",
      "Iframe                         0\n",
      "age_of_domain                  0\n",
      "DNSRecord                      0\n",
      "web_traffic                    0\n",
      "Page_Rank                      0\n",
      "Google_Index                   0\n",
      "Links_pointing_to_page         0\n",
      "Statistical_report             0\n",
      "Result                         0\n",
      "dtype: int64\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 11055 entries, 0 to 11054\n",
      "Data columns (total 31 columns):\n",
      "having_IP_Address              11055 non-null int64\n",
      "URL_Length                     11055 non-null int64\n",
      "Shortining_Service             11055 non-null int64\n",
      "having_At_Symbol               11055 non-null int64\n",
      "double_slash_redirecting       11055 non-null int64\n",
      "Prefix_Suffix                  11055 non-null int64\n",
      "having_Sub_Domain              11055 non-null int64\n",
      "SSLfinal_State                 11055 non-null int64\n",
      "Domain_registeration_length    11055 non-null int64\n",
      "Favicon                        11055 non-null int64\n",
      "port                           11055 non-null int64\n",
      "HTTPS_token                    11055 non-null int64\n",
      "Request_URL                    11055 non-null int64\n",
      "URL_of_Anchor                  11055 non-null int64\n",
      "Links_in_tags                  11055 non-null int64\n",
      "SFH                            11055 non-null int64\n",
      "Submitting_to_email            11055 non-null int64\n",
      "Abnormal_URL                   11055 non-null int64\n",
      "Redirect                       11055 non-null int64\n",
      "on_mouseover                   11055 non-null int64\n",
      "RightClick                     11055 non-null int64\n",
      "popUpWidnow                    11055 non-null int64\n",
      "Iframe                         11055 non-null int64\n",
      "age_of_domain                  11055 non-null int64\n",
      "DNSRecord                      11055 non-null int64\n",
      "web_traffic                    11055 non-null int64\n",
      "Page_Rank                      11055 non-null int64\n",
      "Google_Index                   11055 non-null int64\n",
      "Links_pointing_to_page         11055 non-null int64\n",
      "Statistical_report             11055 non-null int64\n",
      "Result                         11055 non-null int64\n",
      "dtypes: int64(31)\n",
      "memory usage: 2.6 MB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "#performing basic checks\n",
    "print(df.isnull().sum())\n",
    "print(df.info())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "c41c81a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop non related features\n",
    "df.drop(columns=['Statistical_report','Iframe','popUpWidnow',\n",
    "         'on_mouseover','Submitting_to_email',\n",
    "         'HTTPS_token','port','Favicon','having_At_Symbol'],\n",
    "        axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "35ae198a",
   "metadata": {},
   "outputs": [],
   "source": [
    "x= df.iloc[:,:-1].astype(object)\n",
    "y=df['Result']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "ccc4af02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RangeIndex(start=0, stop=21, step=1)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col=x.columns\n",
    "col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "e4c22f76",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv  \n",
    "\n",
    "with open('testing.csv', 'w', encoding='UTF8') as f:\n",
    "    writer = csv.writer(f)\n",
    "\n",
    "    # write the header\n",
    "    writer.writerow(col)\n",
    "\n",
    "    # write the data\n",
    "    writer.writerow(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "56e90d9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 11055 entries, 0 to 11054\n",
      "Data columns (total 21 columns):\n",
      "having_IP_Address              11055 non-null object\n",
      "URL_Length                     11055 non-null object\n",
      "Shortining_Service             11055 non-null object\n",
      "double_slash_redirecting       11055 non-null object\n",
      "Prefix_Suffix                  11055 non-null object\n",
      "having_Sub_Domain              11055 non-null object\n",
      "SSLfinal_State                 11055 non-null object\n",
      "Domain_registeration_length    11055 non-null object\n",
      "Request_URL                    11055 non-null object\n",
      "URL_of_Anchor                  11055 non-null object\n",
      "Links_in_tags                  11055 non-null object\n",
      "SFH                            11055 non-null object\n",
      "Abnormal_URL                   11055 non-null object\n",
      "Redirect                       11055 non-null object\n",
      "RightClick                     11055 non-null object\n",
      "age_of_domain                  11055 non-null object\n",
      "DNSRecord                      11055 non-null object\n",
      "web_traffic                    11055 non-null object\n",
      "Page_Rank                      11055 non-null object\n",
      "Google_Index                   11055 non-null object\n",
      "Links_pointing_to_page         11055 non-null object\n",
      "dtypes: object(21)\n",
      "memory usage: 1.8+ MB\n"
     ]
    }
   ],
   "source": [
    "x.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "7bb51d2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scalar  = StandardScaler()\n",
    "scaled_data = scalar.fit_transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "c4dca627",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.240857</td>\n",
       "      <td>2.790134</td>\n",
       "      <td>-2.460660</td>\n",
       "      <td>-0.651789</td>\n",
       "      <td>0.170625</td>\n",
       "      <td>0.349632</td>\n",
       "      <td>-0.475998</td>\n",
       "      <td>-0.667330</td>\n",
       "      <td>1.257793</td>\n",
       "      <td>0.864782</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.219467</td>\n",
       "      <td>0.138482</td>\n",
       "      <td>0.353048</td>\n",
       "      <td>0.675946</td>\n",
       "      <td>0.894278</td>\n",
       "      <td>-0.063213</td>\n",
       "      <td>-0.541114</td>\n",
       "      <td>0.501641</td>\n",
       "      <td>-1.250294</td>\n",
       "      <td>-0.987347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.658318</td>\n",
       "      <td>-0.224639</td>\n",
       "      <td>-1.080069</td>\n",
       "      <td>-0.999167</td>\n",
       "      <td>-0.988480</td>\n",
       "      <td>0.746357</td>\n",
       "      <td>-0.188653</td>\n",
       "      <td>-0.085084</td>\n",
       "      <td>-1.176667</td>\n",
       "      <td>0.930242</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.242841</td>\n",
       "      <td>-0.316790</td>\n",
       "      <td>0.068591</td>\n",
       "      <td>1.347992</td>\n",
       "      <td>1.088595</td>\n",
       "      <td>-0.118375</td>\n",
       "      <td>-0.210533</td>\n",
       "      <td>-0.260355</td>\n",
       "      <td>-0.101061</td>\n",
       "      <td>-0.053715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.453251</td>\n",
       "      <td>0.686791</td>\n",
       "      <td>-0.873488</td>\n",
       "      <td>-2.215446</td>\n",
       "      <td>0.123773</td>\n",
       "      <td>-0.202833</td>\n",
       "      <td>1.087286</td>\n",
       "      <td>0.707612</td>\n",
       "      <td>-0.341302</td>\n",
       "      <td>0.124591</td>\n",
       "      <td>...</td>\n",
       "      <td>0.889354</td>\n",
       "      <td>0.593509</td>\n",
       "      <td>-0.020307</td>\n",
       "      <td>-0.253006</td>\n",
       "      <td>0.723389</td>\n",
       "      <td>0.024207</td>\n",
       "      <td>-0.529978</td>\n",
       "      <td>1.448838</td>\n",
       "      <td>-0.125124</td>\n",
       "      <td>0.012569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-2.059316</td>\n",
       "      <td>0.237690</td>\n",
       "      <td>-0.112323</td>\n",
       "      <td>-0.905261</td>\n",
       "      <td>-1.232791</td>\n",
       "      <td>0.668653</td>\n",
       "      <td>1.176992</td>\n",
       "      <td>-0.143339</td>\n",
       "      <td>0.517313</td>\n",
       "      <td>-0.129502</td>\n",
       "      <td>...</td>\n",
       "      <td>0.390874</td>\n",
       "      <td>0.737615</td>\n",
       "      <td>0.021002</td>\n",
       "      <td>0.586084</td>\n",
       "      <td>0.649596</td>\n",
       "      <td>1.406424</td>\n",
       "      <td>0.460842</td>\n",
       "      <td>0.385251</td>\n",
       "      <td>-0.024130</td>\n",
       "      <td>0.019499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.083504</td>\n",
       "      <td>0.571745</td>\n",
       "      <td>-0.557610</td>\n",
       "      <td>-0.633726</td>\n",
       "      <td>-1.366600</td>\n",
       "      <td>0.336196</td>\n",
       "      <td>-1.236697</td>\n",
       "      <td>-0.080389</td>\n",
       "      <td>-0.515733</td>\n",
       "      <td>0.103772</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.135934</td>\n",
       "      <td>-0.215368</td>\n",
       "      <td>0.118408</td>\n",
       "      <td>0.961872</td>\n",
       "      <td>0.650594</td>\n",
       "      <td>-0.154942</td>\n",
       "      <td>0.021585</td>\n",
       "      <td>-0.872449</td>\n",
       "      <td>1.398305</td>\n",
       "      <td>0.297530</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1         2         3         4         5         6   \\\n",
       "0  0.240857  2.790134 -2.460660 -0.651789  0.170625  0.349632 -0.475998   \n",
       "1  0.658318 -0.224639 -1.080069 -0.999167 -0.988480  0.746357 -0.188653   \n",
       "2  0.453251  0.686791 -0.873488 -2.215446  0.123773 -0.202833  1.087286   \n",
       "3 -2.059316  0.237690 -0.112323 -0.905261 -1.232791  0.668653  1.176992   \n",
       "4  1.083504  0.571745 -0.557610 -0.633726 -1.366600  0.336196 -1.236697   \n",
       "\n",
       "         7         8         9   ...        11        12        13        14  \\\n",
       "0 -0.667330  1.257793  0.864782  ... -0.219467  0.138482  0.353048  0.675946   \n",
       "1 -0.085084 -1.176667  0.930242  ... -0.242841 -0.316790  0.068591  1.347992   \n",
       "2  0.707612 -0.341302  0.124591  ...  0.889354  0.593509 -0.020307 -0.253006   \n",
       "3 -0.143339  0.517313 -0.129502  ...  0.390874  0.737615  0.021002  0.586084   \n",
       "4 -0.080389 -0.515733  0.103772  ... -0.135934 -0.215368  0.118408  0.961872   \n",
       "\n",
       "         15        16        17        18        19        20  \n",
       "0  0.894278 -0.063213 -0.541114  0.501641 -1.250294 -0.987347  \n",
       "1  1.088595 -0.118375 -0.210533 -0.260355 -0.101061 -0.053715  \n",
       "2  0.723389  0.024207 -0.529978  1.448838 -0.125124  0.012569  \n",
       "3  0.649596  1.406424  0.460842  0.385251 -0.024130  0.019499  \n",
       "4  0.650594 -0.154942  0.021585 -0.872449  1.398305  0.297530  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca = PCA()\n",
    "x = pd.DataFrame(pca.fit_transform(x))\n",
    "x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "69da8ae1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmEAAAJNCAYAAAB5m6IGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdd3iV9cH/8c83m4QMslghYW+ZYSgiuHDU2WoVrRtxVH209nlqbWvtY9vraf21daLFugfDaiuOylCGVZkyAxISVgJkEbLIPuf7+wNqkYadk+8Z79d1eZUkR/gcWe/e933uY6y1AgAAQNsKcz0AAAAgFBFhAAAADhBhAAAADhBhAAAADhBhAAAADhBhAAAADkS4HnCiUlNTbffu3V3PAAAAOKZVq1aVWWvTWvpawEVY9+7dtXLlStczAAAAjskYs+NIX+N0JAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgANEGAAAgAM+izBjzEvGmBJjzIYjfN0YY54yxuQZY9YZY0b4agsAAIC/8eWRsFckXXiUr18kqc/Bf6ZKes6HWwAAAPyKzyLMWrtEUvlRHnK5pNfsAUslJRljOvtqDwAAgD+JcPhjd5VUcMjHhQc/t8fNHAAAECzqmzwqqWpQSXW9SqobVFJ18H//9U9VvS4c3En3n9fX2UaXEWZa+Jxt8YHGTNWBU5bKzMz05SYAAOCnrLWqqm9WafWBuCqtbjgstP797er65v/49yPCjFLbRys9IVoZHdqpc2KMg2dxyB6HP3ahpG6HfJwhaXdLD7TWTpc0XZKys7NbDDUAABC4PF6r4qp67SyvVUF5bQtHrw5EV32T9z/+3eiIMKUnRCs9PkZ9O8brzN6pSk+IUVp8tNLjD3w+PSFaybFRCgtr6RiQGy4jbI6ke4wxMyWNkVRpreVUJAAAQaqmoVk799Z+E1o7y//97cJ9dWr0fDuw4mMilB4frbT4aA3v1uFAUB2MrX99Oy0+RgkxETLGf+LqePkswowxMyRNlJRqjCmU9EtJkZJkrX1e0keSLpaUJ6lW0i2+2gIAAHzP47UqqqrXzr3fjqx/hdbe/Y3fenx8TISyUmLVv3O8zh/UUZnJscpMjlW3DrHqmBCjdlHhjp5J2/BZhFlrJx/j61bSD3314wMAgNZXXd+kgvK6b8JqR/l+7SyvO3g0q1ZNnn9fNRQeZtQlKUaZybGaNKijuiXHKis57pvYSoyNdPhM3HN5OhIAAPi5kup6fZm/V0u37tUX+Xu1Y2/tt77+r6NZAzsn6IJBnb4JrMzkWHVOilFkOG/OcyREGAAA+Ma+/Y1atu1AcH2Zv1dbSmokHYitMT1S9P3sbuqewtGs1kCEAQAQwqrrm7R8W7m+zD8QXpuKqmSt1C4yXKN6JOt7IzN0Rq8UDeqSqHA/emVhMCDCAAAIIXWNHq3cUf7Nka71uyrl8VpFRYRpZGYHPXBeX53RK0VDMpIUFcGpRF8iwgAACGINzR6t2VnxTXStLtinJo9VRJjR0G5JuntiL53eM0UjsjooJjK4X43ob4gwAACCSLPHq3W7KvXlwehauaNc9U1eGSMN7pKoW8f10Om9UjSqe7LioskAl/ivDwBAALPWKq+kRotzS/VF/l4t31aumoYDb9nTv1O8rh2VqTN6pWhMjxQuovczRBgAAAGmur5Jn+ft1eLcUi3JLdWuijpJUs/UOF0+rItO75WisT1TlNo+2vFSHA0RBgCAn7PWKmd3lRbnlmpxbqm+2rFPzV6r9tERGtc7RT88u7cm9EtT16R2rqfiBBBhAAD4oX37G/VZXpkWby7Vki2lKq1ukCQN7JygqWf11IS+aRqR1YGboQYwIgwAAD/g8VqtK6zQ4txSLdpcqrWFFbJWSoqN1Pg+aZrQN01n9UlVekKM66loJUQYAACOlFTXa0lumRbnluqzLaWqqG2SMdLQjCTdd04fTeyXpiEZSdwkNUgRYQAAtJEmj1df7dj3zbVdOburJEmp7aN1bv+OmtAvTeN7p6pDXJTjpWgLRBgAAD5UWt2gBZuKtWhziT7P26uahmaFhxmNzOqg/76gnyb0TdPAzgkK42hXyCHCAABoZZW1TZqbU6Q5a3fri/wyea3UJTFGlw7trAl903VG7xQlxHDPrlBHhAEA0Ar2NzRrwaZivb92txbnlqrJY5WZHKu7J/bWd4Z0Vv9O8TKGo134NyIMAICTVN/k0eLcUs1Zu1ufbCpWfZNXnRJidNPp3XXp0C4akpFIeOGIiDAAAE5Ak8erL/L3as6a3ZqXU6TqhmYlx0XpqpEZunRIF43qnsz1XTguRBgAAMfg9Vqt2F6uOWt36x8bilS+v1Hx0RG6YHAnXTq0i8b1SlEEN03FCSLCAABogbVW6worNWftbn24bo+KquoVExmm8wZ01GVDu+isvmmKiQx3PRMBjAgDAOAQm4uqNWftLr2/do92ltcqMtxoQt90PfydATq3f7riovmrE62DX0kAgJC3vWy/3l+7W++v263c4hqFGWlc71Tdc05vXTCwkxJjuZ0EWh8RBgAISZV1TXr3q0L9bfUurSuslCSN6t5Bj10+SBed1lmp7aMdL0SwI8IAACFl4+4qvb50h/6+epfqmjwa3DVBP7t4gL4zpLO6JLVzPQ8hhAgDAAS9hmaPPt5QpNe/3KGVO/YpOiJMlw/rohvGdtdpGYmu5yFEEWEAgKC1q6JOby3boVkrClRW06juKbH6+XcG6KqRGUqK5U2y4RYRBgAIKl6v1ef5ZXrtyx36ZFOxJOmc/h11w+lZGt87lRupwm8QYQCAoFBZ26S/flWoN5bu0Lay/UqJi9KdE3rpujGZyugQ63oe8B+IMABAQNuwq1JvLN2hv6/Zpfomr0ZkJum/rhmmi07rpOgIbqYK/0WEAQACTkOzRx+t36PXv9yhr3ZWKCYyTFcM66ofjM3S4K5caI/AQIQBAAJG4b5avblsp2atKFD5/kb1SI3TLy4ZqKtGZHBDVQQcIgwA4Ne8XqvP8sr0+pfb9enXJZKk8wYcuNB+XC8utEfgIsIAAH6psrZJb68q0BtLd2j73lqlxEXprom9dN2YLHXlpqoIAkQYAMCvlFY3aPqSfL2xdKfqmjzKzuqgB87vqwsHc6E9ggsRBgDwCyXV9frz4q16c9kONTZ7dfmwrrp9fE8N7JLgehrgE0QYAMCpkqp6Pbc4X28t26kmj1dXDO+qe87urZ5p7V1PA3yKCAMAOFFcVa/nFuVrxvKdavZaXXkwvrqnxrmeBrQJIgwA0KaKKuv13KI8zVhRII/X6nsjuuqHZ/dWVgrxhdBChAEA2sTuijo9tyhfs1YUyGutrhqZoR+e3VvdknlLIYQmIgwA4FO7Kur03KI8zV5RKK+1ujo7Q3dPJL4AIgwA4BOF+2o1bVG+3l5ZIEm6Orub7p7YizfTBg4iwgAAraqgvFbTFuXpr6sKZWR0zahuumtib26wChyGCAMAtIqde2v17MI8vfNVocKM0eTRmbpzQi91Ib6AFhFhAIBTsmPvfj3zaZ7eXb1L4WFGPxibpTsn9FKnxBjX0wC/RoQBAE7K9rL9emZhnv62epciwoxuGJuluyb2UscE4gs4HkQYAOCEFO6r1R/n5+q9NbsVEWZ00+nddeeEnkonvoATQoQBAI6Lx2v1yhfb9Yd5m+W1Vrec0V1TJ/RUejzxBZwMIgwAcEyb9lTpoXfWaW1hpc7ul6bHrhjMrSaAU0SEAQCOqL7Jo6c+2aLpS7YqsV2knrx2mC4b2kXGGNfTgIBHhAEAWvRFfpkefne9tu+t1VUjM/SziweoQ1yU61lA0CDCAADfUlnbpN9+tEmzVhYoKyVWb04Zo3G9U13PAoIOEQYAkCRZa/Xh+j16dM5G7att1J0Teum/zu2jdlHhrqcBQYkIAwBod0WdHnlvgxZsKtFpXRP1yi2jNLhroutZQFAjwgAghHm8Vm8s3aHff/y1vFb62cUDdMu47ooID3M9DQh6RBgAhKjc4mr95J11Wr2zQuP7pOq3V56mbsncdgJoK0QYAISY+iaPpi3M03OL89U+OkJ/umaorhjWldtOAG2MCAOAELJ8W7keenedtpbu15XDu+rn3xmglPbRrmcBIYkIA4AQUFnXpP/7x9easXynMjq006u3jtaEvmmuZwEhjQgDgCD38YY9euS9HJXVNGjKmT30o0l9FRvFH/+Aa/wuBIAgVVxVr0fe26C5OcUa2DlBf7kpW0MyklzPAnAQEQYAQcbrtXpr+U797h9fq9Hj1U8u7K8p43sokttOAH6FCAOAIJJXUqOfvrtOK7bv0xm9UvTbK09T99Q417MAtIAIA4AgYK3Vm8t26rEPNiomMlyPXzVEV43M4LYTgB8jwgAgwFXUNup//rpO8zYWa3yfVP3h+0OVHh/jehaAYyDCACCALd26Vw/MWqOymgb97OIBuu3MHgoL4+gXEAiIMAAIQM0er576ZIueWZinrJQ4vXvXOJ2WwRtuA4GECAOAAFNQXqv7Z63Rqh37dNXIDP3qskGKi+aPcyDQ8LsWAALIB+t266fvrpes9OS1w3T5sK6uJwE4SUQYAASA2sZmPTonR7NXFmp4ZpKeuna4uiXHup4F4BQQYQDg5zbsqtR9M1drW9l+/fDsXrr/vL7ceBUIAkQYAPgpa61e+ny7fvePr9UhLlJvThmjM3qlup4FoJUQYQDgh8pqGvTjt9dq0eZSnTego35/1RAlx0W5ngWgFRFhAOBnluSW6kez16qqvkmPXT5IPxibxZ3vgSBEhAGAn2hs9ur/zdus6Uu2qm/H9npjymj175TgehYAHyHCAMAPbCvbr/tmrNb6XZW6fkymfv6dgWoXFe56FgAfIsIAwCFrrd75apceeW+DIsPD9PwPRurCwZ1czwLQBogwAHCkur5JP//7Br23ZrdG90jWE9cMU5ekdq5nAWgjRBgAOLB65z7dN3O1dlfU68Hz++rus3srnDfeBkIKEQYAbcjjtXp+cb7+ND9XHRNiNPuOsRqZlex6FgAHiDAAaCPFVfV6YNYafZG/V5cM6azfXHmaEttFup4FwBEiDADawMKvS/Sj2WtU3+TV7783RFdnZ3DvLyDEEWEA4ENNHq/+MC9Xzy/O14DOCXp68nD1Tm/vehYAP0CEAYCP7Kms071vrdbKHft03ZhMPXLJQMVEcu8vAAcQYQDgAws3l+hHs9aosdmrJ68dpsuHdXU9CYCfIcIAoBU1e7z6w/xcPbcoX/07xWva9SPUM43TjwD+ExEGAK1kT2Wd7puxWiu279Pk0Zn65aWcfgRwZEQYALSCRZtL9KPZa1Xf5OH0I4DjQoQBwClo9nj1x/m5mnbw9OOz149QL04/AjgORBgAnKSiynrdN2O1lm8v1+TR3fTLSwdx+hHAcSPCAOAkLM4t1QOz1qi+yaMnrhmmK4Zz+hHAiSHCAOAENHu8+tOCXD27MF/9Oh44/cjNVwGcDCIMAI5TcVW97p2xWsu3levaUQdOP7aL4vQjgJNDhAHAcVhy8PRjXZNHf7pmqK4cnuF6EoAAR4QBwFE0e7x6YsEWPbsoT33TOf0IoPUQYQBwBMVVB179uGxbua7J7qZHL+P0I4DWQ4QBQAs+21Kq+2euUW2jR3/8/lB9dwSnHwG0LiIMAA7h8Vo9uSBXTy/MU5/09pp1/Qj1To93PQtAECLCAOCgkqp63TdztZZuLdf3szP0q8sGc/oRgM8QYQCgA6cfH5i1RvsbPPrD1UP1vZGcfgTgW0QYgJDm9Vo9/WmenvgkV73T2mvG7SPUpyOnHwH4HhEGIGRV1TfpR7PWasGmYn13eFf9+srBio3ij0UAbYM/bQCEpLySak19bZV2ltfqV5cN0o2nZ8kY43oWgBBChAEIOR9v2KMHZ69Vu6hwvXX7WI3ukex6EoAQRIQBCBker9Uf52/WswvzNaxbkp7/wUh1SoxxPQtAiCLCAISEytom3TdztRbnluraUd30q8sHKTqC208AcIcIAxD0vi6q0h2vr9Luijr99srTdN2YTNeTAIAIAxDcPli3W//99jrFx0Ro5tTTNTKrg+tJACCJCAMQpJo9Xj0+d7P+vGSrsrM6aNr1I5SewPVfAPxHmC+/c2PMhcaYzcaYPGPMQy18PdEY874xZq0xJscYc4sv9wAIDeX7G3XTy8v15yVbdcPYLL11+1gCDIDf8dmRMGNMuKRnJZ0vqVDSCmPMHGvtxkMe9kNJG621lxpj0iRtNsa8aa1t9NUuAMFtw65K3fH6KpXWNOj3Vw3R97O7uZ4EAC3y5enI0ZLyrLVbJckYM1PS5ZIOjTArKd4cuENie0nlkpp9uAlAEPvb6kI99M56JcdF6e07TtfQbkmuJwHAEfkywrpKKjjk40JJYw57zDOS5kjaLSle0jXWWq8PNwEIQk0er3770Sa9/Pl2jemRrGevH6HU9tGuZwHAUfkywlp6/w972McXSFoj6RxJvSTNN8Z8Zq2t+tZ3ZMxUSVMlKTOTl5YD+Leymgb98M2vtGxbuW4Z110PXzxAkeE+vdwVAFqFL/+kKpR06MUYGTpwxOtQt0h61x6QJ2mbpP6Hf0fW2unW2mxrbXZaWprPBgMILGsLKnTp0//UmoIK/emaofrlpYMIMAABw5d/Wq2Q1McY08MYEyXpWh049XionZLOlSRjTEdJ/SRt9eEmAEFi9soCXf3nLxVmjN656wxdOTzD9SQAOCE+Ox1prW02xtwjaa6kcEkvWWtzjDF3Hvz685Iek/SKMWa9Dpy+/Im1tsxXmwAEvsZmrx77YKNeX7pD43qn6OnJI5QcF+V6FgCcMJ/erNVa+5Gkjw773POHfHu3pEm+3AAgeJRU1evuN7/Syh37dMdZPfXfF/RTBKcfAQQo7pgPICCs2rFPd72xStX1zXp68nBdOrSL60kAcEqIMAB+zVqrt5bv1KNzctQ5sZ1eu220+ndKcD0LAE4ZEQbAbzV7vPrFezmasXynJvRN05PXDlNSLNd/AQgORBgAv1TX6NE9b32lT74u0V0Te+nHk/opPKyl2w8CQGAiwgD4ncraJt326gqt2rlPj10xWDeMzXI9CQBaHREGwK8UVdbrxpeWaXtZrZ69boQuPq2z60kA4BNEGAC/kVdSo5teWq7Kuia9cssondE71fUkAPAZIgyAX1hTUKFbXl6u8DCjmVPHanDXRNeTAMCniDAAzi3OLdVdb6xSavtovXbraHVPjXM9CQB8jggD4NR7a3bpwdlr1adjvF69dZTS42NcTwKANkGEAXDmpX9u0/9+sFFjeiTrhZuylRAT6XoSALQZIgxAm7PW6vG5mzVtUb4uHNRJT1w7TDGR4a5nAUCbIsIAtKlmj1cP/229Zq8s1OTRmfr1FYO5CSuAkESEAWgz9U0e3fPWai3YVKz7zumtB87vK2MIMAChiQgD0CYqa5s05bUVWrljn3512SDddEZ315MAwCkiDIDPFVfV68YXl2trWY2enjxclwzp4noSADhHhAHwqa2lNbrhxeWqqG3UyzeP1pl9uAs+AEhEGAAfWldYoZtfXiEjacbUsRqSkeR6EgD4DSIMgE98tqVUd76+Sh3iovT6bWPUg7vgA8C3EGEAWt2ctbv14Ow16pXWXq/eOlodE7gLPgAcjggD0Kpe+XybfvXBRo3KOnAX/MR23AUfAFpChAFoFdZa/XF+rp7+NE+TBnbUU5OHcxd8ADgKIgzAKWv2ePWL9zZoxvICXZPdTb+5crAiwsNczwIAv0aEATgl9U0e3TdjteZtLNY9Z/fWg5O4Cz4AHA8iDMBJq6xr0u2vrdTybeV69NKBunlcD9eTACBgEGEATkppdYNueHGZ8ktr9OS1w3T5sK6uJwFAQCHCAJywkqp6TX5hqXZX1OvFm0bprL5pricBQMAhwgCckD2VdbruhWUqqarXq7eO1ugeya4nAUBAIsIAHLddFXWaPH2pyvc36rXbRmtkFgEGACeLCANwXArKazX5haWqrGvS67eN1vDMDq4nAUBAI8IAHNOOvft13QvLVNPQrDenjOGNuAGgFRBhAI5qW9l+TZ6+VA3NHr05ZYwGd010PQkAggIRBuCI8ktrNHn6UjV7rd66fawGdE5wPQkAggYRBqBFW4qrNfmFZZKsZtw+Vv06xbueBABBhQgD8B++LqrS9S8sU1iY0Yzbx6p3OgEGAK2Nd9gF8C05uys1efpSRYaHadZUAgwAfIUjYQC+sWFXpa7/yzLFRYVrxtSxykqJcz0JAIIWEQZAkrS2oEI3vLhM8TGRmjl1rLolx7qeBABBjdORALRqxz794C/LlBQbpVl3EGAA0BY4EgaEuBXby3XzS8uVFh+tGVPHqnNiO9eTACAkEGFACFu6da9ufWWFOiXEaMbUseqYEON6EgCEDE5HAiHqi7wy3fzycnVJaqeZdxBgANDWiDAgBC3JLdUtr6xQVnKcZk4dq/R4AgwA2hqnI4EQs3Bzie54fZV6pbXXm1PGKDkuyvUkAAhJRBgQQhZsLNbdb36lvp3a643bxigplgADAFc4HQmEiI83FOmuN1dpQOd4vXnbWAIMABzjSBgQAj5av0f3zVit0zIS9eqto5UQE+l6EgCEPI6EAUFuztrdunfGag3rlqTXCDAA8BscCQOC2N9WF+rB2WuV3T1ZL988SnHR/JYHAH/BkTAgSL29skA/mr1WY3um6JVbCDAA8Df8qQwEobdXFuh/3lmnM3unavoN2WoXFe56EgDgMEQYEGQ+21Kqh95drzN7p+qFG7MVE0mAAYA/4nQkEERyi6t19xtfqU96e027fgQBBgB+jAgDgkRpdYNueXmFYqLC9eLNoxTPqyABwK8RYUAQqG/y6PbXVmrv/ga9eFO2uia1cz0JAHAMXBMGBDiv1+pHs9dobWGFnv/BSA3JSHI9CQBwHDgSBgS438/drI/WF+nhiwbogkGdXM8BABwnIgwIYDOX79Tzi/N1/ZhMTRnfw/UcAMAJIMKAAPXPLWX6+d836Ky+afrVZYNkjHE9CQBwAogwIABtKa7WXW+uUq+09nr2uuGKCOe3MgAEGv7kBgJMaXWDbnllhaIjwvXizdncigIAAhQRBgSQf92KoqzmwK0oMjrEup4EADhJ3KICCBBer9WDs9dqbWGFnrt+pIZ241YUABDIOBIGBIjH523Wh+v36KcX9deFg7kVBQAEOiIMCACzVxTouUX5mjw6U7eP7+l6DgCgFRBhgJ/7PK9MD/9tvcb3SdX/Xs6tKAAgWBBhgB/LK6nWnW+sUs+0OD17/QhFcisKAAga/IkO+Kmymgbd/PKBW1G8dPMoJXArCgAIKkQY4IcOvRXFX7gVBQAEJW5RAfgZr9fqwbfXavXOCj13/QgN41YUABCUOBIG+Jk/zN+sD9cduBXFRad1dj0HAOAjRBjgR2avLNCzC/M1eXQ3TT2LW1EAQDAjwgA/8UVemR5+91+3ohjMrSgAIMgRYYAfyCup0Z1vrFKPVG5FAQChgj/pAcf21jTolleWKyoijFtRAEAI4dWRgEP/uhVFSVWDZk4dq27J3IoCAEIFEQY44vVa/fjttfpqZ4WmXT9CwzM7uJ4EAGhDnI4EHPnj/Fx9sG6PfnJhf13MrSgAIOQQYYADb68s0DML83RNdjfdOYFbUQBAKCLCgDb2RX6Zfvrueo3rnaJfX8mtKAAgVBFhQBvaWlqjO18/cCuKadeP5FYUABDC+BsAaCOVtU2a8upKhYcZvXTzKCW241YUABDKeHUk0AaaPV7dM+MrFeyr1Ru3jeFWFAAAIgxoC7/+cJM+21Km333vNI3pmeJ6DgDAD3A6EvCxt5bt1CtfbNet43romlGZrucAAPwEEQb40Jf5e/XIexs0oW+aHr64v+s5AAA/QoQBPrJzb63uenOVslJi9fR1wxXBKyEBAIfgbwXAB6rrm3TbqytkrfTiTbwpNwDgP3FhPtDKPF6r+2as1tay/Xr91tHqnhrnehIAwA9xJAxoZb/7+Gst3FyqRy8bpDN6p7qeAwDwU0QY0IreXlmg6Uu26sbTs3TD2CzXcwAAfowIA1rJyu3levhvB94T8heXDHQ9BwDg54gwoBUUlNfqjtdXKaNDrKZdx3tCAgCOjb8pgFO0v6FZt7+2Uo0er164MVuJsbwSEgBwbEQYcAq8Xqv7Z61RbnG1nr1uhHqnt3c9CQAQIIgw4BT8v3mbNX9jsX5xyUCd1TfN9RwAQAAhwoCT9PfVuzRtUb4mj+6mm8/o7noOACDAEGHASVi9c5/+5511GtMjWb+6bLCMMa4nAQACDBEGnKDdFXWa+voqdUqI0XM/GKmoCH4bAQBOHG9bBJyA2sYDr4Ssa/TozSljlBwX5XoSACBAEWHAcfJ6rX789lpt3FOlF2/KVt+O8a4nAQACGOdRgOP05Cdb9NH6Ij180QCd07+j6zkAgABHhAHH4YN1u/XkJ1t01cgMTRnfw/UcAEAQIMKAY1hfWKkfv71W2Vkd9JsreSUkAKB1EGHAURRX1WvKayuUEhet528YqeiIcNeTAABBggvzgSOob/Jo6msrVV3frHfuOkOp7aNdTwIABBEiDGiBtVb/89d1WrerUn/+wUgN6JzgehIAIMhwOhJowbRF+Zqzdrd+PKmfJg3q5HoOACAI+TTCjDEXGmM2G2PyjDEPHeExE40xa4wxOcaYxb7cAxyPjzcU6fG5m3XFsC66e2Iv13MAAEHKZ6cjjTHhkp6VdL6kQkkrjDFzrLUbD3lMkqRpki601u40xqT7ag9wPHJ2V+qBWWs0tFuS/u97Q3glJADAZ3x5JGy0pDxr7VZrbaOkmZIuP+wx10l611q7U5KstSU+3AMcVVlNg25/daWSYiP1wg0jFRPJKyEBAL7jywjrKqngkI8LD37uUH0ldTDGLDLGrDLG3OjDPcAReb1WD8xao737G/XCjdlKT4hxPQkAEOR8+erIls7j2BZ+/JGSzpXUTtKXxpil1trcb31HxkyVNCoEfJ0AACAASURBVFWSMjMzfTAVoe6Fz7bqsy1l+s2VgzW4a6LrOQCAEODLI2GFkrod8nGGpN0tPOZja+1+a22ZpCWShh7+HVlrp1trs6212WlpaT4bjNC0tqBCj8/drIsGd9J1o4l8AEDb8GWErZDUxxjTwxgTJelaSXMOe8x7ksYbYyKMMbGSxkja5MNNwLdU1zfp3hmr1TEhRv/3XS7EBwC0HZ+djrTWNhtj7pE0V1K4pJestTnGmDsPfv15a+0mY8zHktZJ8kr6i7V2g682AYey1urnf9+gXRV1mjV1rBJjI11PAgCEEJ/eMd9a+5Gkjw773POHffy4pMd9uQNoyV9XFeq9Nbv14Pl9ld092fUcAECIOanTkcaYJGPMz1p7DNBW8ktr9Ms5ORrbM1l3n93b9RwAQAg6aoQZY7oZY6YbYz4wxkwxxsQaY/4gKVcSN1ZFQGpo9ui+GasVHRGmJ64ZrvAwrgMDALS9Y52OfE3SYknvSLpQ0lJJOZKGWGuLfLwN8In/+8fXytldpb/cmK1OidwPDADgxrEiLNla++jBb881xhRLGmWtbfDtLMA3PtlUrJc/366bz+iu8wZ2dD0HABDCjnlhvjGmg/5949UiSbHGmDhJstaW+3Ab0KqKq+r1339dpwGdE/TQRf1dzwEAhLhjRViipK8O+9y/PraSerb6IsAHPF6r+2euUV2jR89cN5z3hQQAOHfUCLPWdm+jHYBPPb84X19u3avfXzVEvdLau54DAMAxXx250RjzsDGGI14IWKt2lOuP83N16dAuunpkhus5AABIOvZ9wiZLipc03xizzBhzvzGmSxvsAlpFZV2T7puxRl2SYvSbKwfztkQAAL9x1Aiz1q611v7UWttL0n9JypK01BjzqTHm9jZZCJwka60efne9iqvq9dS1w5UQw9sSAQD8x3HfMd9au9Ra+4CkGyV1kPSMz1YBrWDmigJ9uH6PHpzUT8MzO7ieAwDAtxzXe0caY0bpwKnJ70naLmm6pLd9Nws4NVuKq/Wr93M0vk+q7jiLSxoBAP7nqBFmjPmtpGsk7ZM0U9I4a21hWwwDTlZ9k0f3zlit9tER+sP3hyqMtyUCAPihYx0Ja5B0kbU2ty3GAK3hNx9u0tdF1XrlllFKj+dtiQAA/ulY14TV/SvAjDFXH/qFg0fJAL/y8YYivb50h24f30MT+/Ee8wAA/3WsCLv2kG//9LCvXdjKW4BTsruiTj95Z52GZCTqvy/gbYkAAP7tWBFmjvDtlj4GnGn2eHX/zDVq9nj11LXDFRVx3C/8BQDAiWNdE2aP8O2WPgacefrTPC3fXq4/XTNU3VPjXM8BAOCYjhVhQ40xVTpw1KvdwW/r4Mdc8Qy/sHTrXj396RZ9d0RXXTmctyUCAASGY72Bd3hbDQFOxr79jXpg1hplpcTpfy8f7HoOAADH7bhu1gr4I2ut/ueddSqradC7d41T+2h+OQMAAgdXLyNgvb50h+ZvLNZPLuyv0zISXc8BAOCEEGEISJv2VOnXH27S2f3SdNuZPVzPAQDghBFhCDi1jc26d8ZqJbaL1ONXD5Ux3C0FABB4uIgGAed/39+o/NIavXHbGKW2j3Y9BwCAk8KRMASUD9bt1swVBbprQi+N653qeg4AACeNCEPAKCiv1U/fXa/hmUl64Py+rucAAHBKiDAEhCaPV/fNXC1Z6alrhysynF+6AIDAxjVhCAhPfbJFq3dW6JnrhqtbcqzrOQAAnDIOJ8Dv5ZfW6PnF+fruiK66ZEgX13MAAGgVRBj8mrVWv3p/o2IiwvXwxQNczwEAoNUQYfBr8zcWa0luqe4/vy+3owAABBUiDH6rvsmjxz7cqD7p7XXj6Vmu5wAA0Kq4MB9+64UlW1VQXqe3pozh1ZAAgKDD32zwS7sq6vTsojxdfFonncFNWQEAQYgIg1/67YebJImL8QEAQYsIg9/5Iq9MH67fo7sm9FZGB+4JBgAITkQY/EqTx6tH389RRod2umNCT9dzAADwGSIMfuWNpTuUW1yjX1wyUDGR4a7nAADgM0QY/EZZTYP+OD9X4/ukatLAjq7nAADgU0QY/MbjH29WXaNHv7x0kIwxrucAAOBTRBj8wtqCCs1eVaBbz+yh3untXc8BAMDniDA45/VaPTInR6nto3XvOb1dzwEAoE0QYXDuna8KtbagQg9d2F/xMZGu5wAA0CaIMDhVVd+k3338tUZkJunK4V1dzwEAoM3w3pFw6skFW7R3f6NeuWW0wsK4GB8AEDo4EgZnthRX69UvtuvaUZka3DXR9RwAANoUEQYnrLV69P0cxUaF68eT+rqeAwBAmyPC4MTHG4r0ed5ePTipn1LaR7ueAwBAmyPC0ObqGj369Yeb1L9TvK4fk+l6DgAATnBhPtrc84vztauiTjOnjlVEOP8/AAAQmvgbEG2qoLxWzy/O16VDu2hszxTXcwAAcIYIQ5v69YcbFWaMHr64v+spAAA4RYShzXy2pVRzc4p1zzm91Tmxnes5AAA4RYShTTR5vHp0To6yUmI1ZXwP13MAAHCOCEObePWL7cov3a9HLhmo6Ihw13MAAHCOCIPPlVTX64kFW3R2vzSdO6Cj6zkAAPgFIgw+97t/bFZDs0ePXDrI9RQAAPwGEQafWrVjn975qlC3ndlTPVLjXM8BAMBvEGHwGa/X6tE5OeqYEK17z+nteg4AAH6FCIPPzF5ZoPW7KvXwxQMUF82bMwAAcCgiDD5RWduk38/drNHdk3XZ0C6u5wAA4HeIMPjEnxbkqqK2Ub+8bKCMMa7nAADgd4gwtLqvi6r0+tIdum5MpgZ1SXQ9BwAAv0SEoVVZe+Bi/PiYCD14fj/XcwAA8FtEGFrVh+v3aOnWcv14Uj91iItyPQcAAL9FhKHV1DY26zcfbtKgLgmaPDrT9RwAAPwa9w1Aq5m2MF97Kuv19OThCg/jYnwAAI6GI2FoFTv27tf0JVt1xbAuyu6e7HoOAAB+jwhDq3jsg42KDDf66cUDXE8BACAgEGE4ZQs3l2jBphLde24fdUyIcT0HAICAQIThlDR5vHrs/Y3qmRqnW8f1cD0HAICAQYThlPx1VaG2lu3Xz74zQFER/HICAOB48bcmTlpDs0fPfJqn4ZlJOqd/uus5AAAEFCIMJ232ykLtqqjTA+f15f0hAQA4QUQYTkp9k0fTFuYpO6uDxvdJdT0HAICAQ4ThpMxaUaA9lfX60fkcBQMA4GQQYThh9U0ePbswT6N7JOv0Ximu5wAAEJCIMJywt5btVEl1A0fBAAA4BUQYTkhdo0fTFuXrjF4pGtuTo2AAAJwsIgwn5I2lO1RW06AHzu/regoAAAGNCMNxq21s1vOL8zW+T6pG8SbdAACcEiIMx+21L3do7/5G3X8eR8EAADhVRBiOS01Ds/68OF8T+qZpZFYH13MAAAh4RBiOy6tfbNe+2iauBQMAoJUQYTim6vomTV+yVef0T9ewbkmu5wAAEBSIMBzTK59vV2Vdkx7gWjAAAFoNEYajqqxr0gufbdX5AzvqtIxE13MAAAgaRBiO6qV/blNVfbPuP6+P6ykAAAQVIgxHVFnbpJf+uU0XDuqkQV04CgYAQGsiwnBEf/nnVlU3NOv+8zkKBgBAayPC0KJ9+xv10j+36TundVb/Tgmu5wAAEHSIMLTohc+2qrbJo//iWjAAAHyCCMN/2FvToFe+2K5Lh3RR347xrucAABCUiDD8h+lLtqq+yaP7zuUoGAAAvkKE4VtKqxv02pc7dPmwruqd3t71HAAAghYRhm/58+J8NXq8HAUDAMDHiDB8o6SqXq8v3aErhnVVj9Q413MAAAhqRBi+8dzifDV7re47t7frKQAABD0iDJKkosp6vblsp64akaGsFI6CAQDga0QYJEnTFuXJ67W65xyOggEA0BaIMGh3RZ1mLi/Q1dnd1C051vUcAABCAhEGPbswT1YcBQMAoC0RYSGucF+tZq8s0DWjuqlrUjvXcwAACBlEWIh7dmGejIx+eDZHwQAAaEtEWAgrKK/V2ysLdd2YTHVO5CgYAABtiQgLYU9/ukXhYUZ3TezlegoAACGHCAtR28v2652vdun6MVnqmBDjeg4AACHHpxFmjLnQGLPZGJNnjHnoKI8bZYzxGGOu8uUe/NtTn25RZLjRnRN7up4CAEBI8lmEGWPCJT0r6SJJAyVNNsYMPMLjfidprq+24Nu2ltbo76t36YaxWUqP5ygYAAAu+PJI2GhJedbardbaRkkzJV3ewuPulfSOpBIfbsEhnvpki6IjwnXHBK4FAwDAFV9GWFdJBYd8XHjwc98wxnSVdKWk5324A4fIK6nWe2t366Yzuiu1fbTrOQAAhCxfRphp4XP2sI+fkPQTa63nqN+RMVONMSuNMStLS0tbbWAoevKTPMVGhmvqWVwLBgCASxE+/L4LJXU75OMMSbsPe0y2pJnGGElKlXSxMabZWvv3Qx9krZ0uabokZWdnHx5yOE6bi6r1wbrduntiLyXHRbmeAwBASPNlhK2Q1McY00PSLknXSrru0AdYa3v869vGmFckfXB4gKH1PPlJruKiInT7eI6CAQDgms9OR1prmyXdowOvetwkaba1NscYc6cx5k5f/bho2aY9VfpofZFuHdddSbEcBQMAwDVfHgmTtfYjSR8d9rkWL8K31t7syy2h7okFuYqPidBtZ3IUDAAAf8Ad80PAhl2VmptTrCln9lRibKTrOQAAQERYSHhiwRYlxEToljO7u54CAAAOIsKC3LrCCi3YVKypZ/VUQgxHwQAA8BdEWJB7YsEWJcVG6uZxPY79YAAA0GaIsCC2euc+ffp1iaae1VPto336GgwAAHCCiLAg9tQnW9QhNlI3nd7d9RQAAHAYIixIrS+s1MLNpZoyvqfiOAoGAIDfIcKC1DMLD7wi8sbTs1xPAQAALSDCgtDXRVWam1OsW8b1UDyviAQAwC8RYUHomU/z1D46QrfyikgAAPwWERZk8kpq9OH6Pbrx9Czujg8AgB8jwoLMtEV5iokI121nchQMAAB/RoQFkZ17a/Xemt26fkymUtpHu54DAACOgggLItMW5Sk8zGjqWT1dTwEAAMdAhAWJXRV1euerQk0e1U3pCTGu5wAAgGMgwoLE84vyJUl3TOjleAkAADgeRFgQKK6q16yVBbpqZIa6JLVzPQcAABwHIiwITF+yVR6v1V0TerueAgAAjhMRFuDKahr05rIdumJYV2WmxLqeAwAAjhMRFuD+8tk2NTR79cOzuRYMAIBAQoQFsIraRr3+5XZdMqSLeqa1dz0HAACcACIsgL30+Xbtb/TonrO5FgwAgEBDhAWoqvomvfz5Nl04qJP6dYp3PQcAAJwgIixAvfbFdlXXN+ueczgKBgBAICLCAtD+hma9+M9tOqd/ugZ3TXQ9BwAAnAQiLAC9uWyH9tU26V6OggEAELCIsABT3+TR9CXbNL5PqoZndnA9BwAAnCQiLMDMWL5TZTUNuvecPq6nAACAU0CEBZCGZo/+vHirxvRI1ugeya7nAACAU0CEBZC3VxaqqKqeo2AAAAQBIixANHm8em5RvoZnJmlc7xTXcwAAwCkiwgLE31bv0q6KOt13Th8ZY1zPAQAAp4gICwDNHq+mLczT4K4JmtgvzfUcAADQCoiwAPDBuj3avrdW95zNUTAAAIIFEebnvF6rZxbmqV/HeE0a2NH1HAAA0EqIMD/3cU6R8kpqdM85vRUWxlEwAACCBRHmx6y1evrTPPVMi9PFp3V2PQcAALQiIsyPLdhUok17qnTP2b0VzlEwAACCChHmpw4cBduizORYXTa0i+s5AACglRFhfmrJljKtK6zU3RN7KSKcnyYAAIINf7v7IWutnv5ki7omtdN3R2S4ngMAAHyACPNDX27dq5U79unOCT0VFcFPEQAAwYi/4f3Q05/kKT0+Wldnd3M9BQAA+AgR5mdWbi/Xl1v3aupZPRUTGe56DgAA8BEizM88/WmeUuKidP2YLNdTAACADxFhfmRtQYUW55ZqyvieahfFUTAAAIIZEeZHnv40T4ntInXD6RwFAwAg2BFhfmLj7iot2FSsW8f1UPvoCNdzAACAjxFhfuLZhXmKj47QzeO6u54CAADaABHmB/JKqvXRhj266YzuSmwX6XoOAABoA0SYH3jm0zy1iwzXrWf2cD0FAAC0ESLMsW1l+zVn7W79YGyWkuOiXM8BAABthAhz7LlFeYoMD9OU8RwFAwAglBBhDhWU1+rdr3Zp8uhMpcfHuJ4DAADaEBHm0POL8xVmjO6c0Mv1FAAA0MaIMEeKKuv19spCXZ2doU6JHAUDACDUEGGO/HlJvrzWchQMAIAQRYQ50OTx6q+rCnXJkM7qlhzreg4AAHCACHNg6da9qq5v1neGdHE9BQAAOEKEOTAvp1jtIsM1vk+q6ykAAMARIqyNeb1W8zcW66y+qYqJDHc9BwAAOEKEtbH1uypVVFWvCwZ1cj0FAAA4RIS1sbk5RQoPMzqnf7rrKQAAwCEirI3N21isMT2SlRTL+0QCABDKiLA2lF9ao7ySGk5FAgAAIqwtzd9YLEk6f2BHx0sAAIBrRFgbmptTpNO6JqpLUjvXUwAAgGNEWBspqarX6p0VmsRRMAAAICKszczfdOBU5AWDuR4MAAAQYW1mbk6xuqfEqk96e9dTAACAHyDC2kBVfZO+zC/TpEGdZIxxPQcAAPgBIqwNLNpcqiaP5XowAADwDSKsDczLKVJq+2gNz+zgegoAAPATRJiPNTR7tGhzqc4fmK7wME5FAgCAA4gwH/sif69qGpo1aSCvigQAAP9GhPnYvJxixUWF64zeKa6nAAAAP0KE+ZDXazV/Y7Em9k9XdES46zkAAMCPEGE+tLpgn8pqGnhVJAAA+A9EmA/NyylWZLjR2f3TXU8BAAB+hgjzEWut5uYU6fReqUqIiXQ9BwAA+BkizEe2lNRo+95aTkUCAIAWEWE+Mi+nSJJ0PhEGAABaQIT5yLyNxRrWLUkdE2JcTwEAAH6ICPOB3RV1WldYqQsGcYNWAADQMiLMB+ZvLJYkTRrEqUgAANAyIswH5m0sUq+0OPVKa+96CgAA8FNEWCurrG3S0q3lnIoEAABHRYS1sk83F8vjtZpEhAEAgKMgwlrZ3A3F6pgQrSFdE11PAQAAfowIa0X1TR4tzi3V+QM7KizMuJ4DAAD8GBHWiv65pUx1TR6uBwMAAMdEhLWieRuLFB8ToTE9UlxPAQAAfo4IayXNHq8WbCrROf3TFRXBf1YAAHB01EIrWbVjn8r3N2rSQE5FAgCAYyPCWsm8jcWKigjThH5prqcAAIAAQIS1Amut5uYU6czeqWofHeF6DgAACABEWCvYtKdahfvqNGkg7xUJAACODxHWCuZtLJIx0nlEGAAAOE5EWCuYl1Os7KwOSm0f7XoKAAAIEETYKSoor9XGPVW8KhIAAJwQIuwUzdtYLEmaNIhTkQAA4PgRYadoXk6R+neKV1ZKnOspAAAggBBhp6B8f6NWbC/nVZEAAOCEEWGnYMGmYnmtNIk37AYAACeICDsF83KK1TWpnQZ1SXA9BQAABBgi7CTVNjbrsy2lOn9gRxljXM8BAAABhgg7SUtyy9TQ7OVVkQAA4KQQYSdpXk6RkmIjNbp7suspAAAgAPk0wowxFxpjNhtj8owxD7Xw9euNMesO/vOFMWaoL/e0liaPV598XaJz+3dURDgdCwAATpzPCsIYEy7pWUkXSRooabIxZuBhD9smaYK1doikxyRN99We1rRiW7kq65o4FQkAAE6aLw/jjJaUZ63daq1tlDRT0uWHPsBa+4W1dt/BD5dKyvDhnlYzN6dIMZFhOqtPmuspAAAgQPkywrpKKjjk48KDnzuS2yT9w4d7WoW1VvM2Fmt8nzS1iwp3PQcAAAQoX0ZYS/dtsC0+0JizdSDCfnKEr081xqw0xqwsLS1txYknbsOuKu2prNcF3KAVAACcAl9GWKGkbod8nCFp9+EPMsYMkfQXSZdba/e29B1Za6dba7OttdlpaW5PAc7bWKQwI53bP93pDgAAENh8GWErJPUxxvQwxkRJulbSnEMfYIzJlPSupBustbk+3NJq5uYUaXSPZHWIi3I9BQAABDCfRZi1tlnSPZLmStokaba1NscYc6cx5s6DD3tEUoqkacaYNcaYlb7a0xq2le1XbnGNJg3kVCQAADg1Eb78zq21H0n66LDPPX/It6dImuLLDa1p/sYiSeLWFAAA4JRxp9ETMC+nWIO6JCijQ6zrKQAAIMARYceptLpBq3bu41QkAABoFUTYcVqwqVjWShcM5lQkAAA4dUTYcZqXU6TM5Fj16xjvegoAAAgCRNhxqGlo1ud5ezVpYEcZ09I9aAEAAE4MEXYcFm0uUaPHq0ncJR8AALQSIuw4zMspVkpclEZmdXA9BQAABAki7Bgam71a+HWJzhvQUeFhnIoEAACtgwg7hi+37lV1QzM3aAUAAK2KCDuGeTlFio0K17jeqa6nAACAIEKEHYXXazV/Y7Em9ktTTGS46zkAACCIEGFHsbawQiXVDdwlHwAAtDoi7Cjm5hQrIszo7H7prqcAAIAgQ4QdxbyNRTq9V4oSYyNdTwEAAEGGCDuCvJIabS3dr0kDeVUkAABofUTYEczbWCRJOo8IAwAAPkCEHcHcnGINzUhU58R2rqcAAIAgRIS1oKiyXmsLKnivSAAA4DNEWAvmbyqWJF3AXfIBAICPEGEtmJdTpJ6pceqV1t71FAAAEKSIsMNU1jXpy/y9On9QRxnDG3YDAADfIMIOU1Beq64d2ukCrgcDAAA+FOF6gL8Z3DVRi3480fUMAAAQ5IiwFnAaEgAA+BqnIwEAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABwgwgAAABww1lrXG06IMaZU0o42+KFSJZW1wY/jj3juoSuUn38oP3cptJ8/zz10tcXzz7LWprX0hYCLsLZijFlprc12vcMFnntoPncptJ9/KD93KbSfP889NJ+75P75czoSAADAASIMAADAASLsyKa7HuAQzz10hfLzD+XnLoX28+e5hy6nz59rwgAAABzgSBgAAIADIR1hxpgLjTGbjTF5xpiHWvi6McY8dfDr64wxI1zs9AVjTDdjzEJjzCZjTI4x5r9aeMxEY0ylMWbNwX8ecbHVF4wx240x6w8+r5UtfD2Yf+77HfJzusYYU2WMuf+wxwTNz70x5iVjTIkxZsMhn0s2xsw3xmw5+L8djvDvHvXPiEBwhOf/uDHm64O/tv9mjEk6wr971N8n/u4Iz/1RY8yuQ35tX3yEfzegf+6P8NxnHfK8txtj1vz/9u421o6ijuP492dbLCCCtRErYApIjFa0LQgFS5WHqEXTgqAgKlVJSdFigJjYpEljYniBoLwwhCZA02oaMI2ALYFYI2CNeCuh9Pbeplge2mhDvajV8hSFtn9f7By7Lrv3njb3noc9v09ycufszOzO3DmzO2d3z05F3m5v99LjW0f2+4joyRcwDngeOAU4AugHPlxIczHwCCBgFrCx3eUexfpPAWam8DHA9pL6fwp4qN1lHaP67wQmDxNf27Yv1HMc8Fey59jUsu2BOcBMYDC37IfAkhReAtxS8b8Zdh/RDa+K+n8aGJ/Ct5TVP8UN2086/VVR9+8D3x0hX9e3fVndC/E/ApbVtN1Lj2+d2O97+UzYWcBzEfFCRLwB3AfML6SZD/w0Mn3AcZKmtLqgYyEidkfEphR+BdgGnNDeUnWU2rZ9wYXA8xHRigcgt0VEbAD2FBbPB1al8CrgkpKszewjOl5Z/SNifUTsS2/7gBNbXrAWqGj7ZnR92w9Xd0kCvgTc29JCtcgwx7eO6/e9PAg7AfhL7v0u3joIaSZN15M0FZgBbCyJPkdSv6RHJE1racHGVgDrJT0l6dqS+J5oe+BKqnfEdW17gOMjYjdkO2zgPSVpeuUz8E2ys75lRuon3WpxuhS7ouKSVN3b/jxgKCKerYivTbsXjm8d1+97eRCmkmXFn4o2k6arSXoH8Avghoh4uRC9iewy1ceAnwAPtrp8Y+gTETETmAt8W9KcQnwvtP0RwDxgTUl0ndu+Wb3wGVgK7ANWVyQZqZ90ozuBU4HpwG6yy3JFdW/7LzP8WbBatPsIx7fKbCXLxqzte3kQtgs4Kff+RODFw0jTtSRNIPuAro6I+4vxEfFyRLyawg8DEyRNbnExx0REvJj+vgQ8QHYKOq/WbZ/MBTZFxFAxos5tnww1Li+nvy+VpKn1Z0DSAuDzwFci3QxT1EQ/6ToRMRQR+yPiAHAX5XWqbdtLGg98Afh5VZo6tHvF8a3j+n0vD8KeBE6TdHI6I3AlsLaQZi1wdfql3Cxgb+NUZrdL9wTcA2yLiB9XpHlvSoeks8g+L/9oXSnHhqSjJR3TCJPdpDxYSFbbts+p/DZc17bPWQssSOEFwC9L0jSzj+hKkj4LfA+YFxGvV6Rppp90ncK9nZdSXqfatj1wEfBMROwqi6xDuw9zfOu8ft+OXy50yovsF3DbyX4JsTQtWwQsSmEBd6T4AeDMdpd5FOs+m+wU6xZgc3pdXKj/YmAr2a9D+oBz213uUar7KalO/al+PdX2qX5HkQ2qjs0tq2Xbkw00dwNvkn3LvQZ4N/Ab4Nn0d1JK+z7g4Vzet+wjuu1VUf/nyO57afT95cX6V/WTbnpV1P1nqU9vITu4Tqlj25fVPS1f2ejnubR1a/eq41vH9Xs/Md/MzMysDXr5cqSZmZlZ23gQZmZmZtYGHoSZmZmZtYEHYWZmZmZt4EGYmZmZWRt4EGZmHU/S3+2hZgAAA3pJREFU45LObMF2viNpm6SqJ8g3u56WlNfMutv4dhfAzGwsSRofByerHsm3gLkRsWMsy2RmBj4TZmajRNLUdBbpLklbJa2XdGSK+9+ZIUmTJe1M4a9LelDSOkk7JC2WdJOkpyX1SZqU28RXJT0haTA9xb/xdO8Vkp5Meebn1rtG0jpgfUlZb0rrGZR0Q1q2nOxBlWsl3VhIP07SbZIG0sTP16flF6btDqRyvL1kW6/mwpdLWpnCKyXdKekxSS9I+mRax7ZGmkZ+STcrm0y9T9LxafkXU/n7JW04xOYysw7gQZiZjabTgDsiYhrwL+CyJvJ8BLiKbH66m4HXI2IG8Afg6ly6oyPiXLKzVSvSsqXAoxHxceB84NY01QrAOcCCiLggvzFJZwDfAM4GZgELJc2IiEVkc8SdHxG3F8p4LXAyMCMiPgqsljSR7OnjV0TE6WRXFq5ror557wIuAG4E1gG3A9OA0yVNb9Qb6ItsMvUNwMK0fBnwmbR83iFu18w6gAdhZjaadkTE5hR+CpjaRJ7HIuKViPgbsJdsMALZ1DL5/PcCRMQG4J2SjiOb126JpM3A48BE4P0p/a8jYk/J9mYDD0TEa5FNUn4/cN4IZbyIbGqffakMe4APpvpuT2lWAXOaqG/eusimLRkAhiJiILKJpbdysO5vAA+lcP5/+ntgpaSFwLhD3K6ZdQDfE2Zmo+k/ufB+4MgU3sfBL30Th8lzIPf+AP+/jyrOsRZkc3xeFhF/ykdIOht4raKMqir8MFSy/WbXk89XVfd8vRvvG3V/Mw7OL7e/sTwiFqV6fg7YLGl6RNRpknWz2vOZMDNrhZ3AGSl8+WGu4woASbOBvRGxF/gVcL0kpbgZTaxnA3CJpKPSpctLgd+NkGc9sEjS+LSdScAzwFRJH0hpvgb8tiTvkKQPSXpb2taokHRqRGyMiGXA34GTRmvdZtYaHoSZWSvcBlwn6Qlg8mGu458p/3LgmrTsB8AEYIukwfR+WBGxiexerj8CG4G7I+LpEbLdDfw5bacfuCoi/k12b9kaSQNkZ6+Wl+RdQnY58VFg90jlOwS3ph8EDJINLPtHcd1m1gI6eJbbzMzMzFrFZ8LMzMzM2sCDMDMzM7M28CDMzMzMrA08CDMzMzNrAw/CzMzMzNrAgzAzMzOzNvAgzMzMzKwNPAgzMzMza4P/AlKbnElcwa8CAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "pca=PCA()\n",
    "pca.fit_transform(x)\n",
    "plt.figure(figsize=(10,10))\n",
    "plt.plot(np.cumsum(pca.explained_variance_ratio_))\n",
    "plt.xlabel('number of columns')\n",
    "plt.ylabel('EVR')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "4b626f75",
   "metadata": {},
   "outputs": [],
   "source": [
    "pca1 = PCA(n_components=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "c5d142d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data1=pca1.fit_transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "312e4e51",
   "metadata": {},
   "outputs": [],
   "source": [
    "s=pd.DataFrame(scalar.transform(np.array([[-1,1,1,-1,-1,-1,-1,-1,1,-1,1,-1,-1,0,1,-1,-1,-1,-1,1,1]])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "2852df72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.3836833 ,  2.13194274,  0.38761387, -2.59541502, -0.39084945,\n",
       "        -1.30150149, -1.37185464, -0.70437413,  0.82777619, -1.29138103,\n",
       "         1.46364816, -0.53253435, -2.40548789, -0.36170514,  0.21211978,\n",
       "        -1.06323483]])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca1.transform(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "7700eeab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca1.n_components_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "8277a43a",
   "metadata": {},
   "outputs": [],
   "source": [
    "x1=new_data1\n",
    "y1=y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "5d89057e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.240857  ,  2.79013417, -2.46065967, ...,  0.35304847,\n",
       "         0.67594644,  0.89427795],\n",
       "       [ 0.65831762, -0.22463916, -1.08006927, ...,  0.06859134,\n",
       "         1.34799202,  1.08859512],\n",
       "       [ 0.45325053,  0.68679073, -0.87348813, ..., -0.02030681,\n",
       "        -0.25300597,  0.72338872],\n",
       "       ...,\n",
       "       [ 0.23925522, -1.09775535, -0.2651509 , ..., -0.23152525,\n",
       "        -0.42969536,  0.22533361],\n",
       "       [-1.85935573,  0.7331113 ,  0.83883434, ...,  0.02109229,\n",
       "        -0.02530897, -0.22784032],\n",
       "       [-2.94541874,  1.0056408 , -0.61802707, ...,  0.28145561,\n",
       "         0.23192245,  0.0297858 ]])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "aebbd843",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test= train_test_split(x1,y1,random_state=10,test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "032b903c",
   "metadata": {},
   "outputs": [],
   "source": [
    "model1=RandomForestClassifier(bootstrap=True,\n",
    "                             random_state=10,                                                     \n",
    "                             max_depth=49,\n",
    "                             n_estimators=800,\n",
    "                             max_features='auto',\n",
    "                             min_samples_split=3,\n",
    "                             min_samples_leaf=1)\n",
    "model1.fit(X_train,y_train)\n",
    "yhat=model1.predict(X_test)\n",
    "ytr=model1.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "e79dca19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy 0.9871099050203528\n",
      "Testing accuracy 0.9638172772501131\n"
     ]
    }
   ],
   "source": [
    "print(\"Training accuracy\",accuracy_score(y_train,ytr))\n",
    "print(\"Testing accuracy\",accuracy_score(y_test,yhat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "320fea75",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "b529c7d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(pca1,open('pca1.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "4c38900f",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(scalar ,open('scalar.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "9fc655d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(model1, open('model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "e6929454",
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_pred= pickle.load(open('pca1.pkl','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "fa64d6a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler_pred= pickle.load(open('scalar.pkl','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "321a08ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_pred= pickle.load(open('model.pkl','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "2417834c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1], dtype=int64)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_pred.predict(pca_pred.transform(scaler_pred.transform([[-1,1,1,-1,-1,-1,-1,-1,1,-1,1,-1,-1,0,1,-1,-1,-1,-1,1,1]])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "ee62277e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "col_0    -1     1\n",
      "Result           \n",
      "-1      912    49\n",
      " 1       31  1219\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.97      0.95      0.96       961\n",
      "           1       0.96      0.98      0.97      1250\n",
      "\n",
      "    accuracy                           0.96      2211\n",
      "   macro avg       0.96      0.96      0.96      2211\n",
      "weighted avg       0.96      0.96      0.96      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(pd.crosstab(y_test,yhat))\n",
    "print(\"\\n\")\n",
    "print(classification_report(y_test,yhat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "4d967c5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "f9ff8ecb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
       "              learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
       "              min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,\n",
       "              nthread=None, objective='binary:logistic', random_state=0,\n",
       "              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "              silent=None, subsample=1, verbosity=1)"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fit model no training data\n",
    "model = XGBClassifier()\n",
    "model.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "bc266ae5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9481004070556309"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict(X_train)\n",
    "predictions = [round(value) for value in y_pred]\n",
    "accuracy = accuracy_score(y_train,predictions)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "8162877c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9384893713251922"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict(X_test)\n",
    "predictions = [round(value) for value in y_pred]\n",
    "accuracy = accuracy_score(y_test,predictions)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "f42bd002",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "f1f989c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid={\n",
    "   \n",
    "    ' learning_rate':[1,0.5,0.1,0.01,0.001],\n",
    "    'max_depth': [3,5,10,20],\n",
    "    'n_estimators':[10,50,100,200]\n",
    "    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "eacdfbc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "grid= GridSearchCV(XGBClassifier(),param_grid, verbose=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "7349621e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 80 candidates, totalling 400 fits\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=10 ..................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=10, score=0.886, total=   0.1s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=10 ..................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]   learning_rate=1, max_depth=3, n_estimators=10, score=0.889, total=   0.1s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=10 ..................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=10, score=0.880, total=   0.1s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=10 ..................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.2s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]   learning_rate=1, max_depth=3, n_estimators=10, score=0.892, total=   0.1s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=10 ..................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=10, score=0.886, total=   0.1s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=50 ..................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=50, score=0.918, total=   0.6s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=50 ..................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=50, score=0.921, total=   0.5s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=50 ..................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=50, score=0.930, total=   0.6s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=50 ..................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=50, score=0.930, total=   0.5s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=50 ..................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=50, score=0.929, total=   0.5s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=100 .................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=100, score=0.932, total=   1.1s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=100 .................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=100, score=0.936, total=   1.3s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=100 .................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=100, score=0.942, total=   1.1s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=100 .................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=100, score=0.945, total=   1.2s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=100 .................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=100, score=0.941, total=   1.1s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=200 .................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=200, score=0.942, total=   2.2s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=200 .................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=200, score=0.946, total=   2.1s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=200 .................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=200, score=0.949, total=   2.1s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=200 .................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=200, score=0.953, total=   2.1s\n",
      "[CV]  learning_rate=1, max_depth=3, n_estimators=200 .................\n",
      "[CV]   learning_rate=1, max_depth=3, n_estimators=200, score=0.949, total=   2.1s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=10 ..................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=10, score=0.910, total=   0.2s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=10 ..................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=10, score=0.915, total=   0.2s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=10 ..................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=10, score=0.916, total=   0.2s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=10 ..................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=10, score=0.930, total=   0.2s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=10 ..................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=10, score=0.922, total=   0.2s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=50 ..................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=50, score=0.943, total=   0.9s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=50 ..................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=50, score=0.940, total=   0.8s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=50 ..................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=50, score=0.950, total=   0.8s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=50 ..................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=50, score=0.949, total=   0.8s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=50 ..................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=50, score=0.945, total=   0.8s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=100 .................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=100, score=0.953, total=   1.7s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=100 .................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=100, score=0.953, total=   1.7s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=100 .................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=100, score=0.956, total=   1.6s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=100 .................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=100, score=0.962, total=   1.6s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=100 .................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=100, score=0.958, total=   1.7s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=200 .................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=200, score=0.958, total=   3.3s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=200 .................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=200, score=0.955, total=   3.2s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=200 .................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=200, score=0.958, total=   3.2s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=200 .................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=200, score=0.964, total=   3.2s\n",
      "[CV]  learning_rate=1, max_depth=5, n_estimators=200 .................\n",
      "[CV]   learning_rate=1, max_depth=5, n_estimators=200, score=0.960, total=   3.3s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=10 .................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=10, score=0.942, total=   0.4s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=10 .................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=10, score=0.947, total=   0.3s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=10 .................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=10, score=0.939, total=   0.3s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=10 .................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=10, score=0.953, total=   0.3s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=10 .................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=10, score=0.945, total=   0.3s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=50 .................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=50, score=0.951, total=   1.5s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=50 .................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=50, score=0.959, total=   2.0s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=50 .................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=50, score=0.956, total=   1.8s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=50 .................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=50, score=0.963, total=   1.7s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=50 .................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=50, score=0.955, total=   1.6s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=100 ................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=100, score=0.956, total=   3.0s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=100 ................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=100, score=0.959, total=   3.0s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=100 ................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=100, score=0.957, total=   3.0s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=100 ................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=100, score=0.967, total=   2.9s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=100 ................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=100, score=0.958, total=   3.4s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=200 ................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=200, score=0.959, total=   6.0s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=200 ................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=200, score=0.957, total=   5.9s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=200 ................\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]   learning_rate=1, max_depth=10, n_estimators=200, score=0.959, total=   6.1s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=200 ................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=200, score=0.967, total=   6.0s\n",
      "[CV]  learning_rate=1, max_depth=10, n_estimators=200 ................\n",
      "[CV]   learning_rate=1, max_depth=10, n_estimators=200, score=0.959, total=   5.8s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=10 .................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=10, score=0.945, total=   0.4s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=10 .................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=10, score=0.949, total=   0.4s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=10 .................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=10, score=0.943, total=   0.4s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=10 .................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=10, score=0.951, total=   0.4s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=10 .................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=10, score=0.945, total=   0.4s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=50 .................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=50, score=0.954, total=   2.0s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=50 .................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=50, score=0.956, total=   2.3s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=50 .................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=50, score=0.959, total=   2.0s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=50 .................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=50, score=0.967, total=   2.3s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=50 .................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=50, score=0.956, total=   2.1s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=100 ................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=100, score=0.955, total=   4.3s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=100 ................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=100, score=0.957, total=   3.7s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=100 ................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=100, score=0.959, total=   4.7s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=100 ................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=100, score=0.967, total=   4.4s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=100 ................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=100, score=0.958, total=   3.6s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=200 ................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=200, score=0.957, total=   6.9s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=200 ................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=200, score=0.957, total=   7.1s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=200 ................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=200, score=0.958, total=   6.9s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=200 ................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=200, score=0.968, total=   7.1s\n",
      "[CV]  learning_rate=1, max_depth=20, n_estimators=200 ................\n",
      "[CV]   learning_rate=1, max_depth=20, n_estimators=200, score=0.959, total=   7.2s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=10, score=0.886, total=   0.2s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=10, score=0.889, total=   0.2s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=10, score=0.880, total=   0.2s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=10, score=0.892, total=   0.2s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=10, score=0.886, total=   0.2s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=50, score=0.918, total=   0.6s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=50, score=0.921, total=   0.6s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=50, score=0.930, total=   0.6s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=50, score=0.930, total=   0.6s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=50, score=0.929, total=   0.6s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=100, score=0.932, total=   1.0s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=100, score=0.936, total=   1.1s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=100, score=0.942, total=   1.2s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=100, score=0.945, total=   1.2s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=100, score=0.941, total=   1.2s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=200, score=0.942, total=   2.3s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=200, score=0.946, total=   2.8s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=200, score=0.949, total=   2.2s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=200, score=0.953, total=   2.6s\n",
      "[CV]  learning_rate=0.5, max_depth=3, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=3, n_estimators=200, score=0.949, total=   2.1s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=10, score=0.910, total=   0.2s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=10, score=0.915, total=   0.3s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=10, score=0.916, total=   0.4s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=10, score=0.930, total=   0.4s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=10, score=0.922, total=   0.4s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=50, score=0.943, total=   1.2s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=50, score=0.940, total=   0.8s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=50, score=0.950, total=   0.8s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=50, score=0.949, total=   0.9s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=50, score=0.945, total=   1.5s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=100, score=0.953, total=   1.7s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=100 ...............\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=100, score=0.953, total=   2.2s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=100, score=0.956, total=   1.8s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=100, score=0.962, total=   2.1s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=100, score=0.958, total=   1.8s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=200, score=0.958, total=   3.9s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=200, score=0.955, total=   4.2s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=200, score=0.958, total=   4.0s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=200, score=0.964, total=   4.6s\n",
      "[CV]  learning_rate=0.5, max_depth=5, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=5, n_estimators=200, score=0.960, total=   4.2s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=10, score=0.942, total=   0.6s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=10, score=0.947, total=   0.4s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=10, score=0.939, total=   0.4s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=10, score=0.953, total=   0.4s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=10, score=0.945, total=   0.4s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=50, score=0.951, total=   1.8s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=50, score=0.959, total=   2.0s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=50, score=0.956, total=   1.6s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=50, score=0.963, total=   2.0s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=50, score=0.955, total=   1.8s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=100, score=0.956, total=   3.0s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=100, score=0.959, total=   3.5s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=100, score=0.957, total=   3.6s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=100, score=0.967, total=   3.6s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=100, score=0.958, total=   3.6s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=200, score=0.959, total=   6.2s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=200, score=0.957, total=   6.3s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=200, score=0.959, total=   6.8s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=200, score=0.967, total=   6.7s\n",
      "[CV]  learning_rate=0.5, max_depth=10, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=10, n_estimators=200, score=0.959, total=   6.8s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=10, score=0.945, total=   0.5s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=10, score=0.949, total=   0.5s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=10, score=0.943, total=   0.4s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=10, score=0.951, total=   0.4s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=10, score=0.945, total=   0.4s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=50, score=0.954, total=   2.9s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=50, score=0.956, total=   1.7s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=50, score=0.959, total=   2.7s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=50, score=0.967, total=   2.0s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=50, score=0.956, total=   2.3s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=100, score=0.955, total=   4.2s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=100, score=0.957, total=   4.4s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=100, score=0.959, total=   4.2s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=100, score=0.967, total=   4.1s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=100, score=0.958, total=   4.7s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=200, score=0.957, total=   7.5s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=200, score=0.957, total=   7.9s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=200, score=0.958, total=   7.3s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=200, score=0.968, total=   8.5s\n",
      "[CV]  learning_rate=0.5, max_depth=20, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.5, max_depth=20, n_estimators=200, score=0.959, total=   7.1s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=10, score=0.886, total=   0.1s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=10, score=0.889, total=   0.1s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=10, score=0.880, total=   0.1s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=10, score=0.892, total=   0.1s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=10, score=0.886, total=   0.1s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=50 ................\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=50, score=0.918, total=   1.0s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=50, score=0.921, total=   1.1s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=50, score=0.930, total=   0.6s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=50, score=0.930, total=   0.5s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=50, score=0.929, total=   0.6s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=100, score=0.932, total=   1.1s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=100, score=0.936, total=   1.7s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=100, score=0.942, total=   1.1s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=100, score=0.945, total=   1.0s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=100, score=0.941, total=   1.3s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=200, score=0.942, total=   2.9s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=200, score=0.946, total=   2.6s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=200, score=0.949, total=   2.7s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=200, score=0.953, total=   2.3s\n",
      "[CV]  learning_rate=0.1, max_depth=3, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=3, n_estimators=200, score=0.949, total=   2.8s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=10, score=0.910, total=   0.2s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=10, score=0.915, total=   0.2s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=10, score=0.916, total=   0.2s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=10, score=0.930, total=   0.2s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=10 ................\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=10, score=0.922, total=   0.2s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=50, score=0.943, total=   0.9s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=50, score=0.940, total=   0.8s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=50, score=0.950, total=   1.8s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=50, score=0.949, total=   1.2s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=50 ................\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=50, score=0.945, total=   0.9s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=100, score=0.953, total=   2.0s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=100, score=0.953, total=   2.7s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=100, score=0.956, total=   1.7s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=100, score=0.962, total=   2.8s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=100 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=100, score=0.958, total=   1.6s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=200, score=0.958, total=   4.1s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=200, score=0.955, total=   5.0s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=200, score=0.958, total=   4.6s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=200, score=0.964, total=   3.9s\n",
      "[CV]  learning_rate=0.1, max_depth=5, n_estimators=200 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=5, n_estimators=200, score=0.960, total=   3.9s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=10, score=0.942, total=   0.4s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=10, score=0.947, total=   0.4s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=10, score=0.939, total=   0.4s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=10, score=0.953, total=   0.4s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=10, score=0.945, total=   0.3s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=50, score=0.951, total=   2.6s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=50, score=0.959, total=   1.9s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=50, score=0.956, total=   1.7s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=50, score=0.963, total=   2.1s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=50, score=0.955, total=   1.6s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=100, score=0.956, total=   4.1s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=100, score=0.959, total=   3.6s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=100, score=0.957, total=   3.6s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=100, score=0.967, total=   3.6s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=100, score=0.958, total=   3.6s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=200, score=0.959, total=   6.5s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=200, score=0.957, total=   6.3s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=200, score=0.959, total=   6.9s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=200 ..............\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=200, score=0.967, total=   6.5s\n",
      "[CV]  learning_rate=0.1, max_depth=10, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=10, n_estimators=200, score=0.959, total=   7.6s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=10, score=0.945, total=   0.5s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=10, score=0.949, total=   0.4s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=10, score=0.943, total=   0.4s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=10, score=0.951, total=   0.4s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=10, score=0.945, total=   0.4s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=50, score=0.954, total=   2.4s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=50, score=0.956, total=   1.9s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=50, score=0.959, total=   3.0s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=50, score=0.967, total=   1.9s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=50, score=0.956, total=   1.9s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=100, score=0.955, total=   4.1s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=100, score=0.957, total=   3.5s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=100, score=0.959, total=   4.7s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=100, score=0.967, total=   3.6s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=100, score=0.958, total=   3.9s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=200, score=0.957, total=   8.5s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=200, score=0.957, total=   7.8s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=200, score=0.958, total=   7.7s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=200, score=0.968, total=   7.2s\n",
      "[CV]  learning_rate=0.1, max_depth=20, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.1, max_depth=20, n_estimators=200, score=0.959, total=   8.4s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=10, score=0.886, total=   0.1s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=10, score=0.889, total=   0.1s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=10, score=0.880, total=   0.2s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=10, score=0.892, total=   0.1s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=10, score=0.886, total=   0.1s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=50, score=0.918, total=   0.6s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=50, score=0.921, total=   0.8s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=50, score=0.930, total=   1.2s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=50, score=0.930, total=   0.8s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=50, score=0.929, total=   0.6s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=100, score=0.932, total=   1.1s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=100, score=0.936, total=   1.1s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=100, score=0.942, total=   1.7s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=100, score=0.945, total=   1.2s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=100, score=0.941, total=   1.2s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=200, score=0.942, total=   3.3s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=200, score=0.946, total=   2.2s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=200, score=0.949, total=   3.3s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=200, score=0.953, total=   2.3s\n",
      "[CV]  learning_rate=0.01, max_depth=3, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=3, n_estimators=200, score=0.949, total=   3.2s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=10, score=0.910, total=   0.2s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=10, score=0.915, total=   0.2s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=10, score=0.916, total=   0.2s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=10, score=0.930, total=   0.2s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=10 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=10, score=0.922, total=   0.2s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=50, score=0.943, total=   1.2s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=50, score=0.940, total=   1.3s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=50, score=0.950, total=   1.1s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=50, score=0.949, total=   1.0s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=50 ...............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=50, score=0.945, total=   1.6s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=100, score=0.953, total=   1.7s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=100 ..............\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=100, score=0.953, total=   2.4s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=100, score=0.956, total=   2.0s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=100, score=0.962, total=   2.7s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=100 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=100, score=0.958, total=   1.9s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=200, score=0.958, total=   5.5s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=200, score=0.955, total=   4.7s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=200, score=0.958, total=   4.5s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=200, score=0.964, total=   4.1s\n",
      "[CV]  learning_rate=0.01, max_depth=5, n_estimators=200 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=5, n_estimators=200, score=0.960, total=   3.4s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=10, score=0.942, total=   0.7s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=10, score=0.947, total=   0.7s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=10, score=0.939, total=   0.7s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=10, score=0.953, total=   0.3s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=10, score=0.945, total=   0.3s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=50, score=0.951, total=   1.5s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=50, score=0.959, total=   2.2s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=50, score=0.956, total=   1.9s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=50, score=0.963, total=   1.5s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=50, score=0.955, total=   2.1s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=100, score=0.956, total=   3.1s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=100, score=0.959, total=   3.1s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=100, score=0.957, total=   3.5s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=100, score=0.967, total=   4.0s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=100, score=0.958, total=   2.8s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=200, score=0.959, total=   6.3s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=200, score=0.957, total=   6.5s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=200, score=0.959, total=   6.4s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=200, score=0.967, total=   5.7s\n",
      "[CV]  learning_rate=0.01, max_depth=10, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.01, max_depth=10, n_estimators=200, score=0.959, total=   5.8s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=10, score=0.945, total=   0.4s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=10, score=0.949, total=   0.4s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=10, score=0.943, total=   0.4s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=10, score=0.951, total=   0.4s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=10, score=0.945, total=   0.4s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=50, score=0.954, total=   2.1s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=50, score=0.956, total=   2.0s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=50, score=0.959, total=   2.4s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=50, score=0.967, total=   1.9s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=50, score=0.956, total=   2.5s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=100, score=0.955, total=   3.6s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=100, score=0.957, total=   4.8s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=100, score=0.959, total=   3.9s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=100, score=0.967, total=   4.2s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=100, score=0.958, total=   4.5s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=200, score=0.957, total=   7.1s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=200, score=0.957, total=   7.8s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=200, score=0.958, total=   7.5s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=200, score=0.968, total=   7.8s\n",
      "[CV]  learning_rate=0.01, max_depth=20, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.01, max_depth=20, n_estimators=200, score=0.959, total=   8.4s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=10, score=0.886, total=   0.1s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=10, score=0.889, total=   0.1s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=10, score=0.880, total=   0.1s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=10, score=0.892, total=   0.1s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=10, score=0.886, total=   0.1s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=50 ..............\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=50, score=0.918, total=   0.6s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=50, score=0.921, total=   0.6s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=50, score=0.930, total=   1.1s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=50, score=0.930, total=   0.6s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=50, score=0.929, total=   0.7s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=100, score=0.932, total=   1.2s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=100, score=0.936, total=   1.8s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=100, score=0.942, total=   1.6s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=100, score=0.945, total=   1.2s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=100, score=0.941, total=   1.1s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=200, score=0.942, total=   3.3s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=200, score=0.946, total=   2.1s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=200, score=0.949, total=   3.3s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=200, score=0.953, total=   2.8s\n",
      "[CV]  learning_rate=0.001, max_depth=3, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.001, max_depth=3, n_estimators=200, score=0.949, total=   2.5s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=10, score=0.910, total=   0.2s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=10, score=0.915, total=   0.2s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=10, score=0.916, total=   0.2s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=10, score=0.930, total=   0.2s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=10 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=10, score=0.922, total=   0.3s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=50, score=0.943, total=   1.4s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=50, score=0.940, total=   0.9s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=50, score=0.950, total=   0.9s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=50, score=0.949, total=   0.9s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=50 ..............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=50, score=0.945, total=   1.9s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=100, score=0.953, total=   1.7s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=100, score=0.953, total=   1.6s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=100, score=0.956, total=   2.7s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=100, score=0.962, total=   1.6s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=100 .............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=100, score=0.958, total=   1.6s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=200, score=0.958, total=   4.3s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=200, score=0.955, total=   3.7s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=200, score=0.958, total=   3.7s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=200, score=0.964, total=   3.7s\n",
      "[CV]  learning_rate=0.001, max_depth=5, n_estimators=200 .............\n",
      "[CV]   learning_rate=0.001, max_depth=5, n_estimators=200, score=0.960, total=   3.3s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=10 .............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=10, score=0.942, total=   0.7s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=10 .............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=10, score=0.947, total=   0.7s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=10 .............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=10, score=0.939, total=   0.5s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=10 .............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=10, score=0.953, total=   0.3s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=10 .............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=10, score=0.945, total=   0.3s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=50 .............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=50, score=0.951, total=   1.5s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=50 .............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=50, score=0.959, total=   1.6s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=50 .............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=50, score=0.956, total=   2.4s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=50 .............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=50, score=0.963, total=   1.7s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=50 .............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=50, score=0.955, total=   2.0s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=100 ............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=100, score=0.956, total=   3.4s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=100 ............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=100, score=0.959, total=   4.1s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=100 ............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=100, score=0.957, total=   4.0s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=100 ............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=100, score=0.967, total=   3.3s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=100 ............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=100, score=0.958, total=   3.6s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=200 ............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=200, score=0.959, total=   6.9s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=200 ............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=200, score=0.957, total=   6.2s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=200 ............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=200, score=0.959, total=   6.2s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=200 ............\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=200, score=0.967, total=   6.8s\n",
      "[CV]  learning_rate=0.001, max_depth=10, n_estimators=200 ............\n",
      "[CV]   learning_rate=0.001, max_depth=10, n_estimators=200, score=0.959, total=   6.2s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=10 .............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=10, score=0.945, total=   0.4s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=10 .............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=10, score=0.949, total=   0.4s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=10 .............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=10, score=0.943, total=   0.5s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=10 .............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=10, score=0.951, total=   0.4s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=10 .............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=10, score=0.945, total=   0.4s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=50 .............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=50, score=0.954, total=   2.4s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=50 .............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=50, score=0.956, total=   1.9s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=50 .............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=50, score=0.959, total=   3.2s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=50 .............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=50, score=0.967, total=   1.8s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=50 .............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=50, score=0.956, total=   2.8s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=100 ............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=100, score=0.955, total=   4.0s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=100 ............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=100, score=0.957, total=   3.8s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=100 ............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=100, score=0.959, total=   4.1s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=100 ............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=100, score=0.967, total=   4.2s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=100 ............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=100, score=0.958, total=   4.1s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=200 ............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=200, score=0.957, total=   7.1s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=200 ............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=200, score=0.957, total=   8.2s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=200 ............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=200, score=0.958, total=   8.0s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=200 ............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=200, score=0.968, total=   7.6s\n",
      "[CV]  learning_rate=0.001, max_depth=20, n_estimators=200 ............\n",
      "[CV]   learning_rate=0.001, max_depth=20, n_estimators=200, score=0.959, total=   7.6s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 400 out of 400 | elapsed: 16.0min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score=nan,\n",
       "             estimator=XGBClassifier(base_score=0.5, booster='gbtree',\n",
       "                                     colsample_bylevel=1, colsample_bynode=1,\n",
       "                                     colsample_bytree=1, gamma=0,\n",
       "                                     learning_rate=0.1, max_delta_step=0,\n",
       "                                     max_depth=3, min_child_weight=1,\n",
       "                                     missing=None, n_estimators=100, n_jobs=1,\n",
       "                                     nthread=None, objective='binary:logistic',\n",
       "                                     random_state=0, reg_alpha=0, reg_lambda=1,\n",
       "                                     scale_pos_weight=1, seed=None, silent=None,\n",
       "                                     subsample=1, verbosity=1),\n",
       "             iid='deprecated', n_jobs=None,\n",
       "             param_grid={' learning_rate': [1, 0.5, 0.1, 0.01, 0.001],\n",
       "                         'max_depth': [3, 5, 10, 20],\n",
       "                         'n_estimators': [10, 50, 100, 200]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=3)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "c4362661",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{' learning_rate': 1, 'max_depth': 10, 'n_estimators': 200}"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "83a1e9d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0, learning_rate=1,\n",
       "              max_delta_step=0, max_depth=10, min_child_weight=1, missing=None,\n",
       "              n_estimators=200, n_jobs=1, nthread=None,\n",
       "              objective='binary:logistic', random_state=0, reg_alpha=0,\n",
       "              reg_lambda=1, scale_pos_weight=1, seed=None, silent=None,\n",
       "              subsample=1, verbosity=1)"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_model=XGBClassifier(learning_rate= 1, max_depth= 10, n_estimators= 200)\n",
    "new_model.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "50e97b4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9638172772501131"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_new = new_model.predict(X_test)\n",
    "predictions_new = [round(value) for value in y_pred_new]\n",
    "accuracy_new = accuracy_score(y_test,predictions_new)\n",
    "accuracy_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "f66d74c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'xgboost_model.pkl'\n",
    "pickle.dump(new_model, open(filename, 'wb'))\n",
    "loaded_model = pickle.load(open(filename, 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "a708346f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1], dtype=int64)"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loaded_model.predict(pca_pred.transform(scaler_pred.transform([[-1,1,1,-1,-1,-1,-1,-1,1,-1,1,-1,-1,0,1,-1,-1,-1,-1,1,1]])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "482586e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "col_0    -1     1\n",
      "Result           \n",
      "-1      912    49\n",
      " 1       31  1219\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.97      0.95      0.96       961\n",
      "           1       0.96      0.98      0.97      1250\n",
      "\n",
      "    accuracy                           0.96      2211\n",
      "   macro avg       0.96      0.96      0.96      2211\n",
      "weighted avg       0.96      0.96      0.96      2211\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(pd.crosstab(y_test,yhat))\n",
    "print(\"\\n\")\n",
    "print(classification_report(y_test,yhat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "0db0f745",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANYAAACMCAYAAAAA9le/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWkUlEQVR4nO3dfVRVdboH8O8DR1DIFxRQRIUUGN7kRZgcM69v6U0ac1rDpDQTjGPLqaV3mrHM22pVt2la1rqtZrqZTo0rs1Zjll611VRWauKMvQi+YFL2okcFuiAIAqK8fu8fZ3M8wAEOcDacY89nrbM857d/v72fvT3P2S/s8xwhCaWUe/kMdABKXYs0sZQygSaWUibQxFLKBJpYSplAE0spE1gGOgBngoODGRkZOdBhKNVBfn5+OcmQ7vp1m1gi8jKAnwIoI5noZLoAeA5ABoA6AL8mediYdosxzRfARpJPuRJ8ZGQk8vLyXOmqVL8SkTOu9HPlUPAVALd0MX0BgGjjsRzABiMAXwAvGNPjAWSJSLwrQSnl7bpNLJK5AC500WURgFdp8ymAESISBuAGAN+SPEWyAcAbRl+lrnnuOMcKB3DO4XWR0easfaobltcjJHH4bCWKKi/DWl6HFodbuGjrcPW58ZLGq9aubPOc9s5Xx7Dzvg6vv794GSTgP8i3Q4wd4na6Ms6anIx11q8vY52F4mKArs/PtW3Ql/k546zbX5akIPg6f5fGd8YdiSVO2thFu/OZiCyH7VASEyZM6FNAu0/8H44XXcSmf53GpYZml8aIXA1YRByeG//iagdp397FeMcxLQRq65sQMSoAvtJu8zjZWs42oLQf12k/Z/NzMtbZYHcvtw+xOA2vT9vAWb+2rS1uuH/WHYlVBGC8w+txAEoA+HXS7hTJlwC8BADp6ek9XrOWFuLVT6z4++dn8XVpLQBgkK8g0M8Xs34UirumRWDCyACMDPSDv8XH6RtFKXdxR2K9DWCliLwB26HeRZLfi8h5ANEicj2AYgBLANzphuXZkcSRc1U4W1GHB7cXoKGpBQAw60cheGhBHH40Zqg7F6eUy1y53L4FwCwAwSJSBOAxAIMAgORfAbwL26X2b2G73L7UmNYkIisB7IbtcvvLJE+4M/hb/nIAJ0trAAChQ/0ROswfr9/9EwwfMsidi1Gqx7pNLJJZ3UwngBWdTHsXtsRzu2/Lau1JtWvFdMSPHYZBvnojifIMHnnnRXcqautx87P7AQAbs9ORPH7EAEekVFte+RG/5KVPAQCxY4ZiTmzoAEejVEdeuce6eLkRAPDefTP06p7ySF6ZWCMCBqGF1KRSHssrDwXLauqRHjFyoMNQqlNel1jfltWiqq4RFy41DHQoSnXK6xLLWn4JAHDXtIgBjkSpznldYrWKHBU40CEo1SmvS6xXP7V9z0yvWyhP5nWJFRzoBwCIDxs2wJEo1TmvS6xvz9cidKg/fHx0l6U8l9cllsVHEDEqYKDDUKpLXpdYADC43TdwlfI0XplYSnk6TSylTKCJpZQJXEosEblFRE6KyLci8p9Opq8WkaPG4wsRaRaRkcY0q4gcN6b1uQpnVV2j3nyrPF63ieVK4U2S/00yhWQKgIcA7CfpWItwtjE9va8Bl9fWI3zE4L7ORilTubLH6mnhzSwAW9wRXHstLUT1lSb4W/SqoPJsriRWZwU5OxCRANjKUW93aCaAD0Qk36gd2GvflNnKmtU3uVYrUKmB4soXHXtSeHMhgH+1OwycTrJEREIBfCgiXxllq9suxIWCna1fFblxUrALYSs1cFzZY3VWkNOZJWh3GEiyxPi3DMAO2A4tOyD5Esl0kukhIV3/Skpfy/8qZTZXEusQjMKbIuIHW/K83b6TiAwHMBPALoe2QBEZ2vocwHwAX7gjcKU8mSt1BZ0W3hSRe4zpfzW63g7gA5KXHIaPBrDDuDxuAfB3ku+7cwWU8kQuFZNxVnjTIaFaX78C229pObadApDcpwiV8kJ654VSJtDEUsoEmlhKmcCrEssdPwimVH/wqsQqrrwMALjO3ysL+KofEK9KrIZm2w/LjR6ufyBWns2rEkspb6GJpZQJNLGUMoEmllIm8KrEutyg38NS3sGrEuvshToAQICfXm5Xns2rEsvXR3Cdv0X/jqU8nlclFgBoyXblDbwusZTyBppYSpnAXQU7Z4nIRYeinY+6Olapa1G3VwEcCnbOg62wzCEReZtkYbuuB0j+tJdjlbqmmFGw011jlfJa7izYOU1EjonIeyKS0MOxEJHlIpInInnnz593ISylPJcrieVKwc7DACJIJgN4HsDOHoy1NfagrqBSns4tBTtJVpOsNZ6/C2CQiAS7Mlapa5FbCnaKyBgxigeKyA3GfCtcGavUtchdBTszAdwrIk0ALgNYQpIAnI41aV2U8hhuKdhJch2Ada6OVepap3deKGUCTSylTKCJpZQJNLGUMoEmllIm0MRSygSaWEqZQBNLKRNoYillAk0spUygiaWUCTSxlDKBJpZSJtDEUsoEmlhKmcBddQV/KSIFxuOgiCQ7TLOKyHGj3mCeO4NXylO5q67gaQAzSVaKyAIALwGY6jB9NslyN8atlEdzS11BkgdJVhovP4WtaIxSP1jurCvYahmA9xxeE8AHIpIvIst7HqJS3seVmhcu1wYUkdmwJdZNDs3TSZaISCiAD0XkK5K5TsYuB7AcACZMmOBCWEp5LrfUFQQAEUkCsBHAIpIVre0kS4x/ywDsgO3QsgMt2KmuJe6qKzgBwP8CuIvk1w7tgSIytPU5gPkAvnBX8Ep5KnfVFXwUwCgA6426nU0k0wGMBrDDaLMA+DvJ901ZE6U8iLvqCt4N4G4n404BSG7frtS1Tu+8UMoEmlhKmUBsJdbdLz8/P9RisWwEkIgeJnBFRUVEWFhYh/aqukbUNTRh7IghbopSKecGDx6McePGYdCgQW3aRSTfuH7QJZfOsXrDYrFsHDNmTFxISEilj49Pj7K3sLAwIi4urkN7SdVlVNY1IG7scLfFqVR7JFFRUYGioiJcf/31vZqHmYeCiSEhIdU9TSqlBpqIYNSoUbhy5Uqv52FmYvloUilvZfyJqNf04oWb1dfXY/HixYiKisLUqVNhtVqd9tu6dSuSkpKQkJCABx980N5+5swZzJ07F0lJSZg1axaKiorajKuurkZ4eDhWrlxpb9u7dy+mTJmCxMRE5OTkoKmpCQBw8eJFLFy4EMnJyUhISMCmTZvsY6qqqpCZmYnY2FjExcXhk08+AQBcuHAB8+bNQ3R0NObNm4fKStu91VarFUOGDEFKSgpSUlJwzz332OfV0NCA5cuXIyYmBrGxsdi+fTsAIDc3F1OmTIHFYsG2bdvarMeaNWuQmJiIxMREbN261d6+bt06REVFQURQXt7xCxGHDh2Cr69vm/l1ti6LFy+2xxsZGYmUlBQAwIcffoi0tDRMnjwZaWlp2Lt3r9P/oz4hacrj6NGjVpJ5vXmcOHGCzhRX1vGL4iqn0zzFCy+8wN/+9rckyS1btvCOO+7o0Ke8vJzjx49nWVkZSTI7O5sfffQRSTIzM5OvvPIKSXLPnj381a9+1Wbs7373O2ZlZXHFihUkyebmZo4bN44nT54kST7yyCPcuHEjSfLJJ5/kgw8+SJIsKytjUFAQ6+vr7cv829/+RpKsr69nZWUlSXL16tVcu3YtSXLt2rX28adPn2ZCQoLTdX700Uf58MMP2+M5f/68fcyxY8d411138a233rL3f+edd3jzzTezsbGRtbW1TEtL48WLF0mShw8f5unTpxkREWGfT6umpibOnj2bCxYsaDO/ztbF0apVq/j444/bl1FcXEySPH78OMeOHet0vQoLCzu0AcijC+//H8we695770V6ejoSEhLw2GOP2dsjIyPtn4x5eXmYNWsWAKC2thZLly7F5MmTkZSUZP8U7s6uXbuQk5MDAMjMzMSePXvAdldeT506hZiYGLTeE3nzzTfb519YWIi5c+cCAGbPno1du3bZx+Xn56O0tBTz58+3t1VUVMDf3x8xMTEAgHnz5tnnJSKoqakBSdTW1mLkyJGwWCyorq5Gbm4uli1bBgDw8/PDiBEjOsSfk5ODnTt3ojsvv/wyHnroIQCAj48PgoODAdi2bVJSEnx82r7NCgsLMXPmTFgsFgQGBiI5ORnvv2+7ISc1NRWRkZFOl/P888/j5z//OUJDQ+1tXa1LK5J48803kZWVZV/G2LFjAQAJCQm4cuUK6uvru13PnjDtqqCj1duOjf/6/2oCXO3f0liPgAMXO7Q3NLWgqaUFAX4WxI8dhscWJrgcw5NPPomRI0eiubkZc+fORUFBAZKSkjrt/8QTT2D48OE4fvw4ANgPiRYvXoyTJ0926L9q1SpkZ2ejuLgY48fb7lm2WCwYPnw4Kioq7G82AIiKisJXX30Fq9WKcePGYefOnWhoaAAAJCcnY/v27bjvvvuwY8cO1NTUoKKiAkFBQbj//vvx2muvYc+ePfZ5BQcHo7GxEXl5eUhPT8e2bdtw7pztWz4rV67EbbfdhrFjx6KmpgZbt26Fj48PTp06hZCQECxduhTHjh1DWloannvuOQQGBqK0tBStf+oICwtDWVmZfVmnT59Gamoqhg0bhj/96U+YMWMGqqqqAACPPPIIPv74Y0yaNAnr1q3D6NGjO922ycnJePzxx7Fq1SrU1dVh3759iI+P7+J/DyguLsaOHTuwd+9eHDp0yN7e1bq0OnDgAEaPHo3o6OgO892+fTtSU1Ph7+/f5fJ76gezx3rzzTcxZcoUpKam4sSJEygsLOyy/0cffYQVK1bYXwcFBQGwnRsdPXq0wyM7OxsAOuydgI4nwkFBQdiwYQMWL16MGTNmIDIyEhaL7TPumWeewf79+5Gamor9+/cjPDwcFosF69evR0ZGhj1pHef9xhtv4A9/+ANuuOEGDB061D6v3bt3IyUlBSUlJTh69ChWrlyJ6upqNDU14fDhw7j33ntx5MgRBAYG4qmnnupye4SFheHs2bM4cuQInn32Wdx55532eRUVFWH69Ok4fPgwpk2bhgceeKDLec2fPx8ZGRm48cYbkZWVhWnTptlj7szvf/97PP300/D19W3T7sq6bNmyxb63cnTixAmsWbMGL774YpfL7hVXjhd78/Ckc6xTp05x0qRJvHDhAkkyJyeHmzZtIklOmjSJpaWlJMkDBw5w5syZJMnU1FR+8803HeZ1xx13MDk5ucNj8+bNJMn58+fz4MGDJMnGxkaOGjWKLS0tXcb34osvcvXq1R3aa2pqGB4eTpK88847OX78eEZERHDUqFEcOnQo16xZ02HM7t27+Ytf/IIkmZGRwdzcXPu02bNn87PPPuP333/PiIgIe3tubi4zMjJIkjExMSwpKSFJlpSUMCYmxmnMM2fO5KFDh9jS0sKAgAA2NzeTJM+ePcv4+Pg2fXNyctqcE7WXlZXFf/zjH23a2p9jRUZGMiIighEREQwMDGRISAh37NjR5bqQtv+D0NBQnjt3rs38z507x+joaP7zn//sNC49x+pGdXU1AgMDMXz4cJSWluK9965+wTkyMhL5+fkA0OY8av78+Vi37urvlbceCna3x7rtttuwefNmAMC2bdswZ84cp5duWw+xKisrsX79etx9t+0e5vLycrS0tAAA1q5di9/85jcAgNdffx1nz56F1WrFM888g+zsbPsnc+u86uvr8fTTT9uv2E2YMMF+2FhaWoqTJ09i4sSJGDNmDMaPH28/pN2zZ4/9UMwx/s2bN2PRIlsVhvPnz6O5uRmA7fDrm2++wcSJEyEiWLhwIT7++OMO8+pMc3MzKipsX9krKChAQUFBm/NGZ06fPg2r1Qqr1YrMzEysX78eP/vZz7pcF8B25BEbG4tx465Wi6iqqsKtt96KtWvXYvr06V0ut9dcyb7ePDxpj0XaPjVjY2OZkZHB22+/3b7Hys3NZXR0NG+66Sbef//99j1WTU0Ns7OzmZCQwKSkJG7fvt2l5Vy+fJmZmZmcNGkSf/zjH/O7776zT0tOTrY/X7JkCePi4hgXF8ctW7bY29966y1GRUUxOjqay5Yt45UrVzosY9OmTfargiT5wAMPMDY2ljExMfzzn/9sby8uLua8efOYmJjIhIQEvvbaa/ZpR44cYVpaGidPnsxFixbZ9+bl5eWcM2cOo6KiOGfOHFZUVJAkt23bxvj4eCYlJTE1NZVvv/22fV5Wq5UzZszg5MmTOWfOHJ45c4Yk+fnnnzM8PJwBAQEcOXKkfU92+fJl+7pPnTqVR44csc/rueeeY3h4OH19fRkWFsZly5Z1WP/2e8DO1qW174YNG9qMf+KJJxgQENDmiKP1qMVRX/ZYpt0reOzYMWtycnKvKjMVFhamOfvUa72lKUFvaVL94Msvv0T7W+tcvVfwB3EoqFR/c1fBThGR/zGmF4jIlNZpFy5cGFZQUJBYUFCQWFRUNMadwSvlqdxVsHMBgGjjMRXABsB2/lZUVDQhOjr6a39//8bCwsK4oKCgqsDAwN7f3aiUF3BLwU7j9avG+d2nAEY0NzdLdXV1oJ+fX/2QIUMafHx8OGLEiAuVlZUj2i9AKU/T12sP7irY2aFPdXW1tby8fITFYmlobfTz82tobGz063W0SvUDGt/HGjx4cK/n4a6CnR36/PGPf3xi9erVTw0ZMiTm0qVLwQBQV1cX2NjY6F9dXR3cvv+lS5eG1tXVXQfYVszZ337qGprQ0ER8eXFQh2lKuVPrN4h7y5XEcqVgZ4c++/btO7Fv377/APBfJP8dAETkIQAgubarBaanpzMvT3+YRHkvtxTsNF5nG1cHfwLgIsnvXRyr1DXHXQU73wWQAeBbAHUAlnY11pQ1UcqDmHbnRV/ooaDyVK7eeeGRiSUi5wGc6WRyMABP/BE7jatnvDWuCJLd/mqHRyZWV0Qkz5VPjP6mcfXMtR6X3iuolAk0sZQygTcm1ksDHUAnNK6euabj8rpzLKW8gTfusZTyeB6VWH353ld3Y02O65dGPAUiclBEkh2mWUXkuIgcFRG3/nHOhbhmichFY9lHReRRV8eaHNdqh5i+EJFmERlpTDNle4nIyyJSJiJOf6rX7e8tV76/3x8P2O7M+A7ARAB+AI4BiG/XJwPAe7Dd9PsTAJ+5OtbkuG4EEGQ8X9Aal/HaCiB4gLbXLADv9GasmXG1678QwN5+2F7/BmAKgC86me7W95Yn7bF69b0vEQlzcaxpcZE8SLLSePkpbDcqm60v6zyg26udLABb3LTsTpHMBXChiy5ufW95UmL16ntfRpsrY82My9Ey2D75WhHAByKSLyLL3RRTT+KaJiLHROQ9EWktHewR20tEAgDcAsCxfrdZ26s7bn1v9UuJaRf16ntfRh9XxvaWy/MWkdmwJdZNDs3TSZaISCiAD0XkK+PTsz/iOgzbLTi1IpIBYCds5RM8YnvBdhj4L5KOexKztld33Pre8qQ9Vq++92X0cWWsmXFBRJIAbASwiGRFazvJEuPfMgA7YDu06Je4SFaTrDWevwtgkIgEuzLWzLgcLEG7w0ATt1d33PvecvdJYh9OLi0ATgG4HldPEhPa9bkVbU8wP3d1rMlxTYDtKzM3tmsPBDDU4flBALf0Y1xjcPVvlTcAOGtsuwHdXka/4bCd8wT2x/Yy5hmJzi9euPW9NeAJ1W7lMgB8DdtVmIeNtnsA3GM8F9gqRn0H4DiA9K7G9mNcGwFUAjhqPPKM9onGf8QxACcGIK6VxnKPwXZR5cauxvZXXMbrXwN4o90407YXbHvG7wE0wrYXWmbme0vvvFDKBJ50jqXUNUMTSykTaGIpZQJNLKVMoImllAk0sZQygSaWUibQxFLKBP8PD7cRxF+wbRYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 216x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_proba = new_model.predict_proba(X_test)[::,1]\n",
    "fpr, tpr, _ = roc_curve(y_test,  y_proba)\n",
    "auc = roc_auc_score(y_test, y_proba)\n",
    "plt.plot(fpr,tpr,label=\" auc=\"+str(auc))\n",
    "rcParams['figure.figsize'] = 3,2\n",
    "plt.legend(loc=4)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd405308",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}