supervised.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np, pandas as pd\n",
    "import ast \n",
    "from sklearn import linear_model\n",
    "from sklearn import metrics\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import spacy\n",
    "from nltk import Tree\n",
    "en_nlp = spacy.load('en')\n",
    "from nltk.stem.lancaster import LancasterStemmer\n",
    "st = LancasterStemmer()\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import RandomizedSearchCV, GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"train_detect_sent.csv\").reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(87598, 14)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer_start</th>\n",
       "      <th>context</th>\n",
       "      <th>question</th>\n",
       "      <th>text</th>\n",
       "      <th>sentences</th>\n",
       "      <th>quest_emb</th>\n",
       "      <th>target</th>\n",
       "      <th>sent_emb</th>\n",
       "      <th>cosine_sim</th>\n",
       "      <th>euclidean_dis</th>\n",
       "      <th>pred_idx_cos</th>\n",
       "      <th>pred_idx_euc</th>\n",
       "      <th>root_match_idx</th>\n",
       "      <th>root_match_idx_first</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>515</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>To whom did the Virgin Mary allegedly appear i...</td>\n",
       "      <td>Saint Bernadette Soubirous</td>\n",
       "      <td>['Architecturally, the school has a Catholic c...</td>\n",
       "      <td>[[ 0.11010079  0.11422941  0.11560896 ...  0.0...</td>\n",
       "      <td>5</td>\n",
       "      <td>[array([ 0.05519997,  0.05013141,  0.04787038,...</td>\n",
       "      <td>[0.424736299052452, 0.36405004106069117, 0.347...</td>\n",
       "      <td>[14.563858, 15.262212, 17.398178, 14.272491, 1...</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>[5]</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>188</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>What is in front of the Notre Dame Main Building?</td>\n",
       "      <td>a copper statue of Christ</td>\n",
       "      <td>['Architecturally, the school has a Catholic c...</td>\n",
       "      <td>[[ 0.10951651  0.11030623  0.05210007 ... -0.0...</td>\n",
       "      <td>2</td>\n",
       "      <td>[array([ 0.05519997,  0.05013141,  0.04787038,...</td>\n",
       "      <td>[0.45407456884452513, 0.32262004808444933, 0.3...</td>\n",
       "      <td>[12.889506, 12.285219, 16.843704, 8.361172, 11...</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>279</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>The Basilica of the Sacred heart at Notre Dame...</td>\n",
       "      <td>the Main Building</td>\n",
       "      <td>['Architecturally, the school has a Catholic c...</td>\n",
       "      <td>[[ 0.01195647  0.14930707  0.02660049 ...  0.0...</td>\n",
       "      <td>3</td>\n",
       "      <td>[array([ 0.05519997,  0.05013141,  0.04787038,...</td>\n",
       "      <td>[0.39585783692319865, 0.29170832145169434, 0.3...</td>\n",
       "      <td>[11.857297, 11.392319, 15.061656, 7.1847134, 8...</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>[1, 2, 3, 4, 5, 6]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   answer_start                                            context  \\\n",
       "0           515  Architecturally, the school has a Catholic cha...   \n",
       "1           188  Architecturally, the school has a Catholic cha...   \n",
       "2           279  Architecturally, the school has a Catholic cha...   \n",
       "\n",
       "                                            question  \\\n",
       "0  To whom did the Virgin Mary allegedly appear i...   \n",
       "1  What is in front of the Notre Dame Main Building?   \n",
       "2  The Basilica of the Sacred heart at Notre Dame...   \n",
       "\n",
       "                         text  \\\n",
       "0  Saint Bernadette Soubirous   \n",
       "1   a copper statue of Christ   \n",
       "2           the Main Building   \n",
       "\n",
       "                                           sentences  \\\n",
       "0  ['Architecturally, the school has a Catholic c...   \n",
       "1  ['Architecturally, the school has a Catholic c...   \n",
       "2  ['Architecturally, the school has a Catholic c...   \n",
       "\n",
       "                                           quest_emb  target  \\\n",
       "0  [[ 0.11010079  0.11422941  0.11560896 ...  0.0...       5   \n",
       "1  [[ 0.10951651  0.11030623  0.05210007 ... -0.0...       2   \n",
       "2  [[ 0.01195647  0.14930707  0.02660049 ...  0.0...       3   \n",
       "\n",
       "                                            sent_emb  \\\n",
       "0  [array([ 0.05519997,  0.05013141,  0.04787038,...   \n",
       "1  [array([ 0.05519997,  0.05013141,  0.04787038,...   \n",
       "2  [array([ 0.05519997,  0.05013141,  0.04787038,...   \n",
       "\n",
       "                                          cosine_sim  \\\n",
       "0  [0.424736299052452, 0.36405004106069117, 0.347...   \n",
       "1  [0.45407456884452513, 0.32262004808444933, 0.3...   \n",
       "2  [0.39585783692319865, 0.29170832145169434, 0.3...   \n",
       "\n",
       "                                       euclidean_dis  pred_idx_cos  \\\n",
       "0  [14.563858, 15.262212, 17.398178, 14.272491, 1...             5   \n",
       "1  [12.889506, 12.285219, 16.843704, 8.361172, 11...             3   \n",
       "2  [11.857297, 11.392319, 15.061656, 7.1847134, 8...             3   \n",
       "\n",
       "   pred_idx_euc      root_match_idx  root_match_idx_first  \n",
       "0             5                 [5]                     5  \n",
       "1             3                  []                     0  \n",
       "2             3  [1, 2, 3, 4, 5, 6]                     1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Architecturally, the school has a Catholic character.',\n",
       " \"Atop the Main Building's gold dome is a golden statue of the Virgin Mary.\",\n",
       " 'Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\".',\n",
       " 'Next to the Main Building is the Basilica of the Sacred Heart.',\n",
       " 'Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection.',\n",
       " 'It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858.',\n",
       " 'At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ast.literal_eval(data[\"sentences\"][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = data[data[\"sentences\"].apply(lambda x: len(ast.literal_eval(x)))<11].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_features(data):\n",
    "    train = pd.DataFrame()\n",
    "     \n",
    "    for k in range(len(data[\"euclidean_dis\"])):\n",
    "        dis = ast.literal_eval(data[\"euclidean_dis\"][k])\n",
    "        for i in range(len(dis)):\n",
    "            train.loc[k, \"column_euc_\"+\"%s\"%i] = dis[i]\n",
    "    \n",
    "    print(\"Finished\")\n",
    "    \n",
    "    for k in range(len(data[\"cosine_sim\"])):\n",
    "        dis = ast.literal_eval(data[\"cosine_sim\"][k].replace(\"nan\",\"1\"))\n",
    "        for i in range(len(dis)):\n",
    "            train.loc[k, \"column_cos_\"+\"%s\"%i] = dis[i]\n",
    "            \n",
    "    train[\"target\"] = data[\"target\"]\n",
    "    return train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished\n"
     ]
    }
   ],
   "source": [
    "train = create_features(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column_euc_0</th>\n",
       "      <th>column_euc_1</th>\n",
       "      <th>column_euc_2</th>\n",
       "      <th>column_euc_3</th>\n",
       "      <th>column_euc_4</th>\n",
       "      <th>column_euc_5</th>\n",
       "      <th>column_euc_6</th>\n",
       "      <th>column_euc_7</th>\n",
       "      <th>column_euc_8</th>\n",
       "      <th>column_euc_9</th>\n",
       "      <th>...</th>\n",
       "      <th>column_cos_1</th>\n",
       "      <th>column_cos_2</th>\n",
       "      <th>column_cos_3</th>\n",
       "      <th>column_cos_4</th>\n",
       "      <th>column_cos_5</th>\n",
       "      <th>column_cos_6</th>\n",
       "      <th>column_cos_7</th>\n",
       "      <th>column_cos_8</th>\n",
       "      <th>column_cos_9</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14.563858</td>\n",
       "      <td>15.262212</td>\n",
       "      <td>17.398178</td>\n",
       "      <td>14.272491</td>\n",
       "      <td>13.339654</td>\n",
       "      <td>9.336262</td>\n",
       "      <td>15.720997</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.364050</td>\n",
       "      <td>0.347755</td>\n",
       "      <td>0.394242</td>\n",
       "      <td>0.371025</td>\n",
       "      <td>0.185690</td>\n",
       "      <td>0.351921</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12.889506</td>\n",
       "      <td>12.285219</td>\n",
       "      <td>16.843704</td>\n",
       "      <td>8.361172</td>\n",
       "      <td>11.918098</td>\n",
       "      <td>17.601221</td>\n",
       "      <td>14.929258</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.322620</td>\n",
       "      <td>0.355004</td>\n",
       "      <td>0.271561</td>\n",
       "      <td>0.392342</td>\n",
       "      <td>0.384383</td>\n",
       "      <td>0.362597</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.857297</td>\n",
       "      <td>11.392319</td>\n",
       "      <td>15.061656</td>\n",
       "      <td>7.184713</td>\n",
       "      <td>8.465475</td>\n",
       "      <td>13.927309</td>\n",
       "      <td>12.249868</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.291708</td>\n",
       "      <td>0.309919</td>\n",
       "      <td>0.223061</td>\n",
       "      <td>0.265975</td>\n",
       "      <td>0.293025</td>\n",
       "      <td>0.288711</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   column_euc_0  column_euc_1  column_euc_2  column_euc_3  column_euc_4  \\\n",
       "0     14.563858     15.262212     17.398178     14.272491     13.339654   \n",
       "1     12.889506     12.285219     16.843704      8.361172     11.918098   \n",
       "2     11.857297     11.392319     15.061656      7.184713      8.465475   \n",
       "\n",
       "   column_euc_5  column_euc_6  column_euc_7  column_euc_8  column_euc_9  \\\n",
       "0      9.336262     15.720997           NaN           NaN           NaN   \n",
       "1     17.601221     14.929258           NaN           NaN           NaN   \n",
       "2     13.927309     12.249868           NaN           NaN           NaN   \n",
       "\n",
       "    ...    column_cos_1  column_cos_2  column_cos_3  column_cos_4  \\\n",
       "0   ...        0.364050      0.347755      0.394242      0.371025   \n",
       "1   ...        0.322620      0.355004      0.271561      0.392342   \n",
       "2   ...        0.291708      0.309919      0.223061      0.265975   \n",
       "\n",
       "   column_cos_5  column_cos_6  column_cos_7  column_cos_8  column_cos_9  \\\n",
       "0      0.185690      0.351921           NaN           NaN           NaN   \n",
       "1      0.384383      0.362597           NaN           NaN           NaN   \n",
       "2      0.293025      0.288711           NaN           NaN           NaN   \n",
       "\n",
       "   target  \n",
       "0       5  \n",
       "1       2  \n",
       "2       3  \n",
       "\n",
       "[3 rows x 21 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# train.fillna(10000, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>column_euc_0</th>\n",
       "      <td>14.563858</td>\n",
       "      <td>12.889506</td>\n",
       "      <td>11.857297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_1</th>\n",
       "      <td>15.262212</td>\n",
       "      <td>12.285219</td>\n",
       "      <td>11.392319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_2</th>\n",
       "      <td>17.398178</td>\n",
       "      <td>16.843704</td>\n",
       "      <td>15.061656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_3</th>\n",
       "      <td>14.272491</td>\n",
       "      <td>8.361172</td>\n",
       "      <td>7.184713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_4</th>\n",
       "      <td>13.339654</td>\n",
       "      <td>11.918098</td>\n",
       "      <td>8.465475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_5</th>\n",
       "      <td>9.336262</td>\n",
       "      <td>17.601221</td>\n",
       "      <td>13.927309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_6</th>\n",
       "      <td>15.720997</td>\n",
       "      <td>14.929258</td>\n",
       "      <td>12.249868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_9</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_0</th>\n",
       "      <td>0.424736</td>\n",
       "      <td>0.454075</td>\n",
       "      <td>0.395858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_1</th>\n",
       "      <td>0.364050</td>\n",
       "      <td>0.322620</td>\n",
       "      <td>0.291708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_2</th>\n",
       "      <td>0.347755</td>\n",
       "      <td>0.355004</td>\n",
       "      <td>0.309919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_3</th>\n",
       "      <td>0.394242</td>\n",
       "      <td>0.271561</td>\n",
       "      <td>0.223061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_4</th>\n",
       "      <td>0.371025</td>\n",
       "      <td>0.392342</td>\n",
       "      <td>0.265975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_5</th>\n",
       "      <td>0.185690</td>\n",
       "      <td>0.384383</td>\n",
       "      <td>0.293025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_6</th>\n",
       "      <td>0.351921</td>\n",
       "      <td>0.362597</td>\n",
       "      <td>0.288711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_9</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      0          1          2\n",
       "column_euc_0  14.563858  12.889506  11.857297\n",
       "column_euc_1  15.262212  12.285219  11.392319\n",
       "column_euc_2  17.398178  16.843704  15.061656\n",
       "column_euc_3  14.272491   8.361172   7.184713\n",
       "column_euc_4  13.339654  11.918098   8.465475\n",
       "column_euc_5   9.336262  17.601221  13.927309\n",
       "column_euc_6  15.720997  14.929258  12.249868\n",
       "column_euc_7        NaN        NaN        NaN\n",
       "column_euc_8        NaN        NaN        NaN\n",
       "column_euc_9        NaN        NaN        NaN\n",
       "column_cos_0   0.424736   0.454075   0.395858\n",
       "column_cos_1   0.364050   0.322620   0.291708\n",
       "column_cos_2   0.347755   0.355004   0.309919\n",
       "column_cos_3   0.394242   0.271561   0.223061\n",
       "column_cos_4   0.371025   0.392342   0.265975\n",
       "column_cos_5   0.185690   0.384383   0.293025\n",
       "column_cos_6   0.351921   0.362597   0.288711\n",
       "column_cos_7        NaN        NaN        NaN\n",
       "column_cos_8        NaN        NaN        NaN\n",
       "column_cos_9        NaN        NaN        NaN\n",
       "target         5.000000   2.000000   3.000000"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(3).transpose()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fitting Multinomial Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Standardize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "column_euc_0    64.051060\n",
       "column_euc_1    70.236871\n",
       "column_euc_2    57.691372\n",
       "column_euc_3    56.178432\n",
       "column_euc_4    58.976977\n",
       "column_euc_5    50.716494\n",
       "column_euc_6    60.221045\n",
       "column_euc_7          NaN\n",
       "column_euc_8          NaN\n",
       "column_euc_9          NaN\n",
       "column_cos_0     1.466755\n",
       "column_cos_1     1.606138\n",
       "column_cos_2     1.552746\n",
       "column_cos_3     1.544334\n",
       "column_cos_4     1.542615\n",
       "column_cos_5     1.477041\n",
       "column_cos_6     1.544371\n",
       "column_cos_7          NaN\n",
       "column_cos_8          NaN\n",
       "column_cos_9          NaN\n",
       "target           9.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.apply(max, axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subset1 = train.iloc[:,:10].fillna(60)\n",
    "subset2 = train.iloc[:,10:].fillna(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column_euc_0</th>\n",
       "      <th>column_euc_1</th>\n",
       "      <th>column_euc_2</th>\n",
       "      <th>column_euc_3</th>\n",
       "      <th>column_euc_4</th>\n",
       "      <th>column_euc_5</th>\n",
       "      <th>column_euc_6</th>\n",
       "      <th>column_euc_7</th>\n",
       "      <th>column_euc_8</th>\n",
       "      <th>column_euc_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14.563858</td>\n",
       "      <td>15.262212</td>\n",
       "      <td>17.398178</td>\n",
       "      <td>14.272491</td>\n",
       "      <td>13.339654</td>\n",
       "      <td>9.336262</td>\n",
       "      <td>15.720997</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12.889506</td>\n",
       "      <td>12.285219</td>\n",
       "      <td>16.843704</td>\n",
       "      <td>8.361172</td>\n",
       "      <td>11.918098</td>\n",
       "      <td>17.601221</td>\n",
       "      <td>14.929258</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.857297</td>\n",
       "      <td>11.392319</td>\n",
       "      <td>15.061656</td>\n",
       "      <td>7.184713</td>\n",
       "      <td>8.465475</td>\n",
       "      <td>13.927309</td>\n",
       "      <td>12.249868</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   column_euc_0  column_euc_1  column_euc_2  column_euc_3  column_euc_4  \\\n",
       "0     14.563858     15.262212     17.398178     14.272491     13.339654   \n",
       "1     12.889506     12.285219     16.843704      8.361172     11.918098   \n",
       "2     11.857297     11.392319     15.061656      7.184713      8.465475   \n",
       "\n",
       "   column_euc_5  column_euc_6  column_euc_7  column_euc_8  column_euc_9  \n",
       "0      9.336262     15.720997          60.0          60.0          60.0  \n",
       "1     17.601221     14.929258          60.0          60.0          60.0  \n",
       "2     13.927309     12.249868          60.0          60.0          60.0  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " subset1.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column_cos_0</th>\n",
       "      <th>column_cos_1</th>\n",
       "      <th>column_cos_2</th>\n",
       "      <th>column_cos_3</th>\n",
       "      <th>column_cos_4</th>\n",
       "      <th>column_cos_5</th>\n",
       "      <th>column_cos_6</th>\n",
       "      <th>column_cos_7</th>\n",
       "      <th>column_cos_8</th>\n",
       "      <th>column_cos_9</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.424736</td>\n",
       "      <td>0.364050</td>\n",
       "      <td>0.347755</td>\n",
       "      <td>0.394242</td>\n",
       "      <td>0.371025</td>\n",
       "      <td>0.185690</td>\n",
       "      <td>0.351921</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.454075</td>\n",
       "      <td>0.322620</td>\n",
       "      <td>0.355004</td>\n",
       "      <td>0.271561</td>\n",
       "      <td>0.392342</td>\n",
       "      <td>0.384383</td>\n",
       "      <td>0.362597</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.395858</td>\n",
       "      <td>0.291708</td>\n",
       "      <td>0.309919</td>\n",
       "      <td>0.223061</td>\n",
       "      <td>0.265975</td>\n",
       "      <td>0.293025</td>\n",
       "      <td>0.288711</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   column_cos_0  column_cos_1  column_cos_2  column_cos_3  column_cos_4  \\\n",
       "0      0.424736      0.364050      0.347755      0.394242      0.371025   \n",
       "1      0.454075      0.322620      0.355004      0.271561      0.392342   \n",
       "2      0.395858      0.291708      0.309919      0.223061      0.265975   \n",
       "\n",
       "   column_cos_5  column_cos_6  column_cos_7  column_cos_8  column_cos_9  \\\n",
       "0      0.185690      0.351921           1.0           1.0           1.0   \n",
       "1      0.384383      0.362597           1.0           1.0           1.0   \n",
       "2      0.293025      0.288711           1.0           1.0           1.0   \n",
       "\n",
       "   target  \n",
       "0       5  \n",
       "1       2  \n",
       "2       3  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " subset2.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train2 = pd.concat([subset1, subset2],axis=1, join_axes=[subset1.index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column_euc_0</th>\n",
       "      <th>column_euc_1</th>\n",
       "      <th>column_euc_2</th>\n",
       "      <th>column_euc_3</th>\n",
       "      <th>column_euc_4</th>\n",
       "      <th>column_euc_5</th>\n",
       "      <th>column_euc_6</th>\n",
       "      <th>column_euc_7</th>\n",
       "      <th>column_euc_8</th>\n",
       "      <th>column_euc_9</th>\n",
       "      <th>...</th>\n",
       "      <th>column_cos_1</th>\n",
       "      <th>column_cos_2</th>\n",
       "      <th>column_cos_3</th>\n",
       "      <th>column_cos_4</th>\n",
       "      <th>column_cos_5</th>\n",
       "      <th>column_cos_6</th>\n",
       "      <th>column_cos_7</th>\n",
       "      <th>column_cos_8</th>\n",
       "      <th>column_cos_9</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14.563858</td>\n",
       "      <td>15.262212</td>\n",
       "      <td>17.398178</td>\n",
       "      <td>14.272491</td>\n",
       "      <td>13.339654</td>\n",
       "      <td>9.336262</td>\n",
       "      <td>15.720997</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.364050</td>\n",
       "      <td>0.347755</td>\n",
       "      <td>0.394242</td>\n",
       "      <td>0.371025</td>\n",
       "      <td>0.185690</td>\n",
       "      <td>0.351921</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12.889506</td>\n",
       "      <td>12.285219</td>\n",
       "      <td>16.843704</td>\n",
       "      <td>8.361172</td>\n",
       "      <td>11.918098</td>\n",
       "      <td>17.601221</td>\n",
       "      <td>14.929258</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.322620</td>\n",
       "      <td>0.355004</td>\n",
       "      <td>0.271561</td>\n",
       "      <td>0.392342</td>\n",
       "      <td>0.384383</td>\n",
       "      <td>0.362597</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.857297</td>\n",
       "      <td>11.392319</td>\n",
       "      <td>15.061656</td>\n",
       "      <td>7.184713</td>\n",
       "      <td>8.465475</td>\n",
       "      <td>13.927309</td>\n",
       "      <td>12.249868</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.291708</td>\n",
       "      <td>0.309919</td>\n",
       "      <td>0.223061</td>\n",
       "      <td>0.265975</td>\n",
       "      <td>0.293025</td>\n",
       "      <td>0.288711</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   column_euc_0  column_euc_1  column_euc_2  column_euc_3  column_euc_4  \\\n",
       "0     14.563858     15.262212     17.398178     14.272491     13.339654   \n",
       "1     12.889506     12.285219     16.843704      8.361172     11.918098   \n",
       "2     11.857297     11.392319     15.061656      7.184713      8.465475   \n",
       "\n",
       "   column_euc_5  column_euc_6  column_euc_7  column_euc_8  column_euc_9  \\\n",
       "0      9.336262     15.720997          60.0          60.0          60.0   \n",
       "1     17.601221     14.929258          60.0          60.0          60.0   \n",
       "2     13.927309     12.249868          60.0          60.0          60.0   \n",
       "\n",
       "    ...    column_cos_1  column_cos_2  column_cos_3  column_cos_4  \\\n",
       "0   ...        0.364050      0.347755      0.394242      0.371025   \n",
       "1   ...        0.322620      0.355004      0.271561      0.392342   \n",
       "2   ...        0.291708      0.309919      0.223061      0.265975   \n",
       "\n",
       "   column_cos_5  column_cos_6  column_cos_7  column_cos_8  column_cos_9  \\\n",
       "0      0.185690      0.351921           1.0           1.0           1.0   \n",
       "1      0.384383      0.362597           1.0           1.0           1.0   \n",
       "2      0.293025      0.288711           1.0           1.0           1.0   \n",
       "\n",
       "   target  \n",
       "0       5  \n",
       "1       2  \n",
       "2       3  \n",
       "\n",
       "[3 rows x 21 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train2.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "column_euc_0    64.051060\n",
       "column_euc_1    70.236871\n",
       "column_euc_2    60.000000\n",
       "column_euc_3    60.000000\n",
       "column_euc_4    60.000000\n",
       "column_euc_5    60.000000\n",
       "column_euc_6    60.221045\n",
       "column_euc_7    60.000000\n",
       "column_euc_8    60.000000\n",
       "column_euc_9    60.000000\n",
       "column_cos_0     1.466755\n",
       "column_cos_1     1.606138\n",
       "column_cos_2     1.552746\n",
       "column_cos_3     1.544334\n",
       "column_cos_4     1.542615\n",
       "column_cos_5     1.477041\n",
       "column_cos_6     1.544371\n",
       "column_cos_7     1.450005\n",
       "column_cos_8     1.118746\n",
       "column_cos_9     1.023689\n",
       "target           9.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train2.apply(max, axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scaler = MinMaxScaler()\n",
    "X = scaler.fit_transform(train2.iloc[:,:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.21673264, 0.20752197, 0.27581956, ..., 0.67938476, 0.88892501,\n",
       "        0.97551015],\n",
       "       [0.19023154, 0.16460763, 0.26639416, ..., 0.67938476, 0.88892501,\n",
       "        0.97551015],\n",
       "       [0.17389408, 0.15173618, 0.23610146, ..., 0.67938476, 0.88892501,\n",
       "        0.97551015],\n",
       "       ...,\n",
       "       [0.19892814, 0.163572  , 0.26380138, ..., 0.67938476, 0.88892501,\n",
       "        0.97551015],\n",
       "       [0.17695844, 0.14612473, 0.25149443, ..., 0.67938476, 0.88892501,\n",
       "        0.97551015],\n",
       "       [0.27575241, 0.23371491, 0.39075045, ..., 0.67938476, 0.88892501,\n",
       "        0.97551015]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_x, test_x, train_y, test_y = train_test_split(X,\n",
    "train.iloc[:,-1], train_size=0.8, random_state = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multinomial Logistic regression Train Accuracy :  0.6373448858212791\n",
      "Multinomial Logistic regression Test Accuracy :  0.6398026315789473\n"
     ]
    }
   ],
   "source": [
    "mul_lr = linear_model.LogisticRegression(multi_class='multinomial', solver='newton-cg')\n",
    "mul_lr.fit(train_x, train_y)\n",
    "\n",
    "print(\"Multinomial Logistic regression Train Accuracy : \", metrics.accuracy_score(train_y, mul_lr.predict(train_x)))\n",
    "print(\"Multinomial Logistic regression Test Accuracy : \", metrics.accuracy_score(test_y, mul_lr.predict(test_x)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Logistic-Regression with Root Match feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predicted = pd.read_csv(\"train_detect_sent.csv\").reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predicted = predicted[predicted[\"sentences\"].apply(lambda x: len(ast.literal_eval(x)))<11].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(85119, 14)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_columns_from_root(train):\n",
    "    \n",
    "    for i in range(train.shape[0]):\n",
    "        if len(ast.literal_eval(train[\"root_match_idx\"][i])) == 0: pass\n",
    "        \n",
    "        else:\n",
    "            for item in ast.literal_eval(train[\"root_match_idx\"][i]):\n",
    "                train.loc[i, \"column_root_\"+\"%s\"%item] = 1\n",
    "    return train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predicted = get_columns_from_root(predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>answer_start</th>\n",
       "      <td>515</td>\n",
       "      <td>188</td>\n",
       "      <td>279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>context</th>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>question</th>\n",
       "      <td>To whom did the Virgin Mary allegedly appear i...</td>\n",
       "      <td>What is in front of the Notre Dame Main Building?</td>\n",
       "      <td>The Basilica of the Sacred heart at Notre Dame...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text</th>\n",
       "      <td>Saint Bernadette Soubirous</td>\n",
       "      <td>a copper statue of Christ</td>\n",
       "      <td>the Main Building</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sentences</th>\n",
       "      <td>['Architecturally, the school has a Catholic c...</td>\n",
       "      <td>['Architecturally, the school has a Catholic c...</td>\n",
       "      <td>['Architecturally, the school has a Catholic c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>quest_emb</th>\n",
       "      <td>[[ 0.11010079  0.11422941  0.11560896 ...  0.0...</td>\n",
       "      <td>[[ 0.10951651  0.11030623  0.05210007 ... -0.0...</td>\n",
       "      <td>[[ 0.01195647  0.14930707  0.02660049 ...  0.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sent_emb</th>\n",
       "      <td>[array([ 0.05519997,  0.05013141,  0.04787038,...</td>\n",
       "      <td>[array([ 0.05519997,  0.05013141,  0.04787038,...</td>\n",
       "      <td>[array([ 0.05519997,  0.05013141,  0.04787038,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cosine_sim</th>\n",
       "      <td>[0.424736299052452, 0.36405004106069117, 0.347...</td>\n",
       "      <td>[0.45407456884452513, 0.32262004808444933, 0.3...</td>\n",
       "      <td>[0.39585783692319865, 0.29170832145169434, 0.3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>euclidean_dis</th>\n",
       "      <td>[14.563858, 15.262212, 17.398178, 14.272491, 1...</td>\n",
       "      <td>[12.889506, 12.285219, 16.843704, 8.361172, 11...</td>\n",
       "      <td>[11.857297, 11.392319, 15.061656, 7.1847134, 8...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_idx_cos</th>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pred_idx_euc</th>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>root_match_idx</th>\n",
       "      <td>[5]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[1, 2, 3, 4, 5, 6]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>root_match_idx_first</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_5</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_9</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                      0  \\\n",
       "answer_start                                                        515   \n",
       "context               Architecturally, the school has a Catholic cha...   \n",
       "question              To whom did the Virgin Mary allegedly appear i...   \n",
       "text                                         Saint Bernadette Soubirous   \n",
       "sentences             ['Architecturally, the school has a Catholic c...   \n",
       "quest_emb             [[ 0.11010079  0.11422941  0.11560896 ...  0.0...   \n",
       "target                                                                5   \n",
       "sent_emb              [array([ 0.05519997,  0.05013141,  0.04787038,...   \n",
       "cosine_sim            [0.424736299052452, 0.36405004106069117, 0.347...   \n",
       "euclidean_dis         [14.563858, 15.262212, 17.398178, 14.272491, 1...   \n",
       "pred_idx_cos                                                          5   \n",
       "pred_idx_euc                                                          5   \n",
       "root_match_idx                                                      [5]   \n",
       "root_match_idx_first                                                  5   \n",
       "column_root_5                                                         1   \n",
       "column_root_1                                                       NaN   \n",
       "column_root_2                                                       NaN   \n",
       "column_root_3                                                       NaN   \n",
       "column_root_4                                                       NaN   \n",
       "column_root_6                                                       NaN   \n",
       "column_root_0                                                       NaN   \n",
       "column_root_7                                                       NaN   \n",
       "column_root_8                                                       NaN   \n",
       "column_root_9                                                       NaN   \n",
       "\n",
       "                                                                      1  \\\n",
       "answer_start                                                        188   \n",
       "context               Architecturally, the school has a Catholic cha...   \n",
       "question              What is in front of the Notre Dame Main Building?   \n",
       "text                                          a copper statue of Christ   \n",
       "sentences             ['Architecturally, the school has a Catholic c...   \n",
       "quest_emb             [[ 0.10951651  0.11030623  0.05210007 ... -0.0...   \n",
       "target                                                                2   \n",
       "sent_emb              [array([ 0.05519997,  0.05013141,  0.04787038,...   \n",
       "cosine_sim            [0.45407456884452513, 0.32262004808444933, 0.3...   \n",
       "euclidean_dis         [12.889506, 12.285219, 16.843704, 8.361172, 11...   \n",
       "pred_idx_cos                                                          3   \n",
       "pred_idx_euc                                                          3   \n",
       "root_match_idx                                                       []   \n",
       "root_match_idx_first                                                  0   \n",
       "column_root_5                                                       NaN   \n",
       "column_root_1                                                       NaN   \n",
       "column_root_2                                                       NaN   \n",
       "column_root_3                                                       NaN   \n",
       "column_root_4                                                       NaN   \n",
       "column_root_6                                                       NaN   \n",
       "column_root_0                                                       NaN   \n",
       "column_root_7                                                       NaN   \n",
       "column_root_8                                                       NaN   \n",
       "column_root_9                                                       NaN   \n",
       "\n",
       "                                                                      2  \n",
       "answer_start                                                        279  \n",
       "context               Architecturally, the school has a Catholic cha...  \n",
       "question              The Basilica of the Sacred heart at Notre Dame...  \n",
       "text                                                  the Main Building  \n",
       "sentences             ['Architecturally, the school has a Catholic c...  \n",
       "quest_emb             [[ 0.01195647  0.14930707  0.02660049 ...  0.0...  \n",
       "target                                                                3  \n",
       "sent_emb              [array([ 0.05519997,  0.05013141,  0.04787038,...  \n",
       "cosine_sim            [0.39585783692319865, 0.29170832145169434, 0.3...  \n",
       "euclidean_dis         [11.857297, 11.392319, 15.061656, 7.1847134, 8...  \n",
       "pred_idx_cos                                                          3  \n",
       "pred_idx_euc                                                          3  \n",
       "root_match_idx                                       [1, 2, 3, 4, 5, 6]  \n",
       "root_match_idx_first                                                  1  \n",
       "column_root_5                                                         1  \n",
       "column_root_1                                                         1  \n",
       "column_root_2                                                         1  \n",
       "column_root_3                                                         1  \n",
       "column_root_4                                                         1  \n",
       "column_root_6                                                         1  \n",
       "column_root_0                                                       NaN  \n",
       "column_root_7                                                       NaN  \n",
       "column_root_8                                                       NaN  \n",
       "column_root_9                                                       NaN  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted.head(3).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subset3 = predicted[[\"column_root_0\",\"column_root_1\",\"column_root_2\",\"column_root_3\",\"column_root_4\",\"column_root_5\",\\\n",
    "             \"column_root_6\",\"column_root_7\",\"column_root_8\",\"column_root_9\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subset3.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train3 = pd.concat([subset3, train2],axis=1, join_axes=[subset3.index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>column_root_0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_5</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_6</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_7</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_8</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_root_9</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_0</th>\n",
       "      <td>14.563858</td>\n",
       "      <td>12.889506</td>\n",
       "      <td>11.857297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_1</th>\n",
       "      <td>15.262212</td>\n",
       "      <td>12.285219</td>\n",
       "      <td>11.392319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_2</th>\n",
       "      <td>17.398178</td>\n",
       "      <td>16.843704</td>\n",
       "      <td>15.061656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_3</th>\n",
       "      <td>14.272491</td>\n",
       "      <td>8.361172</td>\n",
       "      <td>7.184713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_4</th>\n",
       "      <td>13.339654</td>\n",
       "      <td>11.918098</td>\n",
       "      <td>8.465475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_5</th>\n",
       "      <td>9.336262</td>\n",
       "      <td>17.601221</td>\n",
       "      <td>13.927309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_6</th>\n",
       "      <td>15.720997</td>\n",
       "      <td>14.929258</td>\n",
       "      <td>12.249868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_7</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_8</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_euc_9</th>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>60.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_0</th>\n",
       "      <td>0.424736</td>\n",
       "      <td>0.454075</td>\n",
       "      <td>0.395858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_1</th>\n",
       "      <td>0.364050</td>\n",
       "      <td>0.322620</td>\n",
       "      <td>0.291708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_2</th>\n",
       "      <td>0.347755</td>\n",
       "      <td>0.355004</td>\n",
       "      <td>0.309919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_3</th>\n",
       "      <td>0.394242</td>\n",
       "      <td>0.271561</td>\n",
       "      <td>0.223061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_4</th>\n",
       "      <td>0.371025</td>\n",
       "      <td>0.392342</td>\n",
       "      <td>0.265975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_5</th>\n",
       "      <td>0.185690</td>\n",
       "      <td>0.384383</td>\n",
       "      <td>0.293025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_6</th>\n",
       "      <td>0.351921</td>\n",
       "      <td>0.362597</td>\n",
       "      <td>0.288711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_7</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_8</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>column_cos_9</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       0          1          2\n",
       "column_root_0   0.000000   0.000000   0.000000\n",
       "column_root_1   0.000000   0.000000   1.000000\n",
       "column_root_2   0.000000   0.000000   1.000000\n",
       "column_root_3   0.000000   0.000000   1.000000\n",
       "column_root_4   0.000000   0.000000   1.000000\n",
       "column_root_5   1.000000   0.000000   1.000000\n",
       "column_root_6   0.000000   0.000000   1.000000\n",
       "column_root_7   0.000000   0.000000   0.000000\n",
       "column_root_8   0.000000   0.000000   0.000000\n",
       "column_root_9   0.000000   0.000000   0.000000\n",
       "column_euc_0   14.563858  12.889506  11.857297\n",
       "column_euc_1   15.262212  12.285219  11.392319\n",
       "column_euc_2   17.398178  16.843704  15.061656\n",
       "column_euc_3   14.272491   8.361172   7.184713\n",
       "column_euc_4   13.339654  11.918098   8.465475\n",
       "column_euc_5    9.336262  17.601221  13.927309\n",
       "column_euc_6   15.720997  14.929258  12.249868\n",
       "column_euc_7   60.000000  60.000000  60.000000\n",
       "column_euc_8   60.000000  60.000000  60.000000\n",
       "column_euc_9   60.000000  60.000000  60.000000\n",
       "column_cos_0    0.424736   0.454075   0.395858\n",
       "column_cos_1    0.364050   0.322620   0.291708\n",
       "column_cos_2    0.347755   0.355004   0.309919\n",
       "column_cos_3    0.394242   0.271561   0.223061\n",
       "column_cos_4    0.371025   0.392342   0.265975\n",
       "column_cos_5    0.185690   0.384383   0.293025\n",
       "column_cos_6    0.351921   0.362597   0.288711\n",
       "column_cos_7    1.000000   1.000000   1.000000\n",
       "column_cos_8    1.000000   1.000000   1.000000\n",
       "column_cos_9    1.000000   1.000000   1.000000\n",
       "target          5.000000   2.000000   3.000000"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train3.head(3).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train3 = train3[[\"column_root_0\",\"column_root_1\",\"column_root_2\",\"column_root_3\",\"column_root_4\",\"column_root_5\",\\\n",
    "             \"column_root_6\",\"column_root_7\",\"column_root_8\",\"column_root_9\", \"column_cos_0\",\"column_cos_1\",\\\n",
    "           \"column_cos_2\",\"column_cos_3\",\"column_cos_4\",\"column_cos_5\",\\\n",
    "             \"column_cos_6\",\"column_cos_7\",\"column_cos_8\",\"column_cos_9\", \"target\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_x, test_x, train_y, test_y = train_test_split(train3.iloc[:,:-1],\n",
    "train3.iloc[:,-1], train_size=0.8, random_state = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multinomial Logistic regression Train Accuracy :  0.6421910566120861\n",
      "Multinomial Logistic regression Test Accuracy :  0.6449718045112782\n"
     ]
    }
   ],
   "source": [
    "mul_lr = linear_model.LogisticRegression(multi_class='multinomial', solver='newton-cg')\n",
    "mul_lr.fit(train_x, train_y)\n",
    "\n",
    "print(\"Multinomial Logistic regression Train Accuracy : \", metrics.accuracy_score(train_y, mul_lr.predict(train_x)))\n",
    "print(\"Multinomial Logistic regression Test Accuracy : \", metrics.accuracy_score(test_y, mul_lr.predict(test_x)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multinomial Logistic regression Train Accuracy :  0.7216388868492547\n",
      "Multinomial Logistic regression Test Accuracy :  0.670641447368421\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(min_samples_leaf=8, n_estimators=60)\n",
    "rf.fit(train_x, train_y)\n",
    "\n",
    "print(\"Multinomial Logistic regression Train Accuracy : \", metrics.accuracy_score(train_y, rf.predict(train_x)))\n",
    "print(\"Multinomial Logistic regression Test Accuracy : \", metrics.accuracy_score(test_y, rf.predict(test_x)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XgBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 27 candidates, totalling 81 fits\n",
      "[CV] learning_rate=0.07, max_depth=3, min_child_weight=1 .............\n",
      "[CV] learning_rate=0.07, max_depth=3, min_child_weight=1 .............\n",
      "[CV] learning_rate=0.07, max_depth=3, min_child_weight=1 .............\n",
      "[CV] learning_rate=0.07, max_depth=3, min_child_weight=5 .............\n",
      "[CV]  learning_rate=0.07, max_depth=3, min_child_weight=1, score=0.6720993745044489, total= 1.7min\n",
      "[CV] learning_rate=0.07, max_depth=3, min_child_weight=5 .............\n",
      "[CV]  learning_rate=0.07, max_depth=3, min_child_weight=5, score=0.6727601092414766, total= 1.7min\n",
      "[CV] learning_rate=0.07, max_depth=3, min_child_weight=5 .............\n",
      "[CV]  learning_rate=0.07, max_depth=3, min_child_weight=1, score=0.6748612212529739, total= 1.7min\n",
      "[CV] learning_rate=0.07, max_depth=3, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.07, max_depth=3, min_child_weight=1, score=0.6715576118087685, total= 1.7min\n",
      "[CV] learning_rate=0.07, max_depth=3, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.07, max_depth=3, min_child_weight=10, score=0.6744206538020971, total= 1.7min\n",
      "[CV] learning_rate=0.07, max_depth=3, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.07, max_depth=3, min_child_weight=10, score=0.6724077173817284, total= 1.7min\n",
      "[CV] learning_rate=0.07, max_depth=5, min_child_weight=1 .............\n",
      "[CV]  learning_rate=0.07, max_depth=3, min_child_weight=5, score=0.6732751784298177, total= 1.7min\n",
      "[CV] learning_rate=0.07, max_depth=5, min_child_weight=1 .............\n",
      "[CV]  learning_rate=0.07, max_depth=3, min_child_weight=5, score=0.6705001101564221, total= 1.7min\n",
      "[CV] learning_rate=0.07, max_depth=5, min_child_weight=1 .............\n",
      "[CV]  learning_rate=0.07, max_depth=3, min_child_weight=10, score=0.6710288609825953, total= 1.6min\n",
      "[CV] learning_rate=0.07, max_depth=5, min_child_weight=5 .............\n",
      "[CV]  learning_rate=0.07, max_depth=5, min_child_weight=1, score=0.6777263714474554, total= 2.3min\n",
      "[CV] learning_rate=0.07, max_depth=5, min_child_weight=5 .............\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  10 tasks      | elapsed:  5.9min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  learning_rate=0.07, max_depth=5, min_child_weight=1, score=0.6782662320500397, total= 2.3min\n",
      "[CV] learning_rate=0.07, max_depth=5, min_child_weight=5 .............\n",
      "[CV]  learning_rate=0.07, max_depth=5, min_child_weight=1, score=0.6802802008987576, total= 2.3min\n",
      "[CV] learning_rate=0.07, max_depth=5, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.07, max_depth=5, min_child_weight=5, score=0.6796757994890318, total= 2.4min\n",
      "[CV] learning_rate=0.07, max_depth=5, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.07, max_depth=5, min_child_weight=10, score=0.6782221830675711, total= 2.3min\n",
      "[CV] learning_rate=0.07, max_depth=5, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.07, max_depth=5, min_child_weight=5, score=0.6802802008987576, total= 2.4min\n",
      "[CV] learning_rate=0.07, max_depth=10, min_child_weight=1 ............\n",
      "[CV]  learning_rate=0.07, max_depth=5, min_child_weight=5, score=0.67724168319013, total= 2.4min\n",
      "[CV] learning_rate=0.07, max_depth=10, min_child_weight=1 ............\n",
      "[CV]  learning_rate=0.07, max_depth=5, min_child_weight=10, score=0.6797515199577056, total= 2.3min\n",
      "[CV] learning_rate=0.07, max_depth=10, min_child_weight=1 ............\n",
      "[CV]  learning_rate=0.07, max_depth=5, min_child_weight=10, score=0.6768891826393478, total= 2.3min\n",
      "[CV] learning_rate=0.07, max_depth=10, min_child_weight=5 ............\n",
      "[CV]  learning_rate=0.07, max_depth=10, min_child_weight=1, score=0.6742137256629371, total= 3.8min\n",
      "[CV] learning_rate=0.07, max_depth=10, min_child_weight=5 ............\n",
      "[CV]  learning_rate=0.07, max_depth=10, min_child_weight=1, score=0.6802802008987576, total= 3.8min\n",
      "[CV] learning_rate=0.07, max_depth=10, min_child_weight=5 ............\n",
      "[CV]  learning_rate=0.07, max_depth=10, min_child_weight=1, score=0.677065432914739, total= 3.6min\n",
      "[CV] learning_rate=0.07, max_depth=10, min_child_weight=10 ...........\n",
      "[CV]  learning_rate=0.07, max_depth=10, min_child_weight=5, score=0.6750066073473703, total= 3.4min\n",
      "[CV] learning_rate=0.07, max_depth=10, min_child_weight=10 ...........\n",
      "[CV]  learning_rate=0.07, max_depth=10, min_child_weight=5, score=0.6746419916281119, total= 3.4min\n",
      "[CV] learning_rate=0.07, max_depth=10, min_child_weight=10 ...........\n",
      "[CV]  learning_rate=0.07, max_depth=10, min_child_weight=5, score=0.680368314388933, total= 3.5min\n",
      "[CV] learning_rate=0.1, max_depth=3, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.07, max_depth=10, min_child_weight=10, score=0.6758435380142719, total= 3.6min\n",
      "[CV] learning_rate=0.1, max_depth=3, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=3, min_child_weight=1, score=0.6760197339441459, total= 1.7min\n",
      "[CV] learning_rate=0.1, max_depth=3, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.07, max_depth=10, min_child_weight=10, score=0.6793990659970042, total= 3.6min\n",
      "[CV] learning_rate=0.1, max_depth=3, min_child_weight=5 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=3, min_child_weight=1, score=0.6777249096836726, total= 1.6min\n",
      "[CV] learning_rate=0.1, max_depth=3, min_child_weight=5 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=3, min_child_weight=1, score=0.674069178233091, total= 1.7min\n",
      "[CV] learning_rate=0.1, max_depth=3, min_child_weight=5 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=3, min_child_weight=5, score=0.6749185093824333, total= 1.7min\n",
      "[CV] learning_rate=0.1, max_depth=3, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.07, max_depth=10, min_child_weight=10, score=0.6754351178673716, total= 3.6min\n",
      "[CV] learning_rate=0.1, max_depth=3, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.1, max_depth=3, min_child_weight=5, score=0.6771521719975329, total= 1.6min\n",
      "[CV] learning_rate=0.1, max_depth=3, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.1, max_depth=3, min_child_weight=5, score=0.6762282441066314, total= 1.6min\n",
      "[CV] learning_rate=0.1, max_depth=5, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=3, min_child_weight=10, score=0.6754470971720553, total= 1.6min\n",
      "[CV] learning_rate=0.1, max_depth=5, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=3, min_child_weight=10, score=0.6767116045466561, total= 1.6min\n",
      "[CV] learning_rate=0.1, max_depth=5, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=3, min_child_weight=10, score=0.6738488653888521, total= 1.6min\n",
      "[CV] learning_rate=0.1, max_depth=5, min_child_weight=5 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=5, min_child_weight=1, score=0.6799841423663113, total= 2.1min\n",
      "[CV] learning_rate=0.1, max_depth=5, min_child_weight=5 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=5, min_child_weight=1, score=0.6807207683496344, total= 2.1min\n",
      "[CV] learning_rate=0.1, max_depth=5, min_child_weight=5 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=5, min_child_weight=1, score=0.6786957479621062, total= 2.1min\n",
      "[CV] learning_rate=0.1, max_depth=5, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.1, max_depth=5, min_child_weight=5, score=0.6779138401902916, total= 2.2min\n",
      "[CV] learning_rate=0.1, max_depth=5, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.1, max_depth=5, min_child_weight=5, score=0.6801920874085823, total= 2.3min\n",
      "[CV] learning_rate=0.1, max_depth=5, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.1, max_depth=5, min_child_weight=5, score=0.6783873099801718, total= 2.3min\n",
      "[CV] learning_rate=0.1, max_depth=10, min_child_weight=1 .............\n",
      "[CV]  learning_rate=0.1, max_depth=5, min_child_weight=10, score=0.6780019381552286, total= 2.3min\n",
      "[CV] learning_rate=0.1, max_depth=10, min_child_weight=1 .............\n",
      "[CV]  learning_rate=0.1, max_depth=5, min_child_weight=10, score=0.6804564278791083, total= 2.2min\n",
      "[CV] learning_rate=0.1, max_depth=10, min_child_weight=1 .............\n",
      "[CV]  learning_rate=0.1, max_depth=5, min_child_weight=10, score=0.6790041859440406, total= 2.2min\n",
      "[CV] learning_rate=0.1, max_depth=10, min_child_weight=5 .............\n",
      "[CV]  learning_rate=0.1, max_depth=10, min_child_weight=1, score=0.6753589992071183, total= 3.7min\n",
      "[CV] learning_rate=0.1, max_depth=10, min_child_weight=5 .............\n",
      "[CV]  learning_rate=0.1, max_depth=10, min_child_weight=1, score=0.6775486827033219, total= 3.7min\n",
      "[CV] learning_rate=0.1, max_depth=10, min_child_weight=5 .............\n",
      "[CV]  learning_rate=0.1, max_depth=10, min_child_weight=1, score=0.6763604318131747, total= 3.6min\n",
      "[CV] learning_rate=0.1, max_depth=10, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.1, max_depth=10, min_child_weight=5, score=0.6751828032772443, total= 3.5min\n",
      "[CV] learning_rate=0.1, max_depth=10, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.1, max_depth=10, min_child_weight=5, score=0.6794431227420918, total= 3.3min\n",
      "[CV] learning_rate=0.1, max_depth=10, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.1, max_depth=10, min_child_weight=5, score=0.6753029301608284, total= 3.3min\n",
      "[CV] learning_rate=0.2, max_depth=3, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=10, min_child_weight=10, score=0.6754030481895869, total= 3.2min\n",
      "[CV] learning_rate=0.2, max_depth=3, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.2, max_depth=3, min_child_weight=1, score=0.6787948198396617, total= 1.6min\n",
      "[CV] learning_rate=0.2, max_depth=3, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.1, max_depth=10, min_child_weight=10, score=0.6785619878403384, total= 3.3min\n",
      "[CV] learning_rate=0.2, max_depth=3, min_child_weight=5 ..............\n",
      "[CV]  learning_rate=0.2, max_depth=3, min_child_weight=1, score=0.6790025552912151, total= 1.6min\n",
      "[CV] learning_rate=0.2, max_depth=3, min_child_weight=5 ..............\n",
      "[CV]  learning_rate=0.2, max_depth=3, min_child_weight=1, score=0.6782991848424763, total= 1.6min\n",
      "[CV] learning_rate=0.2, max_depth=3, min_child_weight=5 ..............\n",
      "[CV]  learning_rate=0.2, max_depth=3, min_child_weight=5, score=0.6782221830675711, total= 1.6min\n",
      "[CV] learning_rate=0.2, max_depth=3, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.1, max_depth=10, min_child_weight=10, score=0.6760960564000881, total= 3.4min\n",
      "[CV] learning_rate=0.2, max_depth=3, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.2, max_depth=3, min_child_weight=5, score=0.6789144418010398, total= 1.6min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] learning_rate=0.2, max_depth=3, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.2, max_depth=3, min_child_weight=5, score=0.6776823088786076, total= 1.5min\n",
      "[CV] learning_rate=0.2, max_depth=5, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.2, max_depth=3, min_child_weight=10, score=0.6774293013831381, total= 1.6min\n",
      "[CV] learning_rate=0.2, max_depth=5, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.2, max_depth=3, min_child_weight=10, score=0.6798836901929686, total= 1.5min\n",
      "[CV] learning_rate=0.2, max_depth=5, min_child_weight=1 ..............\n",
      "[CV]  learning_rate=0.2, max_depth=3, min_child_weight=10, score=0.6771976206212822, total= 1.5min\n",
      "[CV] learning_rate=0.2, max_depth=5, min_child_weight=5 ..............\n",
      "[CV]  learning_rate=0.2, max_depth=5, min_child_weight=1, score=0.6776054973130121, total= 2.1min\n",
      "[CV] learning_rate=0.2, max_depth=5, min_child_weight=5 ..............\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  64 tasks      | elapsed: 40.3min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  learning_rate=0.2, max_depth=5, min_child_weight=1, score=0.6790466120363028, total= 2.1min\n",
      "[CV] learning_rate=0.2, max_depth=5, min_child_weight=5 ..............\n",
      "[CV]  learning_rate=0.2, max_depth=5, min_child_weight=1, score=0.6768891826393478, total= 2.1min\n",
      "[CV] learning_rate=0.2, max_depth=5, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.2, max_depth=5, min_child_weight=5, score=0.6764161747863624, total= 2.1min\n",
      "[CV] learning_rate=0.2, max_depth=5, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.2, max_depth=5, min_child_weight=5, score=0.6787822715657768, total= 2.3min\n",
      "[CV] learning_rate=0.2, max_depth=5, min_child_weight=10 .............\n",
      "[CV]  learning_rate=0.2, max_depth=5, min_child_weight=5, score=0.6767569949328046, total= 2.3min\n",
      "[CV] learning_rate=0.2, max_depth=10, min_child_weight=1 .............\n",
      "[CV]  learning_rate=0.2, max_depth=5, min_child_weight=10, score=0.677165007488327, total= 2.3min\n",
      "[CV] learning_rate=0.2, max_depth=10, min_child_weight=1 .............\n",
      "[CV]  learning_rate=0.2, max_depth=5, min_child_weight=10, score=0.6793550092519165, total= 2.4min\n",
      "[CV] learning_rate=0.2, max_depth=10, min_child_weight=1 .............\n",
      "[CV]  learning_rate=0.2, max_depth=5, min_child_weight=10, score=0.6772857457589777, total= 2.3min\n",
      "[CV] learning_rate=0.2, max_depth=10, min_child_weight=5 .............\n",
      "[CV]  learning_rate=0.2, max_depth=10, min_child_weight=1, score=0.6698969253810237, total= 3.7min\n",
      "[CV] learning_rate=0.2, max_depth=10, min_child_weight=5 .............\n",
      "[CV]  learning_rate=0.2, max_depth=10, min_child_weight=1, score=0.6766234910564808, total= 3.7min\n",
      "[CV] learning_rate=0.2, max_depth=10, min_child_weight=5 .............\n",
      "[CV]  learning_rate=0.2, max_depth=10, min_child_weight=1, score=0.6675038554747742, total= 3.5min\n",
      "[CV] learning_rate=0.2, max_depth=10, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.2, max_depth=10, min_child_weight=5, score=0.6699850233459607, total= 3.3min\n",
      "[CV] learning_rate=0.2, max_depth=10, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.2, max_depth=10, min_child_weight=5, score=0.6769318882720945, total= 3.2min\n",
      "[CV] learning_rate=0.2, max_depth=10, min_child_weight=10 ............\n",
      "[CV]  learning_rate=0.2, max_depth=10, min_child_weight=5, score=0.67014760960564, total= 3.2min\n",
      "[CV]  learning_rate=0.2, max_depth=10, min_child_weight=10, score=0.6706457580829883, total= 3.1min\n",
      "[CV]  learning_rate=0.2, max_depth=10, min_child_weight=10, score=0.6739360296061327, total= 2.6min\n",
      "[CV]  learning_rate=0.2, max_depth=10, min_child_weight=10, score=0.6715135492399207, total= 2.1min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  81 out of  81 | elapsed: 52.2min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'max_depth': [3, 5, 10], 'min_child_weight': [1, 5, 10], 'learning_rate': [0.07, 0.1, 0.2]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=5)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = xgb.XGBClassifier()\n",
    "param_dist = {\"max_depth\": [3,5,10],\n",
    "              \"min_child_weight\" : [1,5,10],\n",
    "              \"learning_rate\": [0.07, 0.1,0.2],\n",
    "               }\n",
    "\n",
    "# run randomized search\n",
    "grid_search = GridSearchCV(model, param_grid=param_dist, cv = 3, \n",
    "                                   verbose=5, n_jobs=-1)\n",
    "grid_search.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=1, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multinomial Logistic regression Train Accuracy :  0.7062926793450327\n",
      "Multinomial Logistic regression Test Accuracy :  0.685561560150376\n"
     ]
    }
   ],
   "source": [
    "xg = xgb.XGBClassifier(max_depth=5)\n",
    "xg.fit(train_x, train_y)\n",
    "\n",
    "print(\"Multinomial Logistic regression Train Accuracy : \", metrics.accuracy_score(train_y, xg.predict(train_x)))\n",
    "print(\"Multinomial Logistic regression Test Accuracy : \", metrics.accuracy_score(test_y, xg.predict(test_x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xgb.XGBClassifier??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}