stepping_stones/NB_iter3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Bayes Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, a Naive Bayes model is run on a iid sampled data set of approximately 670K rows of data.  This notebook was run on an AWS SageMaker ml.c5.4xlarge instance."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import string\n",
    "import re\n",
    "import string\n",
    "import numpy as np\n",
    "import datetime\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer \n",
    "from nltk.stem import LancasterStemmer \n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import sklearn\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn import svm\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/ec2-user/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import feature_generation_functions as fgf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import model_functions as mf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle_functions as pf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and shuffle data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"s3://advancedml-koch-mathur-hinkson/train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1804874, 45)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Label comments as toxic (\"1\") or nontoxic (\"0\") using 0.5 threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['toxicity_category'] = train.target.apply(lambda x: 1 if x > 0.5 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1804874, 46)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split into train_set and validation_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Citation: https://stackoverflow.com/questions/24147278/how-do-i-create-test-and-train-samples-from-one-dataframe-with-pandas\n",
    "msk = np.random.rand(len(train)) < 0.8\n",
    "train_set = train[msk]\n",
    "validation_set = train[~msk]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    1358850\n",
      "1      85018\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(train_set.toxicity_category.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    339586\n",
      "1     21420\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(validation_set.toxicity_category.value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Randomly sample train_set to create a smaller data frame (train_sample) to run SVM on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sample = train_set.sample(frac=0.5, replace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    679613\n",
      "1     42321\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(train_sample.toxicity_category.value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaned with stopwords...Elapsed Time:  0.177 minutes\n",
      "Cleaned without stopwords...Elapsed Time:  0.245 minutes\n",
      "Stemmed (Porter)...Elapsed Time:  5.661 minutes\n",
      "Stemmed (Lancaster)...Elapsed Time:  4.641 minutes\n",
      "\n",
      "DONE GENERATING FEATURES\n"
     ]
    }
   ],
   "source": [
    "train_df = fgf.generate_NB_SVM_features(train_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pickled and sent to bucket!\n"
     ]
    }
   ],
   "source": [
    "pf.write_pickle_to_s3bucket(filename='NB_final_720K', \n",
    "                            bucket_name='advancedml-koch-mathur-hinkson', \n",
    "                            df=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaned with stopwords...Elapsed Time:  0.088 minutes\n",
      "Cleaned without stopwords...Elapsed Time:  0.122 minutes\n",
      "Stemmed (Porter)...Elapsed Time:  2.797 minutes\n",
      "Stemmed (Lancaster)...Elapsed Time:  2.332 minutes\n",
      "\n",
      "DONE GENERATING FEATURES\n"
     ]
    }
   ],
   "source": [
    "validation_df = fgf.generate_NB_SVM_features(validation_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pf.write_pickle_to_s3bucket(filename='NB_final_720K', \n",
    "                            bucket_name='advancedml-koch-mathur-hinkson', \n",
    "                            df=validation_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reshaping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_df = pf.read_pickle(filename='NB_final_720K', bucket_name='advancedml-koch-mathur-hinkson')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "toxic = train_df[train_df.toxicity_category == 1]\n",
    "nontoxic = train_df[train_df.toxicity_category == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((721934, 50), (42321, 50), (679613, 50))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape, toxic.shape, nontoxic.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reshape the dataset to include an equal number of toxic and nontoxic samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "quarter = len(toxic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_df = train_df.sample(quarter*4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    126963\n",
      "1     42321\n",
      "Name: toxicity_category, dtype: int64\n",
      "1    84642\n",
      "0    84642\n",
      "Name: toxicity_category, dtype: int64\n",
      "1    126963\n",
      "0     42321\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "prepared_25 = toxic.append(nontoxic.sample(len(toxic)*3))\n",
    "prepared_25 = prepared_25.sample(frac=1).reset_index(drop=True)\n",
    "print(prepared_25.toxicity_category.value_counts())\n",
    "\n",
    "prepared_50 = toxic.append(toxic).append(nontoxic.sample(len(toxic)*2))\n",
    "prepared_50 = prepared_50.sample(frac=1).reset_index(drop=True)\n",
    "print(prepared_50.toxicity_category.value_counts())\n",
    "\n",
    "prepared_75 = toxic.append(toxic).append(toxic).append(nontoxic.sample(len(toxic)))\n",
    "prepared_75 = prepared_75.sample(frac=1).reset_index(drop=True)\n",
    "print(prepared_75.toxicity_category.value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1. 2019-05-30 19:17:34.605432\n",
      "['random_df', 'cleaned_w_stopwords_str']\n",
      "fitting model now\n",
      "Overall Accuracy: 0.941, Target Accuracy: 0.004, Non-Target Accuracy: 0.999\n",
      "\n",
      "2. 2019-05-30 19:17:50.552724\n",
      "['random_df', 'cleaned_no_stem_str']\n",
      "fitting model now\n",
      "Overall Accuracy: 0.941, Target Accuracy: 0.005, Non-Target Accuracy: 0.999\n",
      "\n",
      "3. 2019-05-30 19:18:01.751315\n",
      "['random_df', 'cleaned_porter_str']\n",
      "fitting model now\n",
      "Overall Accuracy: 0.941, Target Accuracy: 0.005, Non-Target Accuracy: 0.999\n",
      "\n",
      "4. 2019-05-30 19:18:12.192631\n",
      "['random_df', 'cleaned_lancaster_str']\n",
      "fitting model now\n",
      "Overall Accuracy: 0.941, Target Accuracy: 0.004, Non-Target Accuracy: 0.999\n",
      "\n",
      "5. 2019-05-30 19:18:22.405352\n",
      "['prepared_50', 'cleaned_w_stopwords_str']\n",
      "fitting model now\n",
      "Overall Accuracy: 0.822, Target Accuracy: 0.89, Non-Target Accuracy: 0.756\n",
      "\n",
      "6. 2019-05-30 19:18:36.693717\n",
      "['prepared_50', 'cleaned_no_stem_str']\n",
      "fitting model now\n",
      "Overall Accuracy: 0.821, Target Accuracy: 0.89, Non-Target Accuracy: 0.752\n",
      "\n",
      "7. 2019-05-30 19:18:47.203954\n",
      "['prepared_50', 'cleaned_porter_str']\n",
      "fitting model now\n",
      "Overall Accuracy: 0.817, Target Accuracy: 0.882, Non-Target Accuracy: 0.752\n",
      "\n",
      "8. 2019-05-30 19:18:56.973270\n",
      "['prepared_50', 'cleaned_lancaster_str']\n",
      "fitting model now\n",
      "Overall Accuracy: 0.815, Target Accuracy: 0.879, Non-Target Accuracy: 0.753\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_metric = 0\n",
    "metric_dict = ''\n",
    "model_factors = []\n",
    "\n",
    "SUBSET_OF_INTEREST = \"Target\"\n",
    "METRIC_OF_INTEREST = \"F1\"\n",
    "\n",
    "dfs = [random_df, prepared_50]\n",
    "label = [\"random_df\", \"prepared_50\"]\n",
    "\n",
    "mn = 0\n",
    "\n",
    "for i in range(len(dfs)):\n",
    "    for text in ['cleaned_w_stopwords_str', 'cleaned_no_stem_str', 'cleaned_porter_str',\n",
    "       'cleaned_lancaster_str']:\n",
    "\n",
    "        factors = [label[i], text]\n",
    "        mn += 1\n",
    "        print(\"{}. {}\".format(mn, datetime.datetime.now()))\n",
    "        print(factors)\n",
    "\n",
    "        clf, output = mf.run_model(dfs[i], comments = text, model_type = \"MultiNB\")\n",
    "        metrics = mf.get_metrics(output, should_print=False)\n",
    "        metric_of_interest = metrics[SUBSET_OF_INTEREST][METRIC_OF_INTEREST]\n",
    "        \n",
    "        print(\"Overall Accuracy: {}, Target Accuracy: {}, Non-Target Accuracy: {}\".format(metrics[\"Overall\"][\"Accuracy\"], metrics[\"Target\"][\"Accuracy\"], metrics[\"Non-Target\"][\"Accuracy\"]))\n",
    "        print() \n",
    "        \n",
    "        if (metric_of_interest > best_metric) and metric_of_interest < 0.95:\n",
    "            best_metric = metric_of_interest\n",
    "            \n",
    "            model_factors = factors\n",
    "            metric_dict = metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>severe_toxicity</th>\n",
       "      <th>obscene</th>\n",
       "      <th>identity_attack</th>\n",
       "      <th>insult</th>\n",
       "      <th>threat</th>\n",
       "      <th>asian</th>\n",
       "      <th>atheist</th>\n",
       "      <th>...</th>\n",
       "      <th>likes</th>\n",
       "      <th>disagree</th>\n",
       "      <th>sexual_explicit</th>\n",
       "      <th>identity_annotator_count</th>\n",
       "      <th>toxicity_annotator_count</th>\n",
       "      <th>toxicity_category</th>\n",
       "      <th>cleaned_w_stopwords_str</th>\n",
       "      <th>cleaned_no_stem_str</th>\n",
       "      <th>cleaned_porter_str</th>\n",
       "      <th>cleaned_lancaster_str</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>111477</th>\n",
       "      <td>378509</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>For the small minded GOP, politics are more im...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>For the small minded GOP politics are more imp...</td>\n",
       "      <td>for small minded gop politics important welfar...</td>\n",
       "      <td>for small mind gop polit import welfar countri...</td>\n",
       "      <td>for smal mind gop polit import welf country sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>826004</th>\n",
       "      <td>5131799</td>\n",
       "      <td>0.728571</td>\n",
       "      <td>What a bunch of idiots the religious right are...</td>\n",
       "      <td>0.128571</td>\n",
       "      <td>0.128571</td>\n",
       "      <td>0.485714</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>What a bunch of idiots the religious right are...</td>\n",
       "      <td>what bunch idiots religious right these fools ...</td>\n",
       "      <td>what bunch idiot religi right these fool belie...</td>\n",
       "      <td>what bunch idiot religy right thes fool believ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>576164</th>\n",
       "      <td>947214</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>Brinksmanhip might be okay for high school stu...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>Brinksmanhip might be okay for high school stu...</td>\n",
       "      <td>brinksmanhip might okay high school student co...</td>\n",
       "      <td>brinksmanhip might okay high school student co...</td>\n",
       "      <td>brinksmanhip might okay high school stud counc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1655937</th>\n",
       "      <td>6151798</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Ahhh yes, another  no taxes less government st...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>Ahhh yes another  no taxes less government sto...</td>\n",
       "      <td>ahhh yes another taxes less government story w...</td>\n",
       "      <td>ahhh ye anoth tax less govern stori wow i neve...</td>\n",
       "      <td>ahhh ye anoth tax less govern story wow i nev ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1465266</th>\n",
       "      <td>5912126</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>\"Perfect\" in what regard?</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>Perfect in what regard</td>\n",
       "      <td>perfect regard</td>\n",
       "      <td>perfect regard</td>\n",
       "      <td>perfect regard</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id    target                                       comment_text  \\\n",
       "111477    378509  0.200000  For the small minded GOP, politics are more im...   \n",
       "826004   5131799  0.728571  What a bunch of idiots the religious right are...   \n",
       "576164    947214  0.166667  Brinksmanhip might be okay for high school stu...   \n",
       "1655937  6151798  0.000000  Ahhh yes, another  no taxes less government st...   \n",
       "1465266  5912126  0.000000                          \"Perfect\" in what regard?   \n",
       "\n",
       "         severe_toxicity   obscene  identity_attack    insult  threat  asian  \\\n",
       "111477          0.000000  0.000000         0.000000  0.200000     0.0    0.0   \n",
       "826004          0.128571  0.128571         0.485714  0.714286     0.0    0.0   \n",
       "576164          0.000000  0.000000         0.000000  0.166667     0.0    NaN   \n",
       "1655937         0.000000  0.000000         0.000000  0.000000     0.0    NaN   \n",
       "1465266         0.000000  0.000000         0.000000  0.000000     0.0    NaN   \n",
       "\n",
       "         atheist  ...  likes  disagree  sexual_explicit  \\\n",
       "111477       0.0  ...      8         0              0.0   \n",
       "826004       0.0  ...      0         0              0.0   \n",
       "576164       NaN  ...      3         0              0.0   \n",
       "1655937      NaN  ...      1         1              0.0   \n",
       "1465266      NaN  ...      0         0              0.0   \n",
       "\n",
       "         identity_annotator_count  toxicity_annotator_count  \\\n",
       "111477                          4                        10   \n",
       "826004                          5                        70   \n",
       "576164                          0                         6   \n",
       "1655937                         0                         4   \n",
       "1465266                         0                         4   \n",
       "\n",
       "         toxicity_category                            cleaned_w_stopwords_str  \\\n",
       "111477                   0  For the small minded GOP politics are more imp...   \n",
       "826004                   1  What a bunch of idiots the religious right are...   \n",
       "576164                   0  Brinksmanhip might be okay for high school stu...   \n",
       "1655937                  0  Ahhh yes another  no taxes less government sto...   \n",
       "1465266                  0                             Perfect in what regard   \n",
       "\n",
       "                                       cleaned_no_stem_str  \\\n",
       "111477   for small minded gop politics important welfar...   \n",
       "826004   what bunch idiots religious right these fools ...   \n",
       "576164   brinksmanhip might okay high school student co...   \n",
       "1655937  ahhh yes another taxes less government story w...   \n",
       "1465266                                     perfect regard   \n",
       "\n",
       "                                        cleaned_porter_str  \\\n",
       "111477   for small mind gop polit import welfar countri...   \n",
       "826004   what bunch idiot religi right these fool belie...   \n",
       "576164   brinksmanhip might okay high school student co...   \n",
       "1655937  ahhh ye anoth tax less govern stori wow i neve...   \n",
       "1465266                                     perfect regard   \n",
       "\n",
       "                                     cleaned_lancaster_str  \n",
       "111477   for smal mind gop polit import welf country sh...  \n",
       "826004   what bunch idiot religy right thes fool believ...  \n",
       "576164   brinksmanhip might okay high school stud counc...  \n",
       "1655937  ahhh ye anoth tax less govern story wow i nev ...  \n",
       "1465266                                     perfect regard  \n",
       "\n",
       "[5 rows x 50 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['prepared_50', 'cleaned_w_stopwords_str'], 0.942)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_factors, best_metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Overall': {'Accuracy': 0.822,\n",
       "  'Precision': 0.781,\n",
       "  'Recall': 0.89,\n",
       "  'F1': 0.832,\n",
       "  'ROC_AUC': 0.823},\n",
       " 'Target': {'Accuracy': 0.89, 'Precision': 1.0, 'Recall': 0.89, 'F1': 0.942},\n",
       " 'Non-Target': {'Accuracy': 0.756,\n",
       "  'Precision': 1.0,\n",
       "  'Recall': 0.756,\n",
       "  'F1': 0.861}}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}