stepping_stones/Initial_Models_Exploration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initial Models Exploration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Attempts were first made to see how Naive Bayes, SVM, and Logistic Regression models would perform on the data set.  Data was stored in an AWS S3 bucket.  From the bucket, it was loaded, cleaned and features were generated off of the cleaned data.  Both text and numerical features were generated based off of data exploration.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/ec2-user/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer \n",
    "from nltk.stem import LancasterStemmer \n",
    "\n",
    "nltk.download(\"stopwords\")\n",
    "\n",
    "import string\n",
    "import re\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import sklearn\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "from sklearn import svm\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from scipy import sparse\n",
    "import datetime\n",
    "\n",
    "import s3fs\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "import io"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and split data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read in test.csv and train.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv(\"s3://advancedml-koch-mathur-hinkson/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"s3://advancedml-koch-mathur-hinkson/train.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a new column called \"toxicity_category\" in the train data frame categorizing comments as toxic (\"1\") or non-toxic (\"0\")."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['toxicity_category'] = train.target.apply(lambda x: 1 if x > 0.5 else 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split train.csv into training (80%) and validation sets (20%)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "msk = np.random.rand(len(train)) < 0.8\n",
    "train_set = train[msk]\n",
    "validation_set = train[~msk]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    1359251\n",
      "1      84921\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(train_set.toxicity_category.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    339185\n",
      "1     21517\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(validation_set.toxicity_category.value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create small sample (\"train_sample1\") from the train_set on which to run models.  Ensure that samples are iid by replacing after each draw."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sample1 = train_set.sample(frac=0.05, replace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    67936\n",
      "1     4273\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(train_sample1.toxicity_category.value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls = LancasterStemmer()\n",
    "ps = PorterStemmer() \n",
    "\n",
    "sw = set(stopwords.words('english'))\n",
    "sw.add('')\n",
    "\n",
    "def clean_text(text, stemming=None, remove_sw = True):\n",
    "    '''\n",
    "    This auxiliary function cleans text.\n",
    "    \n",
    "    Methods used for cleaning are: \n",
    "        (1) transform string of text to list of words,\n",
    "        (2) cleaned (lowercase, remove punctuation) and remove stop words,\n",
    "        (3) Porter stemming of cleaned (lowercase, remove punctuation) text, \n",
    "        (4) Lancaster stemming of cleaned (lowercase, remove punctuation), \n",
    "        (5) cleaned (lowercase, remove punctuation) without removing stop words.\n",
    "    \n",
    "    Inputs:\n",
    "        text (string) - A string of text.\n",
    "        stemming (parameter) - either Porter or Lancaster stemming method\n",
    "        remove_sw (boolean) - True/False remove stop words\n",
    "    \n",
    "    Outputs:\n",
    "        Cleaned text per the input parameters.\n",
    "    '''\n",
    "\n",
    "    t = text.replace(\"-\", \" \").split(\" \")\n",
    "    \n",
    "    t = [w.lower() for w in t]\n",
    "    \n",
    "    if remove_sw == True:\n",
    "        t = [w for w in t if w not in sw]\n",
    "    \n",
    "    if stemming == None:\n",
    "        pass;\n",
    "    elif stemming == \"Porter\":\n",
    "        t = [ps.stem(w) for w in t]\n",
    "    elif stemming == \"Lancaster\":\n",
    "        t = [ls.stem(w) for w in t]\n",
    "    else:\n",
    "        print(\"Please enter a valid stemming type\")\n",
    "        \n",
    "    t = [w.strip(string.punctuation) for w in t]\n",
    "\n",
    "    return ' '.join(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_text_cleaning_cols(df):\n",
    "    '''\n",
    "    This function generates features and adds them to the data frame.\n",
    "    \n",
    "    Input:\n",
    "        Data frame with raw text strings.\n",
    "        \n",
    "    Output:\n",
    "        Data frame with added columns:\n",
    "            (1) 'split' - (list) Transforms the string of text into a list of words\n",
    "            (2) 'cleaned_w_stopwords' - (string) A string of text where words have been lowercased, \n",
    "                                        punctuation is removed, and stop words are removed\n",
    "            (3) 'cleaned_no_stem' - (string) A string of text where words have been lowercased, and \n",
    "                                        punctuation is removed (stop words remain in text).\n",
    "                                        \n",
    "            \n",
    "            (4) 'cleaned_porter' - (string) A string of text where words have been stemmed using the \n",
    "                                        Porter method on cleaned (lowercase, remove punctuation) text. \n",
    "            (5) 'cleaned_lancaster' - (string) A string of text where words have been stemmed using the\n",
    "                                        Lancaster method on cleaned (lowercase, remove punctuation) text.\n",
    "            (6) 'perc_upper' - (float) Percent of uppercase letters in the string of text.\n",
    "            (7) 'num_exclam' - (integer) Number of times an exclamation point appears in text.\n",
    "            (8) 'num_words' - (integer) Number of words in text.\n",
    "            \n",
    "    '''\n",
    "    print(datetime.datetime.now())\n",
    "    \n",
    "    df['split'] = df[\"comment_text\"].apply(lambda x: x.split(\" \"))\n",
    "    df['cleaned_w_stopwords'] = df[\"comment_text\"].apply(clean_text,args=(None,False),)\n",
    "\n",
    "    print(datetime.datetime.now())\n",
    "    df['cleaned_no_stem'] = df[\"comment_text\"].apply(clean_text,)\n",
    "    df['cleaned_porter'] = df[\"comment_text\"].apply(clean_text,args=(\"Porter\",),)\n",
    "    df['cleaned_lancaster'] = df[\"comment_text\"].apply(clean_text,args=(\"Lancaster\",),)\n",
    "\n",
    "    print(datetime.datetime.now())\n",
    "\n",
    "    df['perc_upper'] = df[\"comment_text\"].apply(lambda x: round((len(re.findall(r'[A-Z]',x)) / len(x)), 3))\n",
    "\n",
    "    df['num_exclam'] = df[\"comment_text\"].apply(lambda x:(len(re.findall(r'!',x))))\n",
    "    \n",
    "    df['num_words'] = df[\"split\"].apply(lambda x: len(x))\n",
    "    print(\"DONE\")\n",
    "        \n",
    "\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-05-27 20:40:36.194393\n",
      "2019-05-27 20:40:38.012761\n",
      "2019-05-27 20:41:36.698465\n",
      "DONE\n"
     ]
    }
   ],
   "source": [
    "add_text_cleaning_cols(train_sample1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'target', 'comment_text', 'severe_toxicity', 'obscene',\n",
       "       'identity_attack', 'insult', 'threat', 'asian', 'atheist', 'bisexual',\n",
       "       'black', 'buddhist', 'christian', 'female', 'heterosexual', 'hindu',\n",
       "       'homosexual_gay_or_lesbian', 'intellectual_or_learning_disability',\n",
       "       'jewish', 'latino', 'male', 'muslim', 'other_disability',\n",
       "       'other_gender', 'other_race_or_ethnicity', 'other_religion',\n",
       "       'other_sexual_orientation', 'physical_disability',\n",
       "       'psychiatric_or_mental_illness', 'transgender', 'white', 'created_date',\n",
       "       'publication_id', 'parent_id', 'article_id', 'rating', 'funny', 'wow',\n",
       "       'sad', 'likes', 'disagree', 'sexual_explicit',\n",
       "       'identity_annotator_count', 'toxicity_annotator_count',\n",
       "       'toxicity_category', 'split', 'cleaned_w_stopwords', 'cleaned_no_stem',\n",
       "       'cleaned_porter', 'cleaned_lancaster', 'perc_upper', 'num_exclam',\n",
       "       'num_words'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sample1.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(72214, 57)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sample1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>severe_toxicity</th>\n",
       "      <th>obscene</th>\n",
       "      <th>identity_attack</th>\n",
       "      <th>insult</th>\n",
       "      <th>threat</th>\n",
       "      <th>asian</th>\n",
       "      <th>atheist</th>\n",
       "      <th>...</th>\n",
       "      <th>split</th>\n",
       "      <th>cleaned_w_stopwords</th>\n",
       "      <th>cleaned_no_stem</th>\n",
       "      <th>cleaned_porter</th>\n",
       "      <th>cleaned_lancaster</th>\n",
       "      <th>perc_upper</th>\n",
       "      <th>num_exclam</th>\n",
       "      <th>num_words</th>\n",
       "      <th>perc_stopwords</th>\n",
       "      <th>num_upper_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>375600</th>\n",
       "      <td>702857</td>\n",
       "      <td>0.0</td>\n",
       "      <td>You completely misstate how the  Bulletin of t...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>[You, completely, misstate, how, the, , Bullet...</td>\n",
       "      <td>you completely misstate how the  bulletin of t...</td>\n",
       "      <td>completely misstate bulletin atomic scientists...</td>\n",
       "      <td>complet misstat bulletin atom scientist move d...</td>\n",
       "      <td>complet misst bulletin atom sci mov doomsday c...</td>\n",
       "      <td>0.030</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>-2.690</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1566597</th>\n",
       "      <td>6038615</td>\n",
       "      <td>0.3</td>\n",
       "      <td>The good thing about boondoggles is that they ...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>[The, good, thing, about, boondoggles, is, tha...</td>\n",
       "      <td>the good thing about boondoggles is that they ...</td>\n",
       "      <td>good thing boondoggles usually collapse weight...</td>\n",
       "      <td>good thing boondoggl usual collaps weight corr...</td>\n",
       "      <td>good thing boondoggl us collaps weight corruption</td>\n",
       "      <td>0.010</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>-2.353</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275683</th>\n",
       "      <td>579993</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Throwin your votes away eh</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>[Throwin, your, votes, away, eh]</td>\n",
       "      <td>throwin your votes away eh</td>\n",
       "      <td>throwin votes away eh</td>\n",
       "      <td>throwin vote away eh</td>\n",
       "      <td>throwin vot away eh</td>\n",
       "      <td>0.038</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>-3.200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89607</th>\n",
       "      <td>352281</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Mr. Sayre trunk line fibers typically carry 40...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>[Mr., Sayre, trunk, line, fibers, typically, c...</td>\n",
       "      <td>mr sayre trunk line fibers typically carry 40 ...</td>\n",
       "      <td>mr sayre trunk line fibers typically carry 40 ...</td>\n",
       "      <td>mr sayr trunk line fiber typic carri 40 separ ...</td>\n",
       "      <td>mr sayr trunk lin fib typ carry 40 sep 10 gb/s...</td>\n",
       "      <td>0.025</td>\n",
       "      <td>0</td>\n",
       "      <td>183</td>\n",
       "      <td>-2.902</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1347516</th>\n",
       "      <td>5762238</td>\n",
       "      <td>0.0</td>\n",
       "      <td>MSNBC and CNN often mention facts  ..............</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>[MSNBC, and, CNN, often, mention, facts, , ......</td>\n",
       "      <td>msnbc and cnn often mention facts   \\n althoug...</td>\n",
       "      <td>msnbc cnn often mention facts  \\n although sto...</td>\n",
       "      <td>msnbc cnn often mention fact  \\n although stop...</td>\n",
       "      <td>msnbc cnn oft ment fact  \\n although stop watc...</td>\n",
       "      <td>0.085</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>-3.833</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 57 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  target                                       comment_text  \\\n",
       "375600    702857     0.0  You completely misstate how the  Bulletin of t...   \n",
       "1566597  6038615     0.3  The good thing about boondoggles is that they ...   \n",
       "275683    579993     0.0                         Throwin your votes away eh   \n",
       "89607     352281     0.0  Mr. Sayre trunk line fibers typically carry 40...   \n",
       "1347516  5762238     0.0  MSNBC and CNN often mention facts  ..............   \n",
       "\n",
       "         severe_toxicity  obscene  identity_attack  insult  threat  asian  \\\n",
       "375600               0.0      0.0              0.0     0.0     0.0    NaN   \n",
       "1566597              0.0      0.0              0.0     0.3     0.0    NaN   \n",
       "275683               0.0      0.0              0.0     0.0     0.0    NaN   \n",
       "89607                0.0      0.0              0.0     0.0     0.0    NaN   \n",
       "1347516              0.0      0.0              0.0     0.0     0.0    NaN   \n",
       "\n",
       "         atheist  ...                                              split  \\\n",
       "375600       NaN  ...  [You, completely, misstate, how, the, , Bullet...   \n",
       "1566597      NaN  ...  [The, good, thing, about, boondoggles, is, tha...   \n",
       "275683       NaN  ...                   [Throwin, your, votes, away, eh]   \n",
       "89607        NaN  ...  [Mr., Sayre, trunk, line, fibers, typically, c...   \n",
       "1347516      NaN  ...  [MSNBC, and, CNN, often, mention, facts, , ......   \n",
       "\n",
       "                                       cleaned_w_stopwords  \\\n",
       "375600   you completely misstate how the  bulletin of t...   \n",
       "1566597  the good thing about boondoggles is that they ...   \n",
       "275683                          throwin your votes away eh   \n",
       "89607    mr sayre trunk line fibers typically carry 40 ...   \n",
       "1347516  msnbc and cnn often mention facts   \\n althoug...   \n",
       "\n",
       "                                           cleaned_no_stem  \\\n",
       "375600   completely misstate bulletin atomic scientists...   \n",
       "1566597  good thing boondoggles usually collapse weight...   \n",
       "275683                               throwin votes away eh   \n",
       "89607    mr sayre trunk line fibers typically carry 40 ...   \n",
       "1347516  msnbc cnn often mention facts  \\n although sto...   \n",
       "\n",
       "                                            cleaned_porter  \\\n",
       "375600   complet misstat bulletin atom scientist move d...   \n",
       "1566597  good thing boondoggl usual collaps weight corr...   \n",
       "275683                                throwin vote away eh   \n",
       "89607    mr sayr trunk line fiber typic carri 40 separ ...   \n",
       "1347516  msnbc cnn often mention fact  \\n although stop...   \n",
       "\n",
       "                                         cleaned_lancaster  perc_upper  \\\n",
       "375600   complet misst bulletin atom sci mov doomsday c...       0.030   \n",
       "1566597  good thing boondoggl us collaps weight corruption       0.010   \n",
       "275683                                 throwin vot away eh       0.038   \n",
       "89607    mr sayr trunk lin fib typ carry 40 sep 10 gb/s...       0.025   \n",
       "1347516  msnbc cnn oft ment fact  \\n although stop watc...       0.085   \n",
       "\n",
       "         num_exclam  num_words  perc_stopwords  num_upper_words  \n",
       "375600            0         29          -2.690                0  \n",
       "1566597           0         17          -2.353                0  \n",
       "275683            0          5          -3.200                0  \n",
       "89607             0        183          -2.902                3  \n",
       "1347516           0         24          -3.833                3  \n",
       "\n",
       "[5 rows x 57 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sample1.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Due to memory issues, we needed to use a smaller training set.  We used a random iid sample of half of the train_sample1 frame to train NB model:"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "subset_train_sample1 = train_sample1.sample(frac=0.5, replace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "toxic = train_sample1[train_sample1.toxicity_category == 1]\n",
    "nontoxic = train_sample1[train_sample1.toxicity_category == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((72209, 54), (4273, 54), (67936, 54))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sample1.shape, toxic.shape, nontoxic.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reshaping the dataset to be include an equal number of toxic and nontoxic samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    8546\n",
      "0    8546\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "prepared_df = toxic.append(toxic).append(nontoxic.sample(len(toxic)*2))\n",
    "prepared_df = prepared_df.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "print(prepared_df.toxicity_category.value_counts())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Because we are unable to train an NB model on categorical (text) and continuous (numerical) data at the same time, our action plan changed to running two independent models for each type of data and then running a thrid NB model on the resulting predict_proba from the other two trained models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_model(model_df, train_perc=.80, addtl_feats =[''], model_type = \"Multi\", \n",
    "               should_print=False, see_inside=False, comments=\"comment_text\",\n",
    "             target='toxicity_category'):\n",
    "    '''\n",
    "    This function runs a single machine learning model as per the specified parameters.\n",
    "    \n",
    "    Input(s):\n",
    "        model_df: source data frame\n",
    "        train_perc: percentage that should be used for training set\n",
    "        addtl_feats: (list) list of non text columns to include\n",
    "        model_type: which machine learning model to use\n",
    "        see_inside: returns the intermediate tokenized and vectorized arrays\n",
    "        comments: source column for text data\n",
    "        target: source column for y values\n",
    "        \n",
    "    Output(s):\n",
    "    \n",
    "    '''\n",
    "    \n",
    "    train_start = 0\n",
    "    train_end = round(model_df.shape[0]*train_perc) \n",
    "\n",
    "    test_start = train_end\n",
    "    test_end = model_df.shape[0]\n",
    "    \n",
    "    X_all = model_df[comments].values\n",
    "    y_all = model_df[target].values\n",
    "    \n",
    "    # tokenizing text\n",
    "#     count_vect = CountVectorizer()\n",
    "#     X_all_counts = count_vect.fit_transform(X_all.astype('U'))\n",
    "    #print(X_all_counts.shape)\n",
    "\n",
    "    # calculating frequencies\n",
    "    tfidf_vectorizer = TfidfVectorizer(use_idf=True)\n",
    "    fitted_vectorizer=tfidf_vectorizer.fit(model_df[comments])\n",
    "    X_all_tfidf =  fitted_vectorizer.transform(model_df[comments])\n",
    "\n",
    "\n",
    "    print(X_all_tfidf.shape)\n",
    "    \n",
    "    if addtl_feats != ['']: # combine non-text and text features if necessary\n",
    "        print(\"here\")\n",
    "#         others_all = model_df[addtl_feats].values.reshape(-1,1)\n",
    "\n",
    "        others_all = model_df[addtl_feats].values.reshape(-1,len(addtl_feats))\n",
    "        #print(others_all)\n",
    "        newfeatures_all = sparse.hstack((X_all_tfidf, others_all.astype(float))).tocsr()\n",
    "    else:\n",
    "        newfeatures_all = X_all_tfidf\n",
    "    \n",
    "    \n",
    "    X_train = newfeatures_all[train_start:train_end]\n",
    "    y_train = model_df[train_start:train_end][target].values\n",
    "    y_train=y_train.astype('int')\n",
    "    \n",
    "\n",
    "    X_test = newfeatures_all[test_start:test_end]\n",
    "    y_test = model_df[test_start:test_end][target].values\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    if model_type == 'Multi':\n",
    "        clf = MultinomialNB().fit(X_train, y_train)\n",
    "    if model_type == \"Gauss\":\n",
    "        clf = GaussianNB().fit(X_train, y_train) \n",
    "    if model_type == \"SVM\":\n",
    "        clf = svm.SVC(kernel='linear', probability=True, random_state=1008).fit(X_train, y_train) \n",
    "    if model_type == \"LR\":\n",
    "        clf = LogisticRegression(penalty=\"l1\",C=1e5).fit(X_train, y_train)\n",
    "        \n",
    "    preds_for_train = clf.predict(X_train)\n",
    "    \n",
    "    \n",
    "   \n",
    "    predicted = clf.predict(X_test)\n",
    "    accuracy = np.mean(predicted == y_test)\n",
    "    \n",
    "    output = model_df[test_start:test_end]\n",
    "    output['predicted'] = predicted\n",
    "    output['y_test'] = y_test\n",
    "    output['accuracy'] = output.predicted == output.y_test\n",
    "    \n",
    "\n",
    "#     y_scores_sorted, y_true_sorted = joint_sort_descending(np.array(y_scores), np.array(y_true))\n",
    "#     precision = precision_score(y_true_sorted, preds)\n",
    "\n",
    "\n",
    "    if should_print == True:\n",
    "\n",
    "        print(\"The accuracy on the test set is {}%.\".format(round(accuracy*100,2)))    \n",
    "    \n",
    "    if see_inside == True:\n",
    "        return clf, accuracy, X_all_counts, X_all_tfidf\n",
    "    else:\n",
    "        return clf, accuracy, preds_for_train, predicted, output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(17092, 25851)\n",
      "The unique values predicted in the training set include :[0 1]\n",
      "The unique values predicted in the test set include :[0 1]\n"
     ]
    }
   ],
   "source": [
    "clf1, accuracy, preds_for_train, predicted , output = run_model(prepared_df, comments = \"cleaned_lancaster\", should_print=False)\n",
    "\n",
    "print(\"The unique values predicted in the training set include :\" + str(np.unique(preds_for_train)))\n",
    "print(\"The unique values predicted in the test set include :\" + str(np.unique(predicted)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>severe_toxicity</th>\n",
       "      <th>obscene</th>\n",
       "      <th>identity_attack</th>\n",
       "      <th>insult</th>\n",
       "      <th>threat</th>\n",
       "      <th>asian</th>\n",
       "      <th>atheist</th>\n",
       "      <th>...</th>\n",
       "      <th>cleaned_w_stopwords</th>\n",
       "      <th>cleaned_no_stem</th>\n",
       "      <th>cleaned_porter</th>\n",
       "      <th>cleaned_lancaster</th>\n",
       "      <th>perc_upper</th>\n",
       "      <th>num_exclam</th>\n",
       "      <th>num_words</th>\n",
       "      <th>predicted</th>\n",
       "      <th>y_test</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13674</th>\n",
       "      <td>5293834</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Preeeecisely.</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>preeeecisely</td>\n",
       "      <td>preeeecisely</td>\n",
       "      <td>preeeecisely</td>\n",
       "      <td>preeeecisely</td>\n",
       "      <td>0.077</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13675</th>\n",
       "      <td>6043087</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>tol·er·ance\\nˈtäl(ə)rəns/Submit\\nnoun\\n1.\\nthe...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>tol·er·ance\\nˈtäl(ə)rəns/submit\\nnoun\\n1.\\nthe...</td>\n",
       "      <td>tol·er·ance\\nˈtäl(ə)rəns/submit\\nnoun\\n1.\\nthe...</td>\n",
       "      <td>tol·er·ance\\nˈtäl(ə)rəns/submit\\nnoun\\n1.\\nth ...</td>\n",
       "      <td>tol·er·ance\\nˈtäl(ə)rəns/submit\\nnoun\\n1.\\nthe...</td>\n",
       "      <td>0.012</td>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13676</th>\n",
       "      <td>5415503</td>\n",
       "      <td>0.550725</td>\n",
       "      <td>Oh sure, put a black guy in the role of \"Caesa...</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0.565217</td>\n",
       "      <td>0.246377</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>oh sure put a black guy in the role of caesar ...</td>\n",
       "      <td>oh sure put black guy role caesar youd crying ...</td>\n",
       "      <td>oh sure put black guy role caesar youd cri any...</td>\n",
       "      <td>oh sure put black guy rol caesar youd cry anyo...</td>\n",
       "      <td>0.023</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13677</th>\n",
       "      <td>5936191</td>\n",
       "      <td>0.685714</td>\n",
       "      <td>Great work getting more scum off the streets.</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.114286</td>\n",
       "      <td>0.014286</td>\n",
       "      <td>0.628571</td>\n",
       "      <td>0.014286</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>great work getting more scum off the streets</td>\n",
       "      <td>great work getting scum streets</td>\n",
       "      <td>great work get scum streets</td>\n",
       "      <td>gre work get scum streets</td>\n",
       "      <td>0.022</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13678</th>\n",
       "      <td>744227</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>There is an error there..  Something U can not...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>there is an error there  something u can not s...</td>\n",
       "      <td>error there something u see smart</td>\n",
       "      <td>error there someth u see smart</td>\n",
       "      <td>er there someth u see smart</td>\n",
       "      <td>0.038</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 57 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    target                                       comment_text  \\\n",
       "13674  5293834  0.000000                                      Preeeecisely.   \n",
       "13675  6043087  0.166667  tol·er·ance\\nˈtäl(ə)rəns/Submit\\nnoun\\n1.\\nthe...   \n",
       "13676  5415503  0.550725  Oh sure, put a black guy in the role of \"Caesa...   \n",
       "13677  5936191  0.685714      Great work getting more scum off the streets.   \n",
       "13678   744227  0.800000  There is an error there..  Something U can not...   \n",
       "\n",
       "       severe_toxicity   obscene  identity_attack    insult    threat  asian  \\\n",
       "13674         0.000000  0.000000         0.000000  0.000000  0.000000    0.0   \n",
       "13675         0.000000  0.000000         0.000000  0.000000  0.000000    NaN   \n",
       "13676         0.014493  0.014493         0.565217  0.246377  0.000000    0.0   \n",
       "13677         0.071429  0.114286         0.014286  0.628571  0.014286    NaN   \n",
       "13678         0.000000  0.000000         0.000000  0.800000  0.000000    NaN   \n",
       "\n",
       "       atheist  ...                                cleaned_w_stopwords  \\\n",
       "13674      0.0  ...                                       preeeecisely   \n",
       "13675      NaN  ...  tol·er·ance\\nˈtäl(ə)rəns/submit\\nnoun\\n1.\\nthe...   \n",
       "13676      0.0  ...  oh sure put a black guy in the role of caesar ...   \n",
       "13677      NaN  ...       great work getting more scum off the streets   \n",
       "13678      NaN  ...  there is an error there  something u can not s...   \n",
       "\n",
       "                                         cleaned_no_stem  \\\n",
       "13674                                       preeeecisely   \n",
       "13675  tol·er·ance\\nˈtäl(ə)rəns/submit\\nnoun\\n1.\\nthe...   \n",
       "13676  oh sure put black guy role caesar youd crying ...   \n",
       "13677                    great work getting scum streets   \n",
       "13678                  error there something u see smart   \n",
       "\n",
       "                                          cleaned_porter  \\\n",
       "13674                                       preeeecisely   \n",
       "13675  tol·er·ance\\nˈtäl(ə)rəns/submit\\nnoun\\n1.\\nth ...   \n",
       "13676  oh sure put black guy role caesar youd cri any...   \n",
       "13677                        great work get scum streets   \n",
       "13678                     error there someth u see smart   \n",
       "\n",
       "                                       cleaned_lancaster  perc_upper  \\\n",
       "13674                                       preeeecisely       0.077   \n",
       "13675  tol·er·ance\\nˈtäl(ə)rəns/submit\\nnoun\\n1.\\nthe...       0.012   \n",
       "13676  oh sure put black guy rol caesar youd cry anyo...       0.023   \n",
       "13677                          gre work get scum streets       0.022   \n",
       "13678                        er there someth u see smart       0.038   \n",
       "\n",
       "       num_exclam  num_words  predicted  y_test  accuracy  \n",
       "13674           0          1          1       0     False  \n",
       "13675           0         67          0       0      True  \n",
       "13676           0         19          1       1      True  \n",
       "13677           0          8          1       1      True  \n",
       "13678           0         16          0       1     False  \n",
       "\n",
       "[5 rows x 57 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1714\n",
       "1    1704\n",
       "Name: y_test, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.y_test.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    1975\n",
       "0    1443\n",
       "Name: predicted, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.predicted.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8089526038619076"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8879107981220657"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets = output[output.y_test == 1]\n",
    "targets[targets.accuracy == True].shape[0] / targets.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7304550758459744"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nontargets = output[output.y_test == 0]\n",
    "nontargets[nontargets.accuracy == True].shape[0] / nontargets.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     1252\n",
       "False     462\n",
       "Name: accuracy, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output[output.y_test == 0].accuracy.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cleaned_w_stopwords', 0.6]\n",
      "(17092, 30051)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8221442152991078 , Target Accuracy: 0.8695023148148148, Nontarget Accuracy: 0.7737355811889973\n",
      "\n",
      "['cleaned_w_stopwords', 0.7]\n",
      "(17092, 30051)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8260530421216848 , Target Accuracy: 0.8893617021276595, Nontarget Accuracy: 0.7616987809673614\n",
      "\n",
      "['cleaned_w_stopwords', 0.8]\n",
      "(17092, 30051)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8224107665301346 , Target Accuracy: 0.9002347417840375, Nontarget Accuracy: 0.7450408401400234\n",
      "\n",
      "['cleaned_no_stem', 0.6]\n",
      "(17092, 30048)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8206815854907124 , Target Accuracy: 0.8778935185185185, Nontarget Accuracy: 0.7622005323868678\n",
      "\n",
      "['cleaned_no_stem', 0.7]\n",
      "(17092, 30048)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8229329173166927 , Target Accuracy: 0.8924564796905222, Nontarget Accuracy: 0.7522611089264648\n",
      "\n",
      "['cleaned_no_stem', 0.8]\n",
      "(17092, 30048)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.823288472791106 , Target Accuracy: 0.9019953051643192, Nontarget Accuracy: 0.7450408401400234\n",
      "\n",
      "['cleaned_porter', 0.6]\n",
      "(17092, 27260)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8157086441421676 , Target Accuracy: 0.8674768518518519, Nontarget Accuracy: 0.7627920733510796\n",
      "\n",
      "['cleaned_porter', 0.7]\n",
      "(17092, 27260)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8155226209048362 , Target Accuracy: 0.8796905222437137, Nontarget Accuracy: 0.750294927251278\n",
      "\n",
      "['cleaned_porter', 0.8]\n",
      "(17092, 27260)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8115857226448215 , Target Accuracy: 0.8937793427230047, Nontarget Accuracy: 0.7298716452742123\n",
      "\n",
      "['cleaned_lancaster', 0.6]\n",
      "(17092, 25851)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8174637999122422 , Target Accuracy: 0.8666087962962963, Nontarget Accuracy: 0.7672286305826679\n",
      "\n",
      "['cleaned_lancaster', 0.7]\n",
      "(17092, 25851)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8207878315132605 , Target Accuracy: 0.881237911025145, Nontarget Accuracy: 0.7593393629571372\n",
      "\n",
      "['cleaned_lancaster', 0.8]\n",
      "(17092, 25851)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8089526038619076 , Target Accuracy: 0.8879107981220657, Nontarget Accuracy: 0.7304550758459744\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_accuracy = 0\n",
    "model_factors = []\n",
    "\n",
    "for text in ['cleaned_w_stopwords', 'cleaned_no_stem', 'cleaned_porter',\n",
    "    'cleaned_lancaster']:\n",
    "    for tp in [0.6, 0.7, 0.8]:\n",
    "  \n",
    "        factors = [text, tp]\n",
    "        print(factors)\n",
    "\n",
    "        clf, accuracy, preds_for_train, predicted, output = run_model(prepared_df, train_perc = tp, comments = text, should_print=False)\n",
    "        \n",
    "        print(\"The unique values predicted for the training set include :\" + str(np.unique(preds_for_train)))\n",
    "        print(\"The unique values predicted for the test set include :\" + str(np.unique(predicted)))\n",
    "        \n",
    "        targets = output[output.y_test == 1]\n",
    "        target_accuracy = targets[targets.accuracy == True].shape[0] / targets.shape[0]\n",
    "        \n",
    "        nontargets = output[output.y_test == 0]\n",
    "        nontarget_accuracy = nontargets[nontargets.accuracy == True].shape[0] / nontargets.shape[0]\n",
    "        \n",
    "        print(\"Accuracy: {} , Target Accuracy: {}, Nontarget Accuracy: {}\".format(accuracy, target_accuracy, nontarget_accuracy))\n",
    "\n",
    "        if target_accuracy > best_accuracy:\n",
    "            model_factors = factors\n",
    "            best_accuracy = target_accuracy\n",
    "\n",
    "        print()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9019953051643192, ['cleaned_no_stem', 0.8])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_accuracy,model_factors "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(17092, 30048)\n"
     ]
    }
   ],
   "source": [
    "clf, accuracy, preds_for_train, predicted, output = run_model(prepared_df, train_perc = 0.8, comments = 'cleaned_no_stem', should_print=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = output[output.y_test == 1]\n",
    "target_accuracy = targets[targets.accuracy == True].shape[0] / targets.shape[0]\n",
    "\n",
    "nontargets = output[output.y_test == 0]\n",
    "nontarget_accuracy = nontargets[nontargets.accuracy == True].shape[0] / nontargets.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9019953051643192, 0.7450408401400234)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_accuracy, nontarget_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cleaned_w_stopwords', 0.6]\n",
      "(17328, 30033)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.874909825422017 , Target Accuracy: 0.8627733026467204\n",
      "\n",
      "['cleaned_w_stopwords', 0.7]\n",
      "(17328, 30033)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8855328972681801 , Target Accuracy: 0.8772269558481797\n",
      "\n",
      "['cleaned_w_stopwords', 0.8]\n",
      "(17328, 30033)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8993075591459896 , Target Accuracy: 0.906628242074928\n",
      "\n",
      "['cleaned_no_stem', 0.6]\n",
      "(17328, 30032)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.875486942721108 , Target Accuracy: 0.8593210586881473\n",
      "\n",
      "['cleaned_no_stem', 0.7]\n",
      "(17328, 30032)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8897652943439784 , Target Accuracy: 0.8822618125484121\n",
      "\n",
      "['cleaned_no_stem', 0.8]\n",
      "(17328, 30032)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.9013271783035199 , Target Accuracy: 0.899135446685879\n",
      "\n",
      "['cleaned_porter', 0.6]\n",
      "(17328, 27310)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8642331553888328 , Target Accuracy: 0.8483889528193326\n",
      "\n",
      "['cleaned_porter', 0.7]\n",
      "(17328, 27310)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8772604848018468 , Target Accuracy: 0.8690937257939582\n",
      "\n",
      "['cleaned_porter', 0.8]\n",
      "(17328, 27310)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8860357761107905 , Target Accuracy: 0.8853025936599423\n",
      "\n",
      "['cleaned_lancaster', 0.6]\n",
      "(17328, 25803)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8692829317558793 , Target Accuracy: 0.8555811277330264\n",
      "\n",
      "['cleaned_lancaster', 0.7]\n",
      "(17328, 25803)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8768757214313198 , Target Accuracy: 0.8737412858249419\n",
      "\n",
      "['cleaned_lancaster', 0.8]\n",
      "(17328, 25803)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.881130986728217 , Target Accuracy: 0.8829971181556195\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_accuracy_svm = 0\n",
    "model_factors_svm = []\n",
    "\n",
    "for text in ['cleaned_w_stopwords', 'cleaned_no_stem', 'cleaned_porter',\n",
    "    'cleaned_lancaster']:\n",
    "    for tp in [0.6, 0.7, 0.8]:\n",
    "  \n",
    "        factors = [text, tp]\n",
    "        print(factors)\n",
    "\n",
    "        clf, accuracy, preds_for_train, predicted, output = run_model(prepared_df, model_type=\"SVM\", train_perc = tp, comments = text, should_print=False)\n",
    "        \n",
    "        print(\"The unique values predicted for the training set include :\" + str(np.unique(preds_for_train)))\n",
    "        print(\"The unique values predicted for the test set include :\" + str(np.unique(predicted)))\n",
    "        \n",
    "        targets = output[output.y_test == 1]\n",
    "        target_accuracy = targets[targets.accuracy == True].shape[0] / targets.shape[0]\n",
    "        \n",
    "        print(\"Accuracy: {} , Target Accuracy: {}\".format(accuracy, target_accuracy))\n",
    "\n",
    "        if target_accuracy > best_accuracy:\n",
    "            model_factors_svm = factors\n",
    "            best_accuracy_svm = target_accuracy\n",
    "\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, [])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_accuracy_svm, model_factors_svm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cleaned_w_stopwords', 0.6]\n",
      "(17328, 30033)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8482181503390565 , Target Accuracy: 0.8918296892980437\n",
      "\n",
      "['cleaned_w_stopwords', 0.7]\n",
      "(17328, 30033)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.856483262793382 , Target Accuracy: 0.9047250193648335\n",
      "\n",
      "['cleaned_w_stopwords', 0.8]\n",
      "(17328, 30033)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8742065781881131 , Target Accuracy: 0.9342939481268011\n",
      "\n",
      "['cleaned_no_stem', 0.6]\n",
      "(17328, 30032)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8492281056124658 , Target Accuracy: 0.8941311852704258\n",
      "\n",
      "['cleaned_no_stem', 0.7]\n",
      "(17328, 30032)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8649480569449788 , Target Accuracy: 0.9109217660728117\n",
      "\n",
      "['cleaned_no_stem', 0.8]\n",
      "(17328, 30032)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8814195037507213 , Target Accuracy: 0.9435158501440922\n",
      "\n",
      "['cleaned_porter', 0.6]\n",
      "(17328, 27310)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8434569326215553 , Target Accuracy: 0.8806098964326813\n",
      "\n",
      "['cleaned_porter', 0.7]\n",
      "(17328, 27310)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8605232781839169 , Target Accuracy: 0.9109217660728117\n",
      "\n",
      "['cleaned_porter', 0.8]\n",
      "(17328, 27310)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.877668782458165 , Target Accuracy: 0.9377521613832853\n",
      "\n",
      "['cleaned_lancaster', 0.6]\n",
      "(17328, 25803)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8542778819795124 , Target Accuracy: 0.892692750287687\n",
      "\n",
      "['cleaned_lancaster', 0.7]\n",
      "(17328, 25803)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8522508657175837 , Target Accuracy: 0.9012393493415957\n",
      "\n",
      "['cleaned_lancaster', 0.8]\n",
      "(17328, 25803)\n",
      "The unique values predicted for the training set include :[0 1]\n",
      "The unique values predicted for the test set include :[0 1]\n",
      "Accuracy: 0.8684362377380266 , Target Accuracy: 0.9331412103746398\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_accuracy_log = 0\n",
    "model_factors_log = []\n",
    "\n",
    "for text in ['cleaned_w_stopwords', 'cleaned_no_stem', 'cleaned_porter',\n",
    "    'cleaned_lancaster']:\n",
    "    for tp in [0.6, 0.7, 0.8]:\n",
    "  \n",
    "        factors = [text, tp]\n",
    "        print(factors)\n",
    "\n",
    "        clf, accuracy, preds_for_train, predicted, output = run_model(prepared_df, model_type=\"LR\", train_perc = tp, comments = text, should_print=False)\n",
    "        \n",
    "        print(\"The unique values predicted for the training set include :\" + str(np.unique(preds_for_train)))\n",
    "        print(\"The unique values predicted for the test set include :\" + str(np.unique(predicted)))\n",
    "        \n",
    "        targets = output[output.y_test == 1]\n",
    "        target_accuracy = targets[targets.accuracy == True].shape[0] / targets.shape[0]\n",
    "        \n",
    "        print(\"Accuracy: {} , Target Accuracy: {}\".format(accuracy, target_accuracy))\n",
    "\n",
    "        if target_accuracy > best_accuracy:\n",
    "            model_factors_log = factors\n",
    "            best_accuracy_log = target_accuracy\n",
    "\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}