stepping_stones/SVM_iter3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVM -- Iteration 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this iteration of SVM, a preprocessed data set of 100K rows was split into a Train (80%) and Hold out (20%) set.  Due to a very low number of toxic comments (somewhere around 6% of the data is labeled as toxic), it was learned in earlier iterations that the model could not pick up on such a low number of toxic comments.  Therefore the data set was reshaped and weigted to 50% toxic comments and 50% nontoxic comments.  From the results shown on the tested hold out set, recall is most important because this measures if we are identifying the comments of interest.  While recall has improved, precision has decreased because the training data no longer reflects what is observed in the raw data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import string\n",
    "import re\n",
    "import string\n",
    "import numpy as np\n",
    "import datetime\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer \n",
    "from nltk.stem import LancasterStemmer \n",
    "\n",
    "import sklearn\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB,GaussianNB\n",
    "from sklearn import svm\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle_functions as pf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import model_functions as mf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and shuffle data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read in test.csv and train.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pf.read_pickle(bucket_name='advancedml-koch-mathur-hinkson', filename='sub_train_df7_preprocessed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'target', 'comment_text', 'severe_toxicity', 'obscene',\n",
       "       'identity_attack', 'insult', 'threat', 'asian', 'atheist', 'bisexual',\n",
       "       'black', 'buddhist', 'christian', 'female', 'heterosexual', 'hindu',\n",
       "       'homosexual_gay_or_lesbian', 'intellectual_or_learning_disability',\n",
       "       'jewish', 'latino', 'male', 'muslim', 'other_disability',\n",
       "       'other_gender', 'other_race_or_ethnicity', 'other_religion',\n",
       "       'other_sexual_orientation', 'physical_disability',\n",
       "       'psychiatric_or_mental_illness', 'transgender', 'white', 'created_date',\n",
       "       'publication_id', 'parent_id', 'article_id', 'rating', 'funny', 'wow',\n",
       "       'sad', 'likes', 'disagree', 'sexual_explicit',\n",
       "       'identity_annotator_count', 'toxicity_annotator_count', 'split',\n",
       "       'cleaned_w_stopwords_str', 'cleaned_w_stopwords', 'cleaned_no_stem_str',\n",
       "       'cleaned_no_stem', 'cleaned_porter_str', 'cleaned_porter',\n",
       "       'cleaned_lancaster_str', 'cleaned_lancaster', 'bigrams_unstemmed',\n",
       "       'perc_upper', 'num_exclam', 'num_words', 'perc_stopwords',\n",
       "       'num_upper_words'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_cols = ['split', 'cleaned_w_stopwords', 'cleaned_no_stem', 'cleaned_porter', 'cleaned_lancaster', 'bigrams_unstemmed',\n",
    "       'perc_upper', 'num_exclam', 'num_words', 'perc_stopwords',\n",
    "       'num_upper_words']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.drop(drop_cols, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 49)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'target', 'comment_text', 'severe_toxicity', 'obscene',\n",
       "       'identity_attack', 'insult', 'threat', 'asian', 'atheist', 'bisexual',\n",
       "       'black', 'buddhist', 'christian', 'female', 'heterosexual', 'hindu',\n",
       "       'homosexual_gay_or_lesbian', 'intellectual_or_learning_disability',\n",
       "       'jewish', 'latino', 'male', 'muslim', 'other_disability',\n",
       "       'other_gender', 'other_race_or_ethnicity', 'other_religion',\n",
       "       'other_sexual_orientation', 'physical_disability',\n",
       "       'psychiatric_or_mental_illness', 'transgender', 'white', 'created_date',\n",
       "       'publication_id', 'parent_id', 'article_id', 'rating', 'funny', 'wow',\n",
       "       'sad', 'likes', 'disagree', 'sexual_explicit',\n",
       "       'identity_annotator_count', 'toxicity_annotator_count',\n",
       "       'cleaned_w_stopwords_str', 'cleaned_no_stem_str', 'cleaned_porter_str',\n",
       "       'cleaned_lancaster_str'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a new column called \"toxicity_category\" in the train data frame categorizing comments as toxic (\"1\") or non-toxic (\"0\")."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['toxicity_category'] = train.target.apply(lambda x: 1 if x > 0.5 else 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split train.csv into training (80%) and hold out sets (20%)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/24147278/how-do-i-create-test-and-train-samples-from-one-dataframe-with-pandas\n",
    "msk = np.random.rand(len(train)) < 0.8\n",
    "train_set = train[msk]\n",
    "hold_out_set = train[~msk]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    75710\n",
      "1     4115\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(train_set.toxicity_category.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    18803\n",
      "1     1013\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(hold_out_set.toxicity_category.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    75710\n",
      "1     4115\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(train_set.toxicity_category.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "toxic = train_set[train_set.toxicity_category == 1]\n",
    "nontoxic = train_set[train_set.toxicity_category == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((79825, 50), (4115, 50), (75710, 50))"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.shape, toxic.shape, nontoxic.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reshaping the dataset to be include an equal number of toxic and nontoxic samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "quarter = len(toxic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_df = train_set.sample(quarter*4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Double the numbe of toxic comments and then sample and equal number of non-toxic comments, to make a 50%-50% data set of toxic and nontoxic comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    8230\n",
      "0    8230\n",
      "Name: toxicity_category, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "prepared_50 = toxic.append(toxic).append(nontoxic.sample(len(toxic)*2))\n",
    "prepared_50 = prepared_50.sample(frac=1).reset_index(drop=True)\n",
    "print(prepared_50.toxicity_category.value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fitting model now\n"
     ]
    }
   ],
   "source": [
    "classifier, output, fitted_vectorizer = mf.run_model(model_df=prepared_50, \n",
    "                                                     model_type=\"SVM\", \n",
    "                                                     comments = \"cleaned_no_stem_str\", \n",
    "                                                     train_perc=0.95, \n",
    "                                                     target=\"toxicity_category\", \n",
    "                                                     see_inside=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall Accuracy: 0.8991494532199271\n",
      "Overall Precision: 0.8983050847457628\n",
      "Overall Recall: 0.9004854368932039\n",
      "Overall F1 Score: 0.8993939393939395\n",
      "ROC_AUC: 0.899\n",
      "\n",
      "Target Accuracy: 0.9004854368932039\n",
      "Target Precision: 1.0\n",
      "Target Recall: 0.9004854368932039\n",
      "Target F1 Score: 0.9476372924648786\n",
      "\n",
      "Non-Target Accuracy: 0.8978102189781022\n",
      "Non-Target Precision: 1.0\n",
      "Non-Target Recall: 0.8978102189781022\n",
      "Non-Target F1 Score: 0.9461538461538462\n",
      "\n",
      "Strong Identity Accuracy: 0.8947368421052632\n",
      "Strong Identity Precision: 1.0\n",
      "Strong Identity Recall: 0.8947368421052632\n",
      "Strong Identity F1 Score: 1.0\n",
      "\n",
      "Obscenity Accuracy: 0.9354838709677419\n",
      "Obscenity Precision: 1.0\n",
      "Obscenity Recall: 0.9354838709677419\n",
      "Obscenity F1 Score: 1.0\n",
      "\n",
      "Insults Accuracy: 0.9090909090909091\n",
      "Insults Precision: 1.0\n",
      "Insults Recall: 0.9090909090909091\n",
      "Insults F1 Score: 1.0\n",
      "\n",
      "Threats Accuracy: 0.8181818181818182\n",
      "Threats Precision: 1.0\n",
      "Threats Recall: 0.8181818181818182\n",
      "Threats F1 Score: 1.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'Overall': {'Accuracy': 0.8991494532199271,\n",
       "  'Precision': 0.8983050847457628,\n",
       "  'Recall': 0.9004854368932039,\n",
       "  'F1': 0.8993939393939395,\n",
       "  'ROC_AUC': 0.899},\n",
       " 'Target': {'Accuracy': 0.9004854368932039,\n",
       "  'Precision': 1.0,\n",
       "  'Recall': 0.9004854368932039,\n",
       "  'F1': 0.9476372924648786},\n",
       " 'Non-Target': {'Accuracy': 0.8181818181818182,\n",
       "  'Precision': 1.0,\n",
       "  'Recall': 0.8181818181818182,\n",
       "  'F1': 1.0}}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mf.get_metrics(output=output, detailed=True, should_print=True, round_to=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['id', 'target', 'comment_text', 'severe_toxicity', 'obscene',\n",
      "       'identity_attack', 'insult', 'threat', 'asian', 'atheist', 'bisexual',\n",
      "       'black', 'buddhist', 'christian', 'female', 'heterosexual', 'hindu',\n",
      "       'homosexual_gay_or_lesbian', 'intellectual_or_learning_disability',\n",
      "       'jewish', 'latino', 'male', 'muslim', 'other_disability',\n",
      "       'other_gender', 'other_race_or_ethnicity', 'other_religion',\n",
      "       'other_sexual_orientation', 'physical_disability',\n",
      "       'psychiatric_or_mental_illness', 'transgender', 'white', 'created_date',\n",
      "       'publication_id', 'parent_id', 'article_id', 'rating', 'funny', 'wow',\n",
      "       'sad', 'likes', 'disagree', 'sexual_explicit',\n",
      "       'identity_annotator_count', 'toxicity_annotator_count',\n",
      "       'cleaned_w_stopwords_str', 'cleaned_no_stem_str', 'cleaned_porter_str',\n",
      "       'cleaned_lancaster_str', 'toxicity_category', 'predicted', 'y_test'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "hold_out_results = mf.run_model_test(model_df=hold_out_set, \n",
    "                                     clf=classifier, \n",
    "                                     vectorizer=fitted_vectorizer, \n",
    "                                     comments=\"cleaned_no_stem_str\", target=\"toxicity_category\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "hold_out_results.to_csv(\"holdout_results\", sep='|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    18803\n",
       "1     1013\n",
       "Name: toxicity_category, dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hold_out_results.toxicity_category.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3230452674897119"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(hold_out_results.y_test, hold_out_results.predicted, pos_label=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall Accuracy: 0.8968005651998385\n",
      "Overall Precision: 0.3230452674897119\n",
      "Overall Recall: 0.9299111549851925\n",
      "Overall F1 Score: 0.47951132603715957\n",
      "ROC_AUC: 0.912\n",
      "\n",
      "Target Accuracy: 0.9299111549851925\n",
      "Target Precision: 1.0\n",
      "Target Recall: 0.9299111549851925\n",
      "Target F1 Score: 0.963682864450128\n",
      "\n",
      "Non-Target Accuracy: 0.8950167526458543\n",
      "Non-Target Precision: 1.0\n",
      "Non-Target Recall: 0.8950167526458543\n",
      "Non-Target F1 Score: 0.9446003592276605\n",
      "\n",
      "Strong Identity Accuracy: 0.8769230769230769\n",
      "Strong Identity Precision: 0.9473684210526315\n",
      "Strong Identity Recall: 0.9152542372881356\n",
      "Strong Identity F1 Score: 1.0\n",
      "\n",
      "Obscenity Accuracy: 0.9032258064516129\n",
      "Obscenity Precision: 0.9649122807017544\n",
      "Obscenity Recall: 0.9322033898305084\n",
      "Obscenity F1 Score: 1.0\n",
      "\n",
      "Insults Accuracy: 0.9241645244215938\n",
      "Insults Precision: 0.9846153846153847\n",
      "Insults Recall: 0.9361702127659575\n",
      "Insults F1 Score: 1.0\n",
      "\n",
      "Threats Accuracy: 0.8823529411764706\n",
      "Threats Precision: 0.8823529411764706\n",
      "Threats Recall: 1.0\n",
      "Threats F1 Score: 1.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "hold_out_metrics = mf.get_metrics(output=hold_out_results, detailed=True, should_print=True, round_to=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}