Tweet Text Processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reding data ( Stage 1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Reading data\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.io\n",
    "import classifiers as clf\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Unnamed: 0                                                  5  0\n",
      "0      172348  @KMC1121 lol... im going to log off for about ...  0\n",
      "1      802252  @MissGC : Hahaha. Unfortunately not. My lesbia...  4\n",
      "2      108054  @vissy i know it does  super cute!! get an iph...  0\n",
      "3      406610  iss gonna redo her resume and apply at cineple...  0\n",
      "4     1181642    @amypalko wow, exciting. Nice hard bound copy.   4\n",
      "\n",
      "\n",
      "Comments size:  (1360000,) \t Labels size:  (1360000,)\n"
     ]
    }
   ],
   "source": [
    "current_dir = os.getcwd()\n",
    "df1 = pd.read_csv(current_dir + '/data/tweet_sentiment/train1',encoding = \"ISO-8859-1\")\n",
    "df2 = pd.read_csv(current_dir + '/data/tweet_sentiment/train2',encoding = \"ISO-8859-1\")\n",
    "frames = [df1, df2]\n",
    "df = pd.concat(frames)\n",
    "#print(df.info())\n",
    "print(df.head())\n",
    "\n",
    "# Spliting data to obtain comments and labels\n",
    "# Technically should also include summary at this point but ditched to save time\n",
    "\n",
    "#comments = df['Text']\n",
    "#score_label = df['Score']\n",
    "print('\\n\\nComments size: ', df['5'].shape, \"\\t\", \"Labels size: \", df['0'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id comment_text toxic severe_toxic obscene threat insult identity_hate\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# NOT SURE NECESSARY FOR THIS IMPLEMENTATION\n",
    "#toxic_labels = df[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing data ( Stage 2 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocessing function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "# Downloading componnents of nltk (execute just one time nltk.download())\n",
    "# nltk.download()\n",
    "\n",
    "from nltk.corpus import stopwords \n",
    "from nltk.stem import WordNetLemmatizer \n",
    "import string \n",
    "import pandas as pd \n",
    "from nltk import pos_tag \n",
    "from nltk.stem import PorterStemmer\n",
    "\n",
    "# Function to preprocess the text data\n",
    "def preprocessing(text):\n",
    "    # Removing standar punctuation (replacing with blank \"\" spaces)\n",
    "    text2 = \" \".join(\"\".join([\" \" if ch in string.punctuation else ch for ch in text]).split())\n",
    "    \n",
    "    # Tokenizing the text into words (based on white spaces to build the list)\n",
    "    tokens = [word for sent in nltk.sent_tokenize(text2) for word in nltk.word_tokenize(sent)]\n",
    "    \n",
    "    # Changing to lower case every word in the list to reduce duplicates\n",
    "    tokens = [word.lower() for word in tokens]\n",
    "    \n",
    "    # Removing english stop words from the list (stop words do not carry much weight in understanding the sentence)\n",
    "    stopwds = stopwords.words('english')\n",
    "    tokens = [token for token in tokens if token not in stopwds]\n",
    "    \n",
    "    # Removing words which length is lower than 3 (do not apport much of a meaning)\n",
    "    tokens = [word for word in tokens if len(word) >= 3]\n",
    "    \n",
    "    # Using PorterStemmer to stem suffixes in words\n",
    "    stemmer = PorterStemmer()\n",
    "    tokens = [stemmer.stem(word) for word in tokens]\n",
    "    \n",
    "    # Tagging the words\n",
    "    # “NN (noun, common, singular), NNP (noun, proper, singular), \n",
    "    # NNPS (noun, proper, plural), NNS (noun, common, plural), \n",
    "    # VB (verb, base form), VBD (verb, past tense), \n",
    "    # VBG (verb, present participle), VBN (verb, past participle), \n",
    "    # VBP (verb, present tense, not third person singular), \n",
    "    # VBZ (verb, present tense, third person singular)”\n",
    "    tagged_corpus = pos_tag(tokens)\n",
    "    Noun_tags = ['NN','NNP','NNPS','NNS']\n",
    "    Verb_tags = ['VB','VBD','VBG','VBN','VBP','VBZ']\n",
    "    \n",
    "    # Lemmatizing model\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    \n",
    "    # Validating tags and lemmatizing accordingly\n",
    "    def prat_lemmatize(token,tag):\n",
    "        # Nouns\n",
    "        if tag in Noun_tags:\n",
    "            return lemmatizer.lemmatize(token,'n')\n",
    "        # Verbs\n",
    "        elif tag in Verb_tags:\n",
    "            return lemmatizer.lemmatize(token,'v')\n",
    "        # Any other\n",
    "        else:\n",
    "            return lemmatizer.lemmatize(token,'n')\n",
    "    \n",
    "    # Reconstructing text\n",
    "    pre_proc_text =  \" \".join([prat_lemmatize(token,tag) for token,tag in tagged_corpus])             \n",
    "    # Return reconstructed text\n",
    "    return pre_proc_text\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess and storage process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#from tqdm import tqdm\n",
    "import pickle\n",
    "import sys\n",
    "\n",
    "# Increasing depth in recursion limit\n",
    "#sys.setrecursionlimit(5000)\n",
    "\n",
    "# Initialising array to storage preprocessed data\n",
    "#preprocessed_data = []\n",
    "\n",
    "\n",
    "# Pre-processing\n",
    "df['5'] = df['5'].apply(preprocessing)\n",
    "\n",
    "#i = 0\n",
    "#for line in tqdm(comments):\n",
    "#    i = i+1\n",
    "#    preprocessed_data.append(preprocessing(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Saving the preprocessed data\n",
    "pickle_out = open(\"tweet_preprocessed.pickle\",\"wb\")\n",
    "pickle.dump(df, pickle_out)\n",
    "pickle_out.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Splitting in training and test set ( Stage 3 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set size:\t 1088000 \n",
      "Test set size:\t\t 272000\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "# Importing preprocessed text data\n",
    "#pickle_in = open(\"tweet_preprocessed.pickle\",\"rb\")\n",
    "#trainData = pickle.load(pickle_in)\n",
    "\n",
    "#x_train, x_test, y_train, y_test = train_test_split(\n",
    "#    trainData['5'], trainData['0'], test_size=0.2, random_state=37)\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(\n",
    "    df['5'], df['0'], test_size=0.2, random_state=37)\n",
    "\n",
    "# Splitting into train and test set (75% train - 25% set)\n",
    "#train_size = int( round( len(preprocessed_data) * 0.75 ) )\n",
    "\n",
    "# Filling the training set\n",
    "#x_train = np.array([''.join(rec) for rec in preprocessed_data[0 : train_size]])\n",
    "#y_train = np.array([rec for rec in toxic_label[0 : train_size]])\n",
    "\n",
    "# Filling the test set\n",
    "#x_test = np.array([''.join(rec) for rec in preprocessed_data[train_size + 1 : len(preprocessed_data)]])\n",
    "#y_test = np.array([rec for rec in toxic_label[train_size + 1 : len(toxic_label)]])\n",
    "\n",
    "print( \"Training set size:\\t\", len(x_train), \"\\nTest set size:\\t\\t\", len(x_test) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(119678, 6)\n",
      "(39892, 6)\n"
     ]
    }
   ],
   "source": [
    "#ys_train = toxic_labels.as_matrix()[0:train_size];\n",
    "#ys_test = toxic_labels.as_matrix()[train_size + 1:len(toxic_label)];"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Computing TF-IDF features ( Stage 4 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-6b511eedbd8d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     11\u001b[0m                              max_features= 4000,strip_accents='unicode')\n\u001b[0;32m     12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m \u001b[0mfeatures_train\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvectorizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtodense\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     14\u001b[0m \u001b[0mfeatures_test\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvectorizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtodense\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Programming\\Anaconda3\\lib\\site-packages\\scipy\\sparse\\base.py\u001b[0m in \u001b[0;36mtodense\u001b[1;34m(self, order, out)\u001b[0m\n\u001b[0;32m    719\u001b[0m             \u001b[0;31m`\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmatrix\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mobject\u001b[0m \u001b[0mthat\u001b[0m \u001b[0mshares\u001b[0m \u001b[0mthe\u001b[0m \u001b[0msame\u001b[0m \u001b[0mmemory\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    720\u001b[0m         \"\"\"\n\u001b[1;32m--> 721\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masmatrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    722\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    723\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Programming\\Anaconda3\\lib\\site-packages\\scipy\\sparse\\compressed.py\u001b[0m in \u001b[0;36mtoarray\u001b[1;34m(self, order, out)\u001b[0m\n\u001b[0;32m    962\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    963\u001b[0m         \u001b[1;34m\"\"\"See the docstring for `spmatrix.toarray`.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 964\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtocoo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    965\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    966\u001b[0m     \u001b[1;31m##############################################################\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Programming\\Anaconda3\\lib\\site-packages\\scipy\\sparse\\coo.py\u001b[0m in \u001b[0;36mtoarray\u001b[1;34m(self, order, out)\u001b[0m\n\u001b[0;32m    250\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    251\u001b[0m         \u001b[1;34m\"\"\"See the docstring for `spmatrix.toarray`.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 252\u001b[1;33m         \u001b[0mB\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_process_toarray_args\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    253\u001b[0m         \u001b[0mfortran\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mB\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf_contiguous\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    254\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mfortran\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mB\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mc_contiguous\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Programming\\Anaconda3\\lib\\site-packages\\scipy\\sparse\\base.py\u001b[0m in \u001b[0;36m_process_toarray_args\u001b[1;34m(self, order, out)\u001b[0m\n\u001b[0;32m   1037\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1038\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1039\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1040\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1041\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__numpy_ufunc__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpos\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "# Vectorizer model \n",
    "# Ignoring terms with lower frequency than 2, range of sequences of words from 1 to 2,\n",
    "# most frequent 4000 words and normalising with l2\n",
    "#vectorizer = TfidfVectorizer(min_df=2, ngram_range=(1, 2),  stop_words='english', \n",
    "#                             max_features= 4000,strip_accents='unicode',  norm='l2')\n",
    "\n",
    "vectorizer = CountVectorizer(min_df=2, ngram_range=(1, 2),  stop_words='english', \n",
    "                             max_features= 4000,strip_accents='unicode')\n",
    "\n",
    "features_train = vectorizer.fit_transform(x_train).todense()\n",
    "features_test = vectorizer.transform(x_test).todense()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classifying ( Stage 5 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Multinomial Naive Bayes Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "mnb_clf = MultinomialNB().fit(features_train, y_train)\n",
    "\n",
    "mnb_predicted_train = mnb_clf.predict(features_train)\n",
    "mnb_predicted_test = mnb_clf.predict(features_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SVM Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC #,SVC\n",
    "\n",
    "# c = penalty parameter\n",
    "c = 1.0\n",
    "svm_clf = LinearSVC(C = c).fit(features_train, y_train)\n",
    "\n",
    "svm_predicted_train = svm_clf.predict(features_train)\n",
    "svm_predicted_test = svm_clf.predict(features_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'xgboost'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-22-5a6e4db9da03>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mxgboost\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msklearn\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mXGBClassifier\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m# Extreme Gradient Boost\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;31m# md = Max depth\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;31m# ss = Subsample ratio of training instance\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'xgboost'"
     ]
    }
   ],
   "source": [
    "# CANOT CURRENTLY RUN THIS S NOT INSTALLED\n",
    "\n",
    "#from xgboost.sklearn import XGBClassifier\n",
    "\n",
    "# Extreme Gradient Boost\n",
    "# md = Max depth\n",
    "# ss = Subsample ratio of training instance\n",
    "# cs = Subsample ratio of columns when constructing each tree\n",
    "\n",
    "#md = 1\n",
    "#ss = 0.8\n",
    "#cs = 0.8\n",
    "#clf = XGBClassifier( max_depth = md, subsample = ss,\n",
    "#                        colsample_bytree = cs).fit(features_train, y_train)\n",
    "\n",
    "#predicted_train = clf.predict(features_train)\n",
    "#predicted_test = clf.predict(features_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# C = Inverse of regularization strength\n",
    "c = 1.0\n",
    "log_clf = LogisticRegression(C = c).fit(features_train, y_train)\n",
    "\n",
    "log_predicted_train = log_clf.predict(features_train)\n",
    "log_predicted_test = log_clf.predict(features_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Voting Ensemble for Classification\n",
    "import pandas\n",
    "from sklearn import model_selection\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import ensemble\n",
    "\n",
    "# create the sub models\n",
    "estimators = []\n",
    "estimators.append(('logistic', LogisticRegression()))\n",
    "estimators.append(('cart', DecisionTreeClassifier()))\n",
    "estimators.append(('svm', SVC()))\n",
    "\n",
    "## http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.VotingClassifier.html\n",
    "# clf = ensemble.VotingClassifier(estimators)\n",
    "\n",
    "## http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html\n",
    "# clf = ensemble.GradientBoostingClassifier(n_estimators=20, random_state=7, verbose=3)\n",
    "# clf.fit(features_train, y_train)\n",
    "\n",
    "## http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html\n",
    "kfold = model_selection.KFold(n_splits=10, random_state=7)\n",
    "model = ensemble.AdaBoostClassifier(n_estimators=10, random_state=7)\n",
    "results = model_selection.cross_val_score(model, features_train, y_train, cv=kfold)\n",
    "print(results.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metrics ( Stage 6 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Naive Bayes - Train Confusion Matrix\n",
      "\n",
      " Predicted    1   2   3    4      5\n",
      "Actual                            \n",
      "1          921   0   7   14   1968\n",
      "2          145  11   6   41   1490\n",
      "3           79   0  47   75   2239\n",
      "4           27   0   6  238   4238\n",
      "5           53   0   4   41  20350\n",
      "\n",
      "Naive Bayes- Train accuracy 0.674\n",
      "\n",
      "Naive Bayes  - Train Classification Report\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          1       0.75      0.32      0.45      2910\n",
      "          2       1.00      0.01      0.01      1693\n",
      "          3       0.67      0.02      0.04      2440\n",
      "          4       0.58      0.05      0.10      4509\n",
      "          5       0.67      1.00      0.80     20448\n",
      "\n",
      "avg / total       0.68      0.67      0.57     32000\n",
      "\n",
      "------------------------------------------------------------\n",
      "\n",
      "Naive Bayes - Test Confusion Matrix\n",
      "\n",
      " Predicted    1  2  3   4     5\n",
      "Actual                        \n",
      "1          192  1  0   4   543\n",
      "2           27  1  3   9   386\n",
      "3           16  0  2  22   570\n",
      "4           12  0  3  38  1081\n",
      "5           15  0  2  10  5063\n",
      "\n",
      "Naive Bayes- Test accuracy 0.662\n",
      "\n",
      "Naive Bayes  - Test Classification Report\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          1       0.73      0.26      0.38       740\n",
      "          2       0.50      0.00      0.00       426\n",
      "          3       0.20      0.00      0.01       610\n",
      "          4       0.46      0.03      0.06      1134\n",
      "          5       0.66      0.99      0.80      5090\n",
      "\n",
      "avg / total       0.60      0.66      0.55      8000\n",
      "\n",
      "------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report,accuracy_score\n",
    "\n",
    "# Training confusion matrix\n",
    "print (\"\\nNaive Bayes - Train Confusion Matrix\\n\\n\",\n",
    "       pd.crosstab(y_train, mnb_predicted_train, rownames = [\"Actual\"], colnames = [\"Predicted\"]))\n",
    "# Training accuracy\n",
    "print (\"\\nNaive Bayes- Train accuracy\",\n",
    "       round(accuracy_score(y_train, mnb_predicted_train),3))\n",
    "# Training report\n",
    "print (\"\\nNaive Bayes  - Train Classification Report\\n\",\n",
    "       classification_report(y_train, mnb_predicted_train))\n",
    "\n",
    "print(\"------------------------------------------------------------\")\n",
    "\n",
    "# Test confusion matrix\n",
    "print (\"\\nNaive Bayes - Test Confusion Matrix\\n\\n\",\n",
    "       pd.crosstab(y_test,mnb_predicted_test,rownames = [\"Actual\"], colnames = [\"Predicted\"]))  \n",
    "# Test accuracy\n",
    "print (\"\\nNaive Bayes- Test accuracy\",\n",
    "       round(accuracy_score(y_test,mnb_predicted_test),3))\n",
    "# Test report\n",
    "print (\"\\nNaive Bayes  - Test Classification Report\\n\",\n",
    "       classification_report(y_test,mnb_predicted_test))\n",
    "\n",
    "print(\"------------------------------------------------------------\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "SVM - Train Confusion Matrix\n",
      "\n",
      " Predicted     1    2     3     4      5\n",
      "Actual                                 \n",
      "1          2233   39    63    48    527\n",
      "2           247  713   115   118    500\n",
      "3           154   61  1057   255    913\n",
      "4            74   35   140  1526   2734\n",
      "5           135   30   102   376  19805\n",
      "\n",
      "SVM- Train accuracy 0.792\n",
      "\n",
      "SVM  - Train Classification Report\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          1       0.79      0.77      0.78      2910\n",
      "          2       0.81      0.42      0.55      1693\n",
      "          3       0.72      0.43      0.54      2440\n",
      "          4       0.66      0.34      0.45      4509\n",
      "          5       0.81      0.97      0.88     20448\n",
      "\n",
      "avg / total       0.78      0.79      0.77     32000\n",
      "\n",
      "------------------------------------------------------------\n",
      "\n",
      "SVM - Test Confusion Matrix\n",
      "\n",
      " Predicted    1   2    3    4     5\n",
      "Actual                            \n",
      "1          408  32   58   37   205\n",
      "2           90  57   63   41   175\n",
      "3           65  39  124  120   262\n",
      "4           35  28   54  211   806\n",
      "5           78  22   61  182  4747\n",
      "\n",
      "SVM- Test accuracy 0.693\n",
      "\n",
      "SVM  - Test Classification Report\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          1       0.60      0.55      0.58       740\n",
      "          2       0.32      0.13      0.19       426\n",
      "          3       0.34      0.20      0.26       610\n",
      "          4       0.36      0.19      0.24      1134\n",
      "          5       0.77      0.93      0.84      5090\n",
      "\n",
      "avg / total       0.64      0.69      0.65      8000\n",
      "\n",
      "------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Training confusion matrix\n",
    "print (\"\\nSVM - Train Confusion Matrix\\n\\n\",\n",
    "       pd.crosstab(y_train, svm_predicted_train, rownames = [\"Actual\"], colnames = [\"Predicted\"]))\n",
    "# Training accuracy\n",
    "print (\"\\nSVM- Train accuracy\",\n",
    "       round(accuracy_score(y_train, svm_predicted_train),3))\n",
    "# Training report\n",
    "print (\"\\nSVM  - Train Classification Report\\n\",\n",
    "       classification_report(y_train, svm_predicted_train))\n",
    "\n",
    "print(\"------------------------------------------------------------\")\n",
    "\n",
    "# Test confusion matrix\n",
    "print (\"\\nSVM - Test Confusion Matrix\\n\\n\",\n",
    "       pd.crosstab(y_test,svm_predicted_test,rownames = [\"Actual\"], colnames = [\"Predicted\"]))  \n",
    "# Test accuracy\n",
    "print (\"\\nSVM- Test accuracy\",\n",
    "       round(accuracy_score(y_test,svm_predicted_test),3))\n",
    "# Test report\n",
    "print (\"\\nSVM  - Test Classification Report\\n\",\n",
    "       classification_report(y_test,svm_predicted_test))\n",
    "\n",
    "print(\"------------------------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Logistic Regression - Train Confusion Matrix\n",
      "\n",
      " Predicted    1   2   3    4      5\n",
      "Actual                            \n",
      "1          921   0   7   14   1968\n",
      "2          145  11   6   41   1490\n",
      "3           79   0  47   75   2239\n",
      "4           27   0   6  238   4238\n",
      "5           53   0   4   41  20350\n",
      "\n",
      "Logistic Regression- Train accuracy 0.674\n",
      "\n",
      "Logistic Regression  - Train Classification Report\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          1       0.75      0.32      0.45      2910\n",
      "          2       1.00      0.01      0.01      1693\n",
      "          3       0.67      0.02      0.04      2440\n",
      "          4       0.58      0.05      0.10      4509\n",
      "          5       0.67      1.00      0.80     20448\n",
      "\n",
      "avg / total       0.68      0.67      0.57     32000\n",
      "\n",
      "------------------------------------------------------------\n",
      "\n",
      "Logistic Regression - Test Confusion Matrix\n",
      "\n",
      " Predicted    1   2   3    4     5\n",
      "Actual                           \n",
      "1          370   6  37   20   307\n",
      "2           86  22  35   28   255\n",
      "3           59  11  74  109   357\n",
      "4           31   5  24  170   904\n",
      "5           43   0  26   81  4940\n",
      "\n",
      "Logistic Regression- Test accuracy 0.697\n",
      "\n",
      "Logistic Regression  - Test Classification Report\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          1       0.63      0.50      0.56       740\n",
      "          2       0.50      0.05      0.09       426\n",
      "          3       0.38      0.12      0.18       610\n",
      "          4       0.42      0.15      0.22      1134\n",
      "          5       0.73      0.97      0.83      5090\n",
      "\n",
      "avg / total       0.64      0.70      0.63      8000\n",
      "\n",
      "------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Training confusion matrix\n",
    "print (\"\\nLogistic Regression - Train Confusion Matrix\\n\\n\",\n",
    "       pd.crosstab(y_train, mnb_predicted_train, rownames = [\"Actual\"], colnames = [\"Predicted\"]))\n",
    "# Training accuracy\n",
    "print (\"\\nLogistic Regression- Train accuracy\",\n",
    "       round(accuracy_score(y_train, mnb_predicted_train),3))\n",
    "# Training report\n",
    "print (\"\\nLogistic Regression  - Train Classification Report\\n\",\n",
    "       classification_report(y_train, mnb_predicted_train))\n",
    "\n",
    "print(\"------------------------------------------------------------\")\n",
    "\n",
    "# Test confusion matrix\n",
    "print (\"\\nLogistic Regression - Test Confusion Matrix\\n\\n\",\n",
    "       pd.crosstab(y_test,log_predicted_test,rownames = [\"Actual\"], colnames = [\"Predicted\"]))  \n",
    "# Test accuracy\n",
    "print (\"\\nLogistic Regression- Test accuracy\",\n",
    "       round(accuracy_score(y_test,log_predicted_test),3))\n",
    "# Test report\n",
    "print (\"\\nLogistic Regression  - Test Classification Report\\n\",\n",
    "       classification_report(y_test,log_predicted_test))\n",
    "\n",
    "print(\"------------------------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Top 10 features - First ten & Last ten\n",
      "\n",
      "|\t-9.7700\t140             \t\t|\t-5.1244\ttast            |\n",
      "|\t-9.7700\tabl buy         \t\t|\t-5.1302\tproduct         |\n",
      "|\t-9.7700\tabsolut best    \t\t|\t-5.2070\tlike            |\n",
      "|\t-9.7700\tafternoon snack \t\t|\t-5.2512\tbuy             |\n",
      "|\t-9.7700\tagav nectar     \t\t|\t-5.5338\tcoffe           |\n",
      "|\t-9.7700\talway love      \t\t|\t-5.6061\torder           |\n",
      "|\t-9.7700\tamazon best     \t\t|\t-5.6212\tfood            |\n",
      "|\t-9.7700\tauto ship       \t\t|\t-5.6276\ttri             |\n",
      "|\t-9.7700\tbest cup        \t\t|\t-5.6356\tbox             |\n",
      "|\t-9.7700\tbest deal       \t\t|\t-5.6976\tdog             |\n"
     ]
    }
   ],
   "source": [
    "# Getting feature names from vectorizer\n",
    "feature_names = vectorizer.get_feature_names()\n",
    "\n",
    "# Getting weights assigned to the features (it works only with linear kernels)\n",
    "# Empirical log probability of features given a class, P(x_i|y).\n",
    "coefs = mnb_clf.coef_\n",
    "\n",
    "# Smoothed empirical log probability for each class.\n",
    "intercept = mnb_clf.intercept_\n",
    "\n",
    "# Sorted coefs\n",
    "coefs_with_fns = sorted(zip(mnb_clf.coef_[0], feature_names))\n",
    "\n",
    "print (\"\\n\\nTop 10 features - First ten & Last ten\\n\")\n",
    "n = 10\n",
    "top_n_coefs = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1])\n",
    "for (coef_1, fn_1), (coef_2, fn_2) in top_n_coefs:\n",
    "    # %-15s is for padding left\n",
    "    print('|\\t%.4f\\t%-16s\\t\\t|\\t%.4f\\t%-16s|' % (coef_1, fn_1, coef_2, fn_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'y_test' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-c0b7c04e8c21>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0my_test\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'y_test' is not defined"
     ]
    }
   ],
   "source": [
    "y_test.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}