p7.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\reliance\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk import sent_tokenize\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>id</th>\n",
       "      <th>Text</th>\n",
       "      <th>Annotation</th>\n",
       "      <th>oh_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.74948705591165E+017</td>\n",
       "      <td>5.74948705591165E+017</td>\n",
       "      <td>@halalflaws @biebervalue @greenlinerzjm I read...</td>\n",
       "      <td>none</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.71917888690393E+017</td>\n",
       "      <td>5.71917888690393E+017</td>\n",
       "      <td>@ShreyaBafna3 Now you idiots claim that people...</td>\n",
       "      <td>none</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.90255841338601E+017</td>\n",
       "      <td>3.90255841338601E+017</td>\n",
       "      <td>RT @Mooseoftorment Call me sexist, but when I ...</td>\n",
       "      <td>sexism</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.68208850655916E+017</td>\n",
       "      <td>5.68208850655916E+017</td>\n",
       "      <td>@g0ssipsquirrelx Wrong, ISIS follows the examp...</td>\n",
       "      <td>racism</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.75596338802373E+017</td>\n",
       "      <td>5.75596338802373E+017</td>\n",
       "      <td>#mkr No No No No No No</td>\n",
       "      <td>none</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16846</th>\n",
       "      <td>5.75606766236475E+017</td>\n",
       "      <td>5.75606766236475E+017</td>\n",
       "      <td>Feeling so sorry for the girls, they should be...</td>\n",
       "      <td>none</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16847</th>\n",
       "      <td>5.72333822886326E+017</td>\n",
       "      <td>5.72333822886326E+017</td>\n",
       "      <td>#MKR 'pretty good dishes we're happy with' - O...</td>\n",
       "      <td>none</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16848</th>\n",
       "      <td>5.72326950057845E+017</td>\n",
       "      <td>5.72326950057845E+017</td>\n",
       "      <td>RT @colonelkickhead: Deconstructed lemon tart!...</td>\n",
       "      <td>none</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16849</th>\n",
       "      <td>5.74799612642357E+017</td>\n",
       "      <td>5.74799612642357E+017</td>\n",
       "      <td>@versacezaynx @nyazpolitics @greenlinerzjm You...</td>\n",
       "      <td>none</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16850</th>\n",
       "      <td>5.68826121153684E+017</td>\n",
       "      <td>5.68826121153684E+017</td>\n",
       "      <td>And before you protest that you're *not* mad, ...</td>\n",
       "      <td>none</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16851 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       index                     id  \\\n",
       "0      5.74948705591165E+017  5.74948705591165E+017   \n",
       "1      5.71917888690393E+017  5.71917888690393E+017   \n",
       "2      3.90255841338601E+017  3.90255841338601E+017   \n",
       "3      5.68208850655916E+017  5.68208850655916E+017   \n",
       "4      5.75596338802373E+017  5.75596338802373E+017   \n",
       "...                      ...                    ...   \n",
       "16846  5.75606766236475E+017  5.75606766236475E+017   \n",
       "16847  5.72333822886326E+017  5.72333822886326E+017   \n",
       "16848  5.72326950057845E+017  5.72326950057845E+017   \n",
       "16849  5.74799612642357E+017  5.74799612642357E+017   \n",
       "16850  5.68826121153684E+017  5.68826121153684E+017   \n",
       "\n",
       "                                                    Text Annotation  oh_label  \n",
       "0      @halalflaws @biebervalue @greenlinerzjm I read...       none       0.0  \n",
       "1      @ShreyaBafna3 Now you idiots claim that people...       none       0.0  \n",
       "2      RT @Mooseoftorment Call me sexist, but when I ...     sexism       1.0  \n",
       "3      @g0ssipsquirrelx Wrong, ISIS follows the examp...     racism       1.0  \n",
       "4                                 #mkr No No No No No No       none       0.0  \n",
       "...                                                  ...        ...       ...  \n",
       "16846  Feeling so sorry for the girls, they should be...       none       0.0  \n",
       "16847  #MKR 'pretty good dishes we're happy with' - O...       none       0.0  \n",
       "16848  RT @colonelkickhead: Deconstructed lemon tart!...       none       0.0  \n",
       "16849  @versacezaynx @nyazpolitics @greenlinerzjm You...       none       0.0  \n",
       "16850  And before you protest that you're *not* mad, ...       none       0.0  \n",
       "\n",
       "[16851 rows x 5 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=pd.read_csv(\"twitter_parsed_dataset (1).csv\")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['index', 'id', 'Text', 'Annotation', 'oh_label'], dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        @halalflaws @biebervalue @greenlinerzjm I read...\n",
       "1        @ShreyaBafna3 Now you idiots claim that people...\n",
       "2        RT @Mooseoftorment Call me sexist, but when I ...\n",
       "3        @g0ssipsquirrelx Wrong, ISIS follows the examp...\n",
       "4                                   #mkr No No No No No No\n",
       "                               ...                        \n",
       "16846    Feeling so sorry for the girls, they should be...\n",
       "16847    #MKR 'pretty good dishes we're happy with' - O...\n",
       "16848    RT @colonelkickhead: Deconstructed lemon tart!...\n",
       "16849    @versacezaynx @nyazpolitics @greenlinerzjm You...\n",
       "16850    And before you protest that you're *not* mad, ...\n",
       "Name: Text, Length: 16851, dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['@halalflaws @biebervalue @greenlinerzjm I read them in context.No change in meaning. The history of Islamic slavery. https://t.co/xWJzpSodGj',\n",
       " '@ShreyaBafna3 Now you idiots claim that people who tried to stop him from becoming a terrorist made him a terrorist. Islamically brain dead.',\n",
       " \"RT @Mooseoftorment Call me sexist, but when I go to an auto place, I'd rather talk to a guy\",\n",
       " '@g0ssipsquirrelx Wrong, ISIS follows the example of Mohammed and the Quran exactly.',\n",
       " '#mkr No No No No No No',\n",
       " \"RT @TRobinsonNewEra: http://t.co/nkkCbpcHEo Saudi preacher who 'raped and tortured' his five -year-old daughter to death is released after …\",\n",
       " 'RT @Millhouse66 @Maureen_JS nooo not sexist but most women are bad drivers',\n",
       " \"Going to make some pancakes.....Don't hve any strawberries ....🍓🍓🍓🍓but I hve bananas .....🍌🍌🍌🍌. ;))) #MKR\",\n",
       " 'RT @ahtweet: @freebsdgirl How dare you have feelings is a fantastic way to dehumanize someone.',\n",
       " \"RT @Newmanzaa: There's something wrong when a girl wins Wayne Rooney street striker #NotSexist\"]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts=[]\n",
    "for t in data['Text'][0:10]:\n",
    "    texts.append(t)\n",
    "texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['@', 'ShreyaBafna3', 'Now', 'you', 'idiots', 'claim', 'that', 'people', 'who', 'tried', 'to', 'stop', 'him', 'from', 'becoming', 'a', 'terrorist', 'made', 'him', 'a', 'terrorist', '.', 'Islamically', 'brain', 'dead', '.']\n"
     ]
    }
   ],
   "source": [
    "tokenized_word=word_tokenize(texts[1])\n",
    "print(tokenized_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['@ShreyaBafna3 Now you idiots claim that people who tried to stop him from becoming a terrorist made him a terrorist.',\n",
       " 'Islamically brain dead.']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_sent=sent_tokenize(texts[1])\n",
    "tokenized_sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\reliance\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "@\n",
      "ShreyaBafna3\n",
      "Now\n",
      "idiots\n",
      "claim\n",
      "people\n",
      "tried\n",
      "stop\n",
      "becoming\n",
      "terrorist\n",
      "made\n",
      "terrorist\n",
      ".\n",
      "Islamically\n",
      "brain\n",
      "dead\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stopwords=stopwords.words('english')\n",
    "for i in tokenized_word:\n",
    "    if i not in stopwords:\n",
    "        print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n"
     ]
    }
   ],
   "source": [
    "print(stopwords)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "@\n",
      "shreyabafna3\n",
      "now\n",
      "you\n",
      "idiot\n",
      "claim\n",
      "that\n",
      "peopl\n",
      "who\n",
      "tri\n",
      "to\n",
      "stop\n",
      "him\n",
      "from\n",
      "becom\n",
      "a\n",
      "terrorist\n",
      "made\n",
      "him\n",
      "a\n",
      "terrorist\n",
      ".\n",
      "islam\n",
      "brain\n",
      "dead\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import PorterStemmer\n",
    "ps=PorterStemmer()\n",
    "for i in tokenized_word:\n",
    "    print(ps.stem(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "@-----1\n",
      "ShreyaBafna3-----1\n",
      "Now-----1\n",
      "you-----1\n",
      "idiots-----1\n",
      "claim-----1\n",
      "that-----1\n",
      "people-----1\n",
      "who-----1\n",
      "tried-----1\n",
      "to-----1\n",
      "stop-----1\n",
      "him-----2\n",
      "from-----1\n",
      "becoming-----1\n",
      "a-----2\n",
      "terrorist-----2\n",
      "made-----1\n",
      ".-----2\n",
      "Islamically-----1\n",
      "brain-----1\n",
      "dead-----1\n"
     ]
    }
   ],
   "source": [
    "freqdist=nltk.FreqDist(tokenized_word)\n",
    "for i,j in freqdist.items():\n",
    "    print(f'{i}-----{j}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('@', 'ShreyaBafna3'), ('ShreyaBafna3', 'Now'), ('Now', 'you'), ('you', 'idiots'), ('idiots', 'claim'), ('claim', 'that'), ('that', 'people'), ('people', 'who'), ('who', 'tried'), ('tried', 'to'), ('to', 'stop'), ('stop', 'him'), ('him', 'from'), ('from', 'becoming'), ('becoming', 'a'), ('a', 'terrorist'), ('terrorist', 'made'), ('made', 'him'), ('him', 'a'), ('a', 'terrorist'), ('terrorist', '.'), ('.', 'Islamically'), ('Islamically', 'brain'), ('brain', 'dead'), ('dead', '.')]\n",
      "[('@', 'ShreyaBafna3', 'Now'), ('ShreyaBafna3', 'Now', 'you'), ('Now', 'you', 'idiots'), ('you', 'idiots', 'claim'), ('idiots', 'claim', 'that'), ('claim', 'that', 'people'), ('that', 'people', 'who'), ('people', 'who', 'tried'), ('who', 'tried', 'to'), ('tried', 'to', 'stop'), ('to', 'stop', 'him'), ('stop', 'him', 'from'), ('him', 'from', 'becoming'), ('from', 'becoming', 'a'), ('becoming', 'a', 'terrorist'), ('a', 'terrorist', 'made'), ('terrorist', 'made', 'him'), ('made', 'him', 'a'), ('him', 'a', 'terrorist'), ('a', 'terrorist', '.'), ('terrorist', '.', 'Islamically'), ('.', 'Islamically', 'brain'), ('Islamically', 'brain', 'dead'), ('brain', 'dead', '.')]\n",
      "[('@', 'ShreyaBafna3', 'Now', 'you'), ('ShreyaBafna3', 'Now', 'you', 'idiots'), ('Now', 'you', 'idiots', 'claim'), ('you', 'idiots', 'claim', 'that'), ('idiots', 'claim', 'that', 'people'), ('claim', 'that', 'people', 'who'), ('that', 'people', 'who', 'tried'), ('people', 'who', 'tried', 'to'), ('who', 'tried', 'to', 'stop'), ('tried', 'to', 'stop', 'him'), ('to', 'stop', 'him', 'from'), ('stop', 'him', 'from', 'becoming'), ('him', 'from', 'becoming', 'a'), ('from', 'becoming', 'a', 'terrorist'), ('becoming', 'a', 'terrorist', 'made'), ('a', 'terrorist', 'made', 'him'), ('terrorist', 'made', 'him', 'a'), ('made', 'him', 'a', 'terrorist'), ('him', 'a', 'terrorist', '.'), ('a', 'terrorist', '.', 'Islamically'), ('terrorist', '.', 'Islamically', 'brain'), ('.', 'Islamically', 'brain', 'dead'), ('Islamically', 'brain', 'dead', '.')]\n"
     ]
    }
   ],
   "source": [
    "print(list(nltk.bigrams(tokenized_word)))\n",
    "\n",
    "print(list(nltk.trigrams(tokenized_word)))\n",
    "\n",
    "print(list(nltk.ngrams(tokenized_word,4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     C:\\Users\\reliance\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('omw-1.4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\reliance\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "@\n",
      "ShreyaBafna3\n",
      "Now\n",
      "you\n",
      "idiots\n",
      "claim\n",
      "that\n",
      "people\n",
      "who\n",
      "try\n",
      "to\n",
      "stop\n",
      "him\n",
      "from\n",
      "become\n",
      "a\n",
      "terrorist\n",
      "make\n",
      "him\n",
      "a\n",
      "terrorist\n",
      ".\n",
      "Islamically\n",
      "brain\n",
      "dead\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "lem=WordNetLemmatizer()\n",
    "for w in tokenized_word:\n",
    "    print(lem.lemmatize(w,pos='v'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\reliance\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('averaged_perceptron_tagger')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('@', 'JJ'), ('ShreyaBafna3', 'NNP'), ('Now', 'RB'), ('you', 'PRP'), ('idiots', 'VBP'), ('claim', 'VB'), ('that', 'IN'), ('people', 'NNS'), ('who', 'WP'), ('tried', 'VBD'), ('to', 'TO'), ('stop', 'VB'), ('him', 'PRP'), ('from', 'IN'), ('becoming', 'VBG'), ('a', 'DT'), ('terrorist', 'NN'), ('made', 'VBD'), ('him', 'PRP'), ('a', 'DT'), ('terrorist', 'NN'), ('.', '.'), ('Islamically', 'RB'), ('brain', 'NN'), ('dead', 'JJ'), ('.', '.')]\n"
     ]
    }
   ],
   "source": [
    "from nltk.tag.perceptron import AveragedPerceptron\n",
    "\n",
    "print(nltk.pos_tag(tokenized_word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n",
      "C:\\Users\\reliance\\AppData\\Local\\Temp\\ipykernel_6480\\2235300762.py:7: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)\n"
     ]
    }
   ],
   "source": [
    "n_docs = len(tokenized_word)\n",
    "n_words_set = len(tokenized_sent)\n",
    "\n",
    "df_tf = pd.DataFrame(columns=['Word', 'TF'])\n",
    "for word in set(tokenized_word):\n",
    "    tf = tokenized_word.count(word) / n_docs\n",
    "    df_tf = df_tf.append({'Word': word, 'TF': tf}, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>TF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>brain</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>.</td>\n",
       "      <td>0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>@</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>people</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>idiots</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>made</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>you</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>claim</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>to</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ShreyaBafna3</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>who</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Now</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>from</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>a</td>\n",
       "      <td>0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Islamically</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>tried</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>becoming</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>him</td>\n",
       "      <td>0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>dead</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>terrorist</td>\n",
       "      <td>0.076923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>that</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>stop</td>\n",
       "      <td>0.038462</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Word        TF\n",
       "0          brain  0.038462\n",
       "1              .  0.076923\n",
       "2              @  0.038462\n",
       "3         people  0.038462\n",
       "4         idiots  0.038462\n",
       "5           made  0.038462\n",
       "6            you  0.038462\n",
       "7          claim  0.038462\n",
       "8             to  0.038462\n",
       "9   ShreyaBafna3  0.038462\n",
       "10           who  0.038462\n",
       "11           Now  0.038462\n",
       "12          from  0.038462\n",
       "13             a  0.076923\n",
       "14   Islamically  0.038462\n",
       "15         tried  0.038462\n",
       "16      becoming  0.038462\n",
       "17           him  0.076923\n",
       "18          dead  0.038462\n",
       "19     terrorist  0.076923\n",
       "20          that  0.038462\n",
       "21          stop  0.038462"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'@'"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}