mvp_script.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reset -fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "import re\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import pos_tag\n",
    "import spacy\n",
    "from spacy import displacy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>President</th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>First State of the Union Address</td>\n",
       "      <td>['I embrace with great satisfaction the opport...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>Second State of the Union Address</td>\n",
       "      <td>['Fellow-Citizens of the Senate and the House ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1791</td>\n",
       "      <td>Third State of the Union Address</td>\n",
       "      <td>['Fellow-Citizens of the Senate and the House ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1792</td>\n",
       "      <td>Fourth State of the Union Address</td>\n",
       "      <td>['Fellow-Citizens of the Senate and of the Hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1793</td>\n",
       "      <td>Fifth State of the Union Address</td>\n",
       "      <td>['Fellow Citizens of the Senate and of the Hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>2016</td>\n",
       "      <td>Barack Obama's Eighth State of the Union Address</td>\n",
       "      <td>['Mr. Speaker, Mr. Vice President, Members of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2017</td>\n",
       "      <td>Donald Trump's First State of the Union Address</td>\n",
       "      <td>['Mr. Speaker, Mr. Vice President, Members of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2018</td>\n",
       "      <td>Donald Trump's Second State of the Union Address</td>\n",
       "      <td>['Mr. Speaker, Mr. Vice President, Members of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2019</td>\n",
       "      <td>Donald Trump's Third State of the Union Address</td>\n",
       "      <td>['Madam Speaker, Mr. Vice President, Members o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>Joe Biden</td>\n",
       "      <td>2022</td>\n",
       "      <td>Joe Biden's First State of The Union Address</td>\n",
       "      <td>['Madam Speaker, Madam Vice President, our Fir...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>220 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             President  Year  \\\n",
       "0    George Washington  1790   \n",
       "1    George Washington  1790   \n",
       "2    George Washington  1791   \n",
       "3    George Washington  1792   \n",
       "4    George Washington  1793   \n",
       "..                 ...   ...   \n",
       "215       Barack Obama  2016   \n",
       "216       Donald Trump  2017   \n",
       "217       Donald Trump  2018   \n",
       "218       Donald Trump  2019   \n",
       "219          Joe Biden  2022   \n",
       "\n",
       "                                                Title  \\\n",
       "0                    First State of the Union Address   \n",
       "1                   Second State of the Union Address   \n",
       "2                    Third State of the Union Address   \n",
       "3                   Fourth State of the Union Address   \n",
       "4                    Fifth State of the Union Address   \n",
       "..                                                ...   \n",
       "215  Barack Obama's Eighth State of the Union Address   \n",
       "216   Donald Trump's First State of the Union Address   \n",
       "217  Donald Trump's Second State of the Union Address   \n",
       "218   Donald Trump's Third State of the Union Address   \n",
       "219      Joe Biden's First State of The Union Address   \n",
       "\n",
       "                                                  Text  \n",
       "0    ['I embrace with great satisfaction the opport...  \n",
       "1    ['Fellow-Citizens of the Senate and the House ...  \n",
       "2    ['Fellow-Citizens of the Senate and the House ...  \n",
       "3    ['Fellow-Citizens of the Senate and of the Hou...  \n",
       "4    ['Fellow Citizens of the Senate and of the Hou...  \n",
       "..                                                 ...  \n",
       "215  ['Mr. Speaker, Mr. Vice President, Members of ...  \n",
       "216  ['Mr. Speaker, Mr. Vice President, Members of ...  \n",
       "217  ['Mr. Speaker, Mr. Vice President, Members of ...  \n",
       "218  ['Madam Speaker, Mr. Vice President, Members o...  \n",
       "219  ['Madam Speaker, Madam Vice President, our Fir...  \n",
       "\n",
       "[220 rows x 4 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('sotu_texts.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>President</th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>First State of the Union Address</td>\n",
       "      <td>i embrace with great satisfaction the opport...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>Second State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1791</td>\n",
       "      <td>Third State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1792</td>\n",
       "      <td>Fourth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1793</td>\n",
       "      <td>Fifth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>2016</td>\n",
       "      <td>Barack Obama's Eighth State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2017</td>\n",
       "      <td>Donald Trump's First State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2018</td>\n",
       "      <td>Donald Trump's Second State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2019</td>\n",
       "      <td>Donald Trump's Third State of the Union Address</td>\n",
       "      <td>madam speaker  mr  vice president  members o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>Joe Biden</td>\n",
       "      <td>2022</td>\n",
       "      <td>Joe Biden's First State of The Union Address</td>\n",
       "      <td>madam speaker  madam vice president  our fir...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>220 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             President  Year  \\\n",
       "0    George Washington  1790   \n",
       "1    George Washington  1790   \n",
       "2    George Washington  1791   \n",
       "3    George Washington  1792   \n",
       "4    George Washington  1793   \n",
       "..                 ...   ...   \n",
       "215       Barack Obama  2016   \n",
       "216       Donald Trump  2017   \n",
       "217       Donald Trump  2018   \n",
       "218       Donald Trump  2019   \n",
       "219          Joe Biden  2022   \n",
       "\n",
       "                                                Title  \\\n",
       "0                    First State of the Union Address   \n",
       "1                   Second State of the Union Address   \n",
       "2                    Third State of the Union Address   \n",
       "3                   Fourth State of the Union Address   \n",
       "4                    Fifth State of the Union Address   \n",
       "..                                                ...   \n",
       "215  Barack Obama's Eighth State of the Union Address   \n",
       "216   Donald Trump's First State of the Union Address   \n",
       "217  Donald Trump's Second State of the Union Address   \n",
       "218   Donald Trump's Third State of the Union Address   \n",
       "219      Joe Biden's First State of The Union Address   \n",
       "\n",
       "                                                  Text  \n",
       "0      i embrace with great satisfaction the opport...  \n",
       "1      fellow citizens of the senate and the house ...  \n",
       "2      fellow citizens of the senate and the house ...  \n",
       "3      fellow citizens of the senate and of the hou...  \n",
       "4      fellow citizens of the senate and of the hou...  \n",
       "..                                                 ...  \n",
       "215    mr  speaker  mr  vice president  members of ...  \n",
       "216    mr  speaker  mr  vice president  members of ...  \n",
       "217    mr  speaker  mr  vice president  members of ...  \n",
       "218    madam speaker  mr  vice president  members o...  \n",
       "219    madam speaker  madam vice president  our fir...  \n",
       "\n",
       "[220 rows x 4 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alphanumeric = lambda x: re.sub('\\w*\\d\\w*', ' ', x)\n",
    "punc_lower = lambda x: re.sub('[%s]' % re.escape(string.punctuation), ' ', x.lower())\n",
    "\n",
    "df['Text'] = df.Text.map(alphanumeric).map(punc_lower)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatize_text(text):\n",
    "    return [lemmatizer.lemmatize(w) for w in word_tokenize(text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Lemma_Text'] = df.Text.apply(lemmatize_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " 'embrace',\n",
       " 'with',\n",
       " 'great',\n",
       " 'satisfaction',\n",
       " 'the',\n",
       " 'opportunity',\n",
       " 'which',\n",
       " 'now',\n",
       " 'present',\n",
       " 'itself',\n",
       " 'of',\n",
       " 'congratulating',\n",
       " 'you',\n",
       " 'on',\n",
       " 'the',\n",
       " 'present',\n",
       " 'favourable',\n",
       " 'prospect',\n",
       " 'of',\n",
       " 'our',\n",
       " 'public',\n",
       " 'affair',\n",
       " 'the',\n",
       " 'recent',\n",
       " 'accession',\n",
       " 'of',\n",
       " 'the',\n",
       " 'important',\n",
       " 'state',\n",
       " 'of',\n",
       " 'north',\n",
       " 'carolina',\n",
       " 'to',\n",
       " 'the',\n",
       " 'constitution',\n",
       " 'of',\n",
       " 'the',\n",
       " 'united',\n",
       " 'state',\n",
       " 'of',\n",
       " 'which',\n",
       " 'official',\n",
       " 'information',\n",
       " 'ha',\n",
       " 'been',\n",
       " 'received',\n",
       " '—',\n",
       " 'the',\n",
       " 'rising',\n",
       " 'credit',\n",
       " 'and',\n",
       " 'respectability',\n",
       " 'of',\n",
       " 'our',\n",
       " 'country',\n",
       " '—',\n",
       " 'the',\n",
       " 'general',\n",
       " 'and',\n",
       " 'increasing',\n",
       " 'good',\n",
       " 'will',\n",
       " 'towards',\n",
       " 'the',\n",
       " 'government',\n",
       " 'of',\n",
       " 'the',\n",
       " 'union',\n",
       " '—',\n",
       " 'and',\n",
       " 'the',\n",
       " 'concord',\n",
       " 'peace',\n",
       " 'and',\n",
       " 'plenty',\n",
       " 'with',\n",
       " 'which',\n",
       " 'we',\n",
       " 'are',\n",
       " 'blessed',\n",
       " 'are',\n",
       " 'circumstance',\n",
       " 'auspicious',\n",
       " 'in',\n",
       " 'an',\n",
       " 'eminent',\n",
       " 'degree',\n",
       " 'to',\n",
       " 'our',\n",
       " 'national',\n",
       " 'prosperity',\n",
       " 'in',\n",
       " 'resuming',\n",
       " 'your',\n",
       " 'consultation',\n",
       " 'for',\n",
       " 'the',\n",
       " 'general',\n",
       " 'good',\n",
       " 'you',\n",
       " 'can',\n",
       " 'not',\n",
       " 'but',\n",
       " 'derive',\n",
       " 'encouragement',\n",
       " 'from',\n",
       " 'the',\n",
       " 'reflection',\n",
       " 'that',\n",
       " 'the',\n",
       " 'measure',\n",
       " 'of',\n",
       " 'the',\n",
       " 'last',\n",
       " 'session',\n",
       " 'have',\n",
       " 'been',\n",
       " 'a',\n",
       " 'satisfactory',\n",
       " 'to',\n",
       " 'your',\n",
       " 'constituent',\n",
       " 'a',\n",
       " 'the',\n",
       " 'novelty',\n",
       " 'and',\n",
       " 'difficulty',\n",
       " 'of',\n",
       " 'the',\n",
       " 'work',\n",
       " 'allowed',\n",
       " 'you',\n",
       " 'to',\n",
       " 'hope',\n",
       " '—',\n",
       " 'still',\n",
       " 'further',\n",
       " 'to',\n",
       " 'their',\n",
       " 'expectation',\n",
       " 'and',\n",
       " 'to',\n",
       " 'secure',\n",
       " 'the',\n",
       " 'blessing',\n",
       " 'which',\n",
       " 'a',\n",
       " 'gracious',\n",
       " 'providence',\n",
       " 'ha',\n",
       " 'placed',\n",
       " 'within',\n",
       " 'our',\n",
       " 'reach',\n",
       " 'will',\n",
       " 'in',\n",
       " 'the',\n",
       " 'course',\n",
       " 'of',\n",
       " 'the',\n",
       " 'present',\n",
       " 'important',\n",
       " 'session',\n",
       " 'call',\n",
       " 'for',\n",
       " 'the',\n",
       " 'cool',\n",
       " 'and',\n",
       " 'deliberate',\n",
       " 'exertion',\n",
       " 'of',\n",
       " 'your',\n",
       " 'patriotism',\n",
       " 'firmness',\n",
       " 'and',\n",
       " 'wisdom',\n",
       " 'among',\n",
       " 'the',\n",
       " 'many',\n",
       " 'interesting',\n",
       " 'object',\n",
       " 'which',\n",
       " 'will',\n",
       " 'engage',\n",
       " 'your',\n",
       " 'attention',\n",
       " 'that',\n",
       " 'of',\n",
       " 'providing',\n",
       " 'for',\n",
       " 'the',\n",
       " 'common',\n",
       " 'defence',\n",
       " 'will',\n",
       " 'merit',\n",
       " 'particular',\n",
       " 'regard',\n",
       " '—',\n",
       " 'to',\n",
       " 'be',\n",
       " 'prepared',\n",
       " 'for',\n",
       " 'war',\n",
       " 'is',\n",
       " 'one',\n",
       " 'of',\n",
       " 'the',\n",
       " 'most',\n",
       " 'effectual',\n",
       " 'mean',\n",
       " 'of',\n",
       " 'preserving',\n",
       " 'peace',\n",
       " '—',\n",
       " 'a',\n",
       " 'free',\n",
       " 'people',\n",
       " 'ought',\n",
       " 'not',\n",
       " 'only',\n",
       " 'to',\n",
       " 'be',\n",
       " 'armed',\n",
       " 'but',\n",
       " 'disciplined',\n",
       " '—',\n",
       " 'to',\n",
       " 'which',\n",
       " 'end',\n",
       " 'a',\n",
       " 'uniform',\n",
       " 'and',\n",
       " 'well',\n",
       " 'digested',\n",
       " 'plan',\n",
       " 'is',\n",
       " 'requisite',\n",
       " 'and',\n",
       " 'their',\n",
       " 'safety',\n",
       " 'and',\n",
       " 'interest',\n",
       " 'require',\n",
       " 'that',\n",
       " 'they',\n",
       " 'should',\n",
       " 'promote',\n",
       " 'such',\n",
       " 'manufactory',\n",
       " 'a',\n",
       " 'tend',\n",
       " 'to',\n",
       " 'render',\n",
       " 'them',\n",
       " 'independent',\n",
       " 'others',\n",
       " 'for',\n",
       " 'essential',\n",
       " 'particularly',\n",
       " 'for',\n",
       " 'military',\n",
       " 'supply',\n",
       " 'the',\n",
       " 'proper',\n",
       " 'establishment',\n",
       " 'of',\n",
       " 'the',\n",
       " 'troop',\n",
       " 'which',\n",
       " 'may',\n",
       " 'be',\n",
       " 'deemed',\n",
       " 'indispensible',\n",
       " 'will',\n",
       " 'be',\n",
       " 'entitled',\n",
       " 'to',\n",
       " 'mature',\n",
       " 'consideration',\n",
       " '—',\n",
       " 'in',\n",
       " 'the',\n",
       " 'arrangement',\n",
       " 'which',\n",
       " 'may',\n",
       " 'be',\n",
       " 'made',\n",
       " 'respecting',\n",
       " 'it',\n",
       " 'it',\n",
       " 'will',\n",
       " 'be',\n",
       " 'of',\n",
       " 'to',\n",
       " 'conciliate',\n",
       " 'the',\n",
       " 'comfortable',\n",
       " 'support',\n",
       " 'of',\n",
       " 'the',\n",
       " 'officer',\n",
       " 'and',\n",
       " 'soldier',\n",
       " 'with',\n",
       " 'a',\n",
       " 'due',\n",
       " 'regard',\n",
       " 'to',\n",
       " 'oeconomy',\n",
       " 'there',\n",
       " 'wa',\n",
       " 'reason',\n",
       " 'to',\n",
       " 'hope',\n",
       " 'that',\n",
       " 'the',\n",
       " 'pacific',\n",
       " 'measure',\n",
       " 'adopted',\n",
       " 'with',\n",
       " 'regard',\n",
       " 'to',\n",
       " 'certain',\n",
       " 'hostile',\n",
       " 'tribe',\n",
       " 'of',\n",
       " 'indian',\n",
       " 'would',\n",
       " 'have',\n",
       " 'relieved',\n",
       " 'the',\n",
       " 'inhabitant',\n",
       " 'of',\n",
       " 'our',\n",
       " 'southern',\n",
       " 'and',\n",
       " 'western',\n",
       " 'frontier',\n",
       " 'from',\n",
       " 'their',\n",
       " 'depredation',\n",
       " '—',\n",
       " 'but',\n",
       " 'you',\n",
       " 'will',\n",
       " 'percieve',\n",
       " 'from',\n",
       " 'the',\n",
       " 'information',\n",
       " 'contained',\n",
       " 'in',\n",
       " 'the',\n",
       " 'paper',\n",
       " 'which',\n",
       " 'i',\n",
       " 'shall',\n",
       " 'direct',\n",
       " 'to',\n",
       " 'be',\n",
       " 'laid',\n",
       " 'before',\n",
       " 'you',\n",
       " 'comprehending',\n",
       " 'a',\n",
       " 'communication',\n",
       " 'from',\n",
       " 'the',\n",
       " 'commonwealth',\n",
       " 'of',\n",
       " 'virginia',\n",
       " 'that',\n",
       " 'we',\n",
       " 'ought',\n",
       " 'to',\n",
       " 'be',\n",
       " 'prepared',\n",
       " 'to',\n",
       " 'afford',\n",
       " 'protection',\n",
       " 'to',\n",
       " 'those',\n",
       " 'part',\n",
       " 'of',\n",
       " 'the',\n",
       " 'union',\n",
       " 'and',\n",
       " 'if',\n",
       " 'necessary',\n",
       " 'to',\n",
       " 'punish',\n",
       " 'aggressor',\n",
       " 'the',\n",
       " 'interest',\n",
       " 'of',\n",
       " 'the',\n",
       " 'united',\n",
       " 'state',\n",
       " 'require',\n",
       " 'that',\n",
       " 'our',\n",
       " 'intercourse',\n",
       " 'with',\n",
       " 'other',\n",
       " 'nation',\n",
       " 'should',\n",
       " 'be',\n",
       " 'facilitated',\n",
       " 'by',\n",
       " 'such',\n",
       " 'provision',\n",
       " 'a',\n",
       " 'will',\n",
       " 'enable',\n",
       " 'me',\n",
       " 'to',\n",
       " 'fulfill',\n",
       " 'my',\n",
       " 'duty',\n",
       " 'in',\n",
       " 'that',\n",
       " 'respect',\n",
       " 'in',\n",
       " 'the',\n",
       " 'manner',\n",
       " 'which',\n",
       " 'circumstance',\n",
       " 'may',\n",
       " 'render',\n",
       " 'most',\n",
       " 'conducive',\n",
       " 'to',\n",
       " 'the',\n",
       " 'public',\n",
       " 'good',\n",
       " 'and',\n",
       " 'to',\n",
       " 'this',\n",
       " 'end',\n",
       " 'that',\n",
       " 'the',\n",
       " 'compensation',\n",
       " 'to',\n",
       " 'be',\n",
       " 'made',\n",
       " 'to',\n",
       " 'the',\n",
       " 'person',\n",
       " 'who',\n",
       " 'may',\n",
       " 'be',\n",
       " 'employed',\n",
       " 'should',\n",
       " 'according',\n",
       " 'to',\n",
       " 'the',\n",
       " 'nature',\n",
       " 'of',\n",
       " 'their',\n",
       " 'appointment',\n",
       " 'be',\n",
       " 'defined',\n",
       " 'by',\n",
       " 'law',\n",
       " 'and',\n",
       " 'a',\n",
       " 'competent',\n",
       " 'fund',\n",
       " 'for',\n",
       " 'defraying',\n",
       " 'the',\n",
       " 'expense',\n",
       " 'incident',\n",
       " 'to',\n",
       " 'the',\n",
       " 'conduct',\n",
       " 'of',\n",
       " 'foreign',\n",
       " 'affair',\n",
       " 'various',\n",
       " 'consideration',\n",
       " 'also',\n",
       " 'render',\n",
       " 'it',\n",
       " 'expedient',\n",
       " 'that',\n",
       " 'the',\n",
       " 'term',\n",
       " 'on',\n",
       " 'which',\n",
       " 'foreigner',\n",
       " 'may',\n",
       " 'be',\n",
       " 'admitted',\n",
       " 'to',\n",
       " 'the',\n",
       " 'right',\n",
       " 'of',\n",
       " 'citizen',\n",
       " 'should',\n",
       " 'be',\n",
       " 'speedily',\n",
       " 'ascertained',\n",
       " 'by',\n",
       " 'a',\n",
       " 'uniform',\n",
       " 'rule',\n",
       " 'of',\n",
       " 'naturalization',\n",
       " 'uniformity',\n",
       " 'in',\n",
       " 'the',\n",
       " 'currency',\n",
       " 'weight',\n",
       " 'and',\n",
       " 'measure',\n",
       " 'of',\n",
       " 'the',\n",
       " 'united',\n",
       " 'state',\n",
       " 'is',\n",
       " 'an',\n",
       " 'object',\n",
       " 'of',\n",
       " 'great',\n",
       " 'importance',\n",
       " 'and',\n",
       " 'will',\n",
       " 'i',\n",
       " 'am',\n",
       " 'persuaded',\n",
       " 'be',\n",
       " 'duly',\n",
       " 'attended',\n",
       " 'to',\n",
       " 'the',\n",
       " 'advancement',\n",
       " 'of',\n",
       " 'agriculture',\n",
       " 'commerce',\n",
       " 'and',\n",
       " 'manufacture',\n",
       " 'by',\n",
       " 'all',\n",
       " 'proper',\n",
       " 'mean',\n",
       " 'will',\n",
       " 'not',\n",
       " 'i',\n",
       " 'trust',\n",
       " 'need',\n",
       " 'recommendation',\n",
       " 'but',\n",
       " 'i',\n",
       " 'can',\n",
       " 'not',\n",
       " 'forbear',\n",
       " 'intimating',\n",
       " 'to',\n",
       " 'you',\n",
       " 'the',\n",
       " 'expediency',\n",
       " 'of',\n",
       " 'giving',\n",
       " 'effectual',\n",
       " 'encouragement',\n",
       " 'a',\n",
       " 'well',\n",
       " 'to',\n",
       " 'the',\n",
       " 'introduction',\n",
       " 'of',\n",
       " 'new',\n",
       " 'and',\n",
       " 'useful',\n",
       " 'invention',\n",
       " 'from',\n",
       " 'abroad',\n",
       " 'a',\n",
       " 'to',\n",
       " 'the',\n",
       " 'exertion',\n",
       " 'of',\n",
       " 'skill',\n",
       " 'and',\n",
       " 'genius',\n",
       " 'in',\n",
       " 'producing',\n",
       " 'them',\n",
       " 'at',\n",
       " 'home',\n",
       " 'and',\n",
       " 'of',\n",
       " 'facilitating',\n",
       " 'the',\n",
       " 'intercourse',\n",
       " 'between',\n",
       " 'the',\n",
       " 'distant',\n",
       " 'part',\n",
       " 'of',\n",
       " 'our',\n",
       " 'country',\n",
       " 'by',\n",
       " 'a',\n",
       " 'due',\n",
       " 'attention',\n",
       " 'to',\n",
       " 'the',\n",
       " 'post',\n",
       " 'office',\n",
       " 'and',\n",
       " 'post',\n",
       " 'road',\n",
       " 'am',\n",
       " 'i',\n",
       " 'le',\n",
       " 'persuaded',\n",
       " 'that',\n",
       " 'you',\n",
       " 'will',\n",
       " 'agree',\n",
       " 'with',\n",
       " 'me',\n",
       " 'in',\n",
       " 'opinion',\n",
       " 'that',\n",
       " 'there',\n",
       " 'is',\n",
       " 'nothing',\n",
       " 'which',\n",
       " 'can',\n",
       " 'better',\n",
       " 'deserve',\n",
       " 'your',\n",
       " 'patronage',\n",
       " 'than',\n",
       " 'the',\n",
       " 'promotion',\n",
       " 'of',\n",
       " 'science',\n",
       " 'and',\n",
       " 'literature',\n",
       " 'knowledge',\n",
       " 'is',\n",
       " 'in',\n",
       " 'every',\n",
       " 'country',\n",
       " 'the',\n",
       " 'surest',\n",
       " 'basis',\n",
       " 'of',\n",
       " 'public',\n",
       " 'happiness',\n",
       " 'in',\n",
       " 'one',\n",
       " 'in',\n",
       " 'which',\n",
       " 'the',\n",
       " 'measure',\n",
       " 'of',\n",
       " 'government',\n",
       " 'receive',\n",
       " 'their',\n",
       " 'impression',\n",
       " 'so',\n",
       " 'immediately',\n",
       " 'from',\n",
       " 'the',\n",
       " 'sense',\n",
       " 'of',\n",
       " 'the',\n",
       " 'community',\n",
       " 'a',\n",
       " 'in',\n",
       " 'ours',\n",
       " 'it',\n",
       " 'is',\n",
       " 'proportionably',\n",
       " 'essential',\n",
       " 'to',\n",
       " 'the',\n",
       " 'security',\n",
       " 'of',\n",
       " 'a',\n",
       " 'free',\n",
       " 'constitution',\n",
       " 'it',\n",
       " 'contributes',\n",
       " 'in',\n",
       " 'various',\n",
       " 'way',\n",
       " 'by',\n",
       " 'convincing',\n",
       " 'those',\n",
       " 'who',\n",
       " 'are',\n",
       " 'entrusted',\n",
       " 'with',\n",
       " 'the',\n",
       " 'public',\n",
       " 'administration',\n",
       " 'that',\n",
       " 'every',\n",
       " 'valuable',\n",
       " 'end',\n",
       " 'of',\n",
       " 'government',\n",
       " 'is',\n",
       " 'best',\n",
       " 'answered',\n",
       " 'by',\n",
       " 'the',\n",
       " 'enlightened',\n",
       " 'confidence',\n",
       " 'of',\n",
       " 'the',\n",
       " 'people',\n",
       " 'and',\n",
       " 'by',\n",
       " 'teaching',\n",
       " 'the',\n",
       " 'people',\n",
       " 'themselves',\n",
       " 'to',\n",
       " 'know',\n",
       " 'and',\n",
       " 'to',\n",
       " 'value',\n",
       " 'their',\n",
       " 'own',\n",
       " 'right',\n",
       " 'to',\n",
       " 'discern',\n",
       " 'and',\n",
       " 'provide',\n",
       " 'against',\n",
       " 'invasion',\n",
       " 'of',\n",
       " 'them',\n",
       " 'to',\n",
       " 'distinguish',\n",
       " 'between',\n",
       " 'oppression',\n",
       " 'and',\n",
       " 'the',\n",
       " 'necessary',\n",
       " 'exercise',\n",
       " 'of',\n",
       " 'lawful',\n",
       " 'authority',\n",
       " 'between',\n",
       " 'burthen',\n",
       " 'proceeding',\n",
       " 'from',\n",
       " 'a',\n",
       " 'disregard',\n",
       " 'to',\n",
       " 'their',\n",
       " 'convenience',\n",
       " 'and',\n",
       " 'those',\n",
       " 'resulting',\n",
       " 'from',\n",
       " 'the',\n",
       " 'inevitable',\n",
       " 'exigency',\n",
       " 'of',\n",
       " 'society',\n",
       " 'to',\n",
       " 'discriminate',\n",
       " 'the',\n",
       " 'spirit',\n",
       " 'of',\n",
       " 'liberty',\n",
       " 'from',\n",
       " 'that',\n",
       " 'of',\n",
       " 'licentiousness',\n",
       " 'cherishing',\n",
       " 'the',\n",
       " 'first',\n",
       " 'avoiding',\n",
       " 'the',\n",
       " 'last',\n",
       " 'and',\n",
       " 'uniting',\n",
       " 'a',\n",
       " 'speedy',\n",
       " 'but',\n",
       " 'temperate',\n",
       " 'vigilence',\n",
       " 'against',\n",
       " 'encroachment',\n",
       " 'with',\n",
       " 'an',\n",
       " 'inviolable',\n",
       " 'respect',\n",
       " 'to',\n",
       " 'the',\n",
       " 'law',\n",
       " 'whether',\n",
       " 'this',\n",
       " 'desirable',\n",
       " 'object',\n",
       " 'will',\n",
       " 'be',\n",
       " 'best',\n",
       " 'promoted',\n",
       " 'by',\n",
       " 'affording',\n",
       " 'aid',\n",
       " 'to',\n",
       " 'seminary',\n",
       " 'of',\n",
       " 'learning',\n",
       " 'already',\n",
       " 'established',\n",
       " 'by',\n",
       " 'the',\n",
       " 'institution',\n",
       " 'a',\n",
       " 'national',\n",
       " 'university',\n",
       " 'or',\n",
       " 'by',\n",
       " 'any',\n",
       " 'other',\n",
       " 'expedient',\n",
       " 'will',\n",
       " 'be',\n",
       " 'well',\n",
       " 'worthy',\n",
       " 'of',\n",
       " 'a',\n",
       " 'place',\n",
       " 'in',\n",
       " 'the',\n",
       " 'deliberation',\n",
       " 'of',\n",
       " 'the',\n",
       " 'legislature',\n",
       " 'i',\n",
       " 'saw',\n",
       " 'with',\n",
       " 'peculiar',\n",
       " 'pleasure',\n",
       " 'at',\n",
       " 'the',\n",
       " 'close',\n",
       " 'of',\n",
       " 'the',\n",
       " 'last',\n",
       " 'session',\n",
       " 'the',\n",
       " 'resolution',\n",
       " 'entered',\n",
       " 'into',\n",
       " 'by',\n",
       " 'you',\n",
       " 'expressive',\n",
       " 'of',\n",
       " 'your',\n",
       " 'opinion',\n",
       " 'that',\n",
       " 'an',\n",
       " 'adequate',\n",
       " 'provision',\n",
       " 'for',\n",
       " 'the',\n",
       " 'support',\n",
       " 'of',\n",
       " 'the',\n",
       " 'public',\n",
       " 'credit',\n",
       " 'is',\n",
       " 'a',\n",
       " 'matter',\n",
       " 'of',\n",
       " 'high',\n",
       " 'importance',\n",
       " 'to',\n",
       " 'the',\n",
       " 'national',\n",
       " 'honor',\n",
       " 'and',\n",
       " 'prosperity',\n",
       " 'in',\n",
       " 'this',\n",
       " 'sentiment',\n",
       " 'i',\n",
       " 'entirely',\n",
       " 'concur',\n",
       " 'and',\n",
       " 'to',\n",
       " 'a',\n",
       " 'perfect',\n",
       " 'confidence',\n",
       " 'in',\n",
       " 'your',\n",
       " 'best',\n",
       " 'endeavor',\n",
       " 'to',\n",
       " 'devise',\n",
       " 'such',\n",
       " 'a',\n",
       " 'provision',\n",
       " 'a',\n",
       " 'will',\n",
       " 'be',\n",
       " 'truly',\n",
       " 'consistent',\n",
       " 'with',\n",
       " 'the',\n",
       " 'end',\n",
       " 'i',\n",
       " 'add',\n",
       " 'an',\n",
       " 'equal',\n",
       " 'reliance',\n",
       " 'on',\n",
       " 'the',\n",
       " 'chearful',\n",
       " 'co',\n",
       " 'operation',\n",
       " 'of',\n",
       " 'the',\n",
       " 'other',\n",
       " 'branch',\n",
       " 'of',\n",
       " 'the',\n",
       " 'legislature',\n",
       " 'it',\n",
       " 'would',\n",
       " 'be',\n",
       " 'superfluous',\n",
       " 'to',\n",
       " 'specify',\n",
       " 'inducement',\n",
       " 'to',\n",
       " 'a',\n",
       " 'measure',\n",
       " 'in',\n",
       " 'which',\n",
       " 'the',\n",
       " 'character',\n",
       " 'and',\n",
       " 'permanent',\n",
       " 'interest',\n",
       " 'of',\n",
       " 'the',\n",
       " 'united',\n",
       " 'state',\n",
       " 'are',\n",
       " 'so',\n",
       " 'obviously',\n",
       " 'and',\n",
       " 'so',\n",
       " 'deeply',\n",
       " 'concerned',\n",
       " 'and',\n",
       " 'which',\n",
       " 'ha',\n",
       " 'received',\n",
       " 'so',\n",
       " 'explicit',\n",
       " 'a',\n",
       " 'sanction',\n",
       " 'from',\n",
       " 'your',\n",
       " 'declaration',\n",
       " 'i',\n",
       " 'have',\n",
       " 'directed',\n",
       " 'the',\n",
       " 'proper',\n",
       " 'officer',\n",
       " 'to',\n",
       " 'lay',\n",
       " 'before',\n",
       " 'you',\n",
       " 'respectively',\n",
       " 'such',\n",
       " 'paper',\n",
       " 'and',\n",
       " 'estimate',\n",
       " 'a',\n",
       " 'regard',\n",
       " 'the',\n",
       " 'affair',\n",
       " 'particularly',\n",
       " 'recommended',\n",
       " 'to',\n",
       " 'your',\n",
       " 'consideration',\n",
       " 'and',\n",
       " 'necessary',\n",
       " 'to',\n",
       " 'convey',\n",
       " 'to',\n",
       " ...]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.Lemma_Text[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>President</th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "      <th>Lemma_Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>First State of the Union Address</td>\n",
       "      <td>i embrace with great satisfaction the opport...</td>\n",
       "      <td>[i, embrace, with, great, satisfaction, the, o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>Second State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, the, h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1791</td>\n",
       "      <td>Third State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, the, h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1792</td>\n",
       "      <td>Fourth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, of, th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1793</td>\n",
       "      <td>Fifth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, of, th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>2016</td>\n",
       "      <td>Barack Obama's Eighth State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2017</td>\n",
       "      <td>Donald Trump's First State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2018</td>\n",
       "      <td>Donald Trump's Second State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2019</td>\n",
       "      <td>Donald Trump's Third State of the Union Address</td>\n",
       "      <td>madam speaker  mr  vice president  members o...</td>\n",
       "      <td>[madam, speaker, mr, vice, president, member, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>Joe Biden</td>\n",
       "      <td>2022</td>\n",
       "      <td>Joe Biden's First State of The Union Address</td>\n",
       "      <td>madam speaker  madam vice president  our fir...</td>\n",
       "      <td>[madam, speaker, madam, vice, president, our, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>220 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             President  Year  \\\n",
       "0    George Washington  1790   \n",
       "1    George Washington  1790   \n",
       "2    George Washington  1791   \n",
       "3    George Washington  1792   \n",
       "4    George Washington  1793   \n",
       "..                 ...   ...   \n",
       "215       Barack Obama  2016   \n",
       "216       Donald Trump  2017   \n",
       "217       Donald Trump  2018   \n",
       "218       Donald Trump  2019   \n",
       "219          Joe Biden  2022   \n",
       "\n",
       "                                                Title  \\\n",
       "0                    First State of the Union Address   \n",
       "1                   Second State of the Union Address   \n",
       "2                    Third State of the Union Address   \n",
       "3                   Fourth State of the Union Address   \n",
       "4                    Fifth State of the Union Address   \n",
       "..                                                ...   \n",
       "215  Barack Obama's Eighth State of the Union Address   \n",
       "216   Donald Trump's First State of the Union Address   \n",
       "217  Donald Trump's Second State of the Union Address   \n",
       "218   Donald Trump's Third State of the Union Address   \n",
       "219      Joe Biden's First State of The Union Address   \n",
       "\n",
       "                                                  Text  \\\n",
       "0      i embrace with great satisfaction the opport...   \n",
       "1      fellow citizens of the senate and the house ...   \n",
       "2      fellow citizens of the senate and the house ...   \n",
       "3      fellow citizens of the senate and of the hou...   \n",
       "4      fellow citizens of the senate and of the hou...   \n",
       "..                                                 ...   \n",
       "215    mr  speaker  mr  vice president  members of ...   \n",
       "216    mr  speaker  mr  vice president  members of ...   \n",
       "217    mr  speaker  mr  vice president  members of ...   \n",
       "218    madam speaker  mr  vice president  members o...   \n",
       "219    madam speaker  madam vice president  our fir...   \n",
       "\n",
       "                                            Lemma_Text  \n",
       "0    [i, embrace, with, great, satisfaction, the, o...  \n",
       "1    [fellow, citizen, of, the, senate, and, the, h...  \n",
       "2    [fellow, citizen, of, the, senate, and, the, h...  \n",
       "3    [fellow, citizen, of, the, senate, and, of, th...  \n",
       "4    [fellow, citizen, of, the, senate, and, of, th...  \n",
       "..                                                 ...  \n",
       "215  [mr, speaker, mr, vice, president, member, of,...  \n",
       "216  [mr, speaker, mr, vice, president, member, of,...  \n",
       "217  [mr, speaker, mr, vice, president, member, of,...  \n",
       "218  [madam, speaker, mr, vice, president, member, ...  \n",
       "219  [madam, speaker, madam, vice, president, our, ...  \n",
       "\n",
       "[220 rows x 5 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>President</th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "      <th>Lemma_Text</th>\n",
       "      <th>text_no_stopwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>First State of the Union Address</td>\n",
       "      <td>i embrace with great satisfaction the opport...</td>\n",
       "      <td>[i, embrace, with, great, satisfaction, the, o...</td>\n",
       "      <td>embrace great satisfaction opportunity present...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>Second State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, the, h...</td>\n",
       "      <td>fellow citizen senate house representative mee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1791</td>\n",
       "      <td>Third State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, the, h...</td>\n",
       "      <td>fellow citizen senate house representative mee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1792</td>\n",
       "      <td>Fourth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, of, th...</td>\n",
       "      <td>fellow citizen senate house representative aba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1793</td>\n",
       "      <td>Fifth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, of, th...</td>\n",
       "      <td>fellow citizen senate house representative sin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>2016</td>\n",
       "      <td>Barack Obama's Eighth State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>mr speaker mr vice president member congress f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2017</td>\n",
       "      <td>Donald Trump's First State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>mr speaker mr vice president member congress f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2018</td>\n",
       "      <td>Donald Trump's Second State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>mr speaker mr vice president member congress f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2019</td>\n",
       "      <td>Donald Trump's Third State of the Union Address</td>\n",
       "      <td>madam speaker  mr  vice president  members o...</td>\n",
       "      <td>[madam, speaker, mr, vice, president, member, ...</td>\n",
       "      <td>madam speaker mr vice president member congres...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>Joe Biden</td>\n",
       "      <td>2022</td>\n",
       "      <td>Joe Biden's First State of The Union Address</td>\n",
       "      <td>madam speaker  madam vice president  our fir...</td>\n",
       "      <td>[madam, speaker, madam, vice, president, our, ...</td>\n",
       "      <td>madam speaker madam vice president first lady ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>220 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             President  Year  \\\n",
       "0    George Washington  1790   \n",
       "1    George Washington  1790   \n",
       "2    George Washington  1791   \n",
       "3    George Washington  1792   \n",
       "4    George Washington  1793   \n",
       "..                 ...   ...   \n",
       "215       Barack Obama  2016   \n",
       "216       Donald Trump  2017   \n",
       "217       Donald Trump  2018   \n",
       "218       Donald Trump  2019   \n",
       "219          Joe Biden  2022   \n",
       "\n",
       "                                                Title  \\\n",
       "0                    First State of the Union Address   \n",
       "1                   Second State of the Union Address   \n",
       "2                    Third State of the Union Address   \n",
       "3                   Fourth State of the Union Address   \n",
       "4                    Fifth State of the Union Address   \n",
       "..                                                ...   \n",
       "215  Barack Obama's Eighth State of the Union Address   \n",
       "216   Donald Trump's First State of the Union Address   \n",
       "217  Donald Trump's Second State of the Union Address   \n",
       "218   Donald Trump's Third State of the Union Address   \n",
       "219      Joe Biden's First State of The Union Address   \n",
       "\n",
       "                                                  Text  \\\n",
       "0      i embrace with great satisfaction the opport...   \n",
       "1      fellow citizens of the senate and the house ...   \n",
       "2      fellow citizens of the senate and the house ...   \n",
       "3      fellow citizens of the senate and of the hou...   \n",
       "4      fellow citizens of the senate and of the hou...   \n",
       "..                                                 ...   \n",
       "215    mr  speaker  mr  vice president  members of ...   \n",
       "216    mr  speaker  mr  vice president  members of ...   \n",
       "217    mr  speaker  mr  vice president  members of ...   \n",
       "218    madam speaker  mr  vice president  members o...   \n",
       "219    madam speaker  madam vice president  our fir...   \n",
       "\n",
       "                                            Lemma_Text  \\\n",
       "0    [i, embrace, with, great, satisfaction, the, o...   \n",
       "1    [fellow, citizen, of, the, senate, and, the, h...   \n",
       "2    [fellow, citizen, of, the, senate, and, the, h...   \n",
       "3    [fellow, citizen, of, the, senate, and, of, th...   \n",
       "4    [fellow, citizen, of, the, senate, and, of, th...   \n",
       "..                                                 ...   \n",
       "215  [mr, speaker, mr, vice, president, member, of,...   \n",
       "216  [mr, speaker, mr, vice, president, member, of,...   \n",
       "217  [mr, speaker, mr, vice, president, member, of,...   \n",
       "218  [madam, speaker, mr, vice, president, member, ...   \n",
       "219  [madam, speaker, madam, vice, president, our, ...   \n",
       "\n",
       "                                     text_no_stopwords  \n",
       "0    embrace great satisfaction opportunity present...  \n",
       "1    fellow citizen senate house representative mee...  \n",
       "2    fellow citizen senate house representative mee...  \n",
       "3    fellow citizen senate house representative aba...  \n",
       "4    fellow citizen senate house representative sin...  \n",
       "..                                                 ...  \n",
       "215  mr speaker mr vice president member congress f...  \n",
       "216  mr speaker mr vice president member congress f...  \n",
       "217  mr speaker mr vice president member congress f...  \n",
       "218  madam speaker mr vice president member congres...  \n",
       "219  madam speaker madam vice president first lady ...  \n",
       "\n",
       "[220 rows x 6 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text_no_stopwords'] = df['Lemma_Text'].apply(lambda x: ' '.join([word for word in x if word not in (stop_words)]))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['text_no_stopwords'] = [word_tokenize(i) for i in df.text_no_stopwords]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>President</th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "      <th>Lemma_Text</th>\n",
       "      <th>text_no_stopwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>First State of the Union Address</td>\n",
       "      <td>i embrace with great satisfaction the opport...</td>\n",
       "      <td>[i, embrace, with, great, satisfaction, the, o...</td>\n",
       "      <td>[embrace, great, satisfaction, opportunity, pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>Second State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, the, h...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1791</td>\n",
       "      <td>Third State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, the, h...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1792</td>\n",
       "      <td>Fourth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, of, th...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1793</td>\n",
       "      <td>Fifth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, of, th...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>2016</td>\n",
       "      <td>Barack Obama's Eighth State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, con...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2017</td>\n",
       "      <td>Donald Trump's First State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, con...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2018</td>\n",
       "      <td>Donald Trump's Second State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, con...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2019</td>\n",
       "      <td>Donald Trump's Third State of the Union Address</td>\n",
       "      <td>madam speaker  mr  vice president  members o...</td>\n",
       "      <td>[madam, speaker, mr, vice, president, member, ...</td>\n",
       "      <td>[madam, speaker, mr, vice, president, member, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>Joe Biden</td>\n",
       "      <td>2022</td>\n",
       "      <td>Joe Biden's First State of The Union Address</td>\n",
       "      <td>madam speaker  madam vice president  our fir...</td>\n",
       "      <td>[madam, speaker, madam, vice, president, our, ...</td>\n",
       "      <td>[madam, speaker, madam, vice, president, first...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>220 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             President  Year  \\\n",
       "0    George Washington  1790   \n",
       "1    George Washington  1790   \n",
       "2    George Washington  1791   \n",
       "3    George Washington  1792   \n",
       "4    George Washington  1793   \n",
       "..                 ...   ...   \n",
       "215       Barack Obama  2016   \n",
       "216       Donald Trump  2017   \n",
       "217       Donald Trump  2018   \n",
       "218       Donald Trump  2019   \n",
       "219          Joe Biden  2022   \n",
       "\n",
       "                                                Title  \\\n",
       "0                    First State of the Union Address   \n",
       "1                   Second State of the Union Address   \n",
       "2                    Third State of the Union Address   \n",
       "3                   Fourth State of the Union Address   \n",
       "4                    Fifth State of the Union Address   \n",
       "..                                                ...   \n",
       "215  Barack Obama's Eighth State of the Union Address   \n",
       "216   Donald Trump's First State of the Union Address   \n",
       "217  Donald Trump's Second State of the Union Address   \n",
       "218   Donald Trump's Third State of the Union Address   \n",
       "219      Joe Biden's First State of The Union Address   \n",
       "\n",
       "                                                  Text  \\\n",
       "0      i embrace with great satisfaction the opport...   \n",
       "1      fellow citizens of the senate and the house ...   \n",
       "2      fellow citizens of the senate and the house ...   \n",
       "3      fellow citizens of the senate and of the hou...   \n",
       "4      fellow citizens of the senate and of the hou...   \n",
       "..                                                 ...   \n",
       "215    mr  speaker  mr  vice president  members of ...   \n",
       "216    mr  speaker  mr  vice president  members of ...   \n",
       "217    mr  speaker  mr  vice president  members of ...   \n",
       "218    madam speaker  mr  vice president  members o...   \n",
       "219    madam speaker  madam vice president  our fir...   \n",
       "\n",
       "                                            Lemma_Text  \\\n",
       "0    [i, embrace, with, great, satisfaction, the, o...   \n",
       "1    [fellow, citizen, of, the, senate, and, the, h...   \n",
       "2    [fellow, citizen, of, the, senate, and, the, h...   \n",
       "3    [fellow, citizen, of, the, senate, and, of, th...   \n",
       "4    [fellow, citizen, of, the, senate, and, of, th...   \n",
       "..                                                 ...   \n",
       "215  [mr, speaker, mr, vice, president, member, of,...   \n",
       "216  [mr, speaker, mr, vice, president, member, of,...   \n",
       "217  [mr, speaker, mr, vice, president, member, of,...   \n",
       "218  [madam, speaker, mr, vice, president, member, ...   \n",
       "219  [madam, speaker, madam, vice, president, our, ...   \n",
       "\n",
       "                                     text_no_stopwords  \n",
       "0    [embrace, great, satisfaction, opportunity, pr...  \n",
       "1    [fellow, citizen, senate, house, representativ...  \n",
       "2    [fellow, citizen, senate, house, representativ...  \n",
       "3    [fellow, citizen, senate, house, representativ...  \n",
       "4    [fellow, citizen, senate, house, representativ...  \n",
       "..                                                 ...  \n",
       "215  [mr, speaker, mr, vice, president, member, con...  \n",
       "216  [mr, speaker, mr, vice, president, member, con...  \n",
       "217  [mr, speaker, mr, vice, president, member, con...  \n",
       "218  [madam, speaker, mr, vice, president, member, ...  \n",
       "219  [madam, speaker, madam, vice, president, first...  \n",
       "\n",
       "[220 rows x 6 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>President</th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "      <th>Lemma_Text</th>\n",
       "      <th>text_no_stopwords</th>\n",
       "      <th>tagged_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>First State of the Union Address</td>\n",
       "      <td>i embrace with great satisfaction the opport...</td>\n",
       "      <td>[i, embrace, with, great, satisfaction, the, o...</td>\n",
       "      <td>[embrace, great, satisfaction, opportunity, pr...</td>\n",
       "      <td>[(embrace, NN), (great, JJ), (satisfaction, NN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>Second State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, the, h...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "      <td>[(fellow, JJ), (citizen, NN), (senate, NN), (h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1791</td>\n",
       "      <td>Third State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, the, h...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "      <td>[(fellow, JJ), (citizen, NN), (senate, NN), (h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1792</td>\n",
       "      <td>Fourth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, of, th...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "      <td>[(fellow, JJ), (citizen, NN), (senate, NN), (h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1793</td>\n",
       "      <td>Fifth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, of, th...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "      <td>[(fellow, JJ), (citizen, NN), (senate, NN), (h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>2016</td>\n",
       "      <td>Barack Obama's Eighth State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, con...</td>\n",
       "      <td>[(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2017</td>\n",
       "      <td>Donald Trump's First State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, con...</td>\n",
       "      <td>[(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2018</td>\n",
       "      <td>Donald Trump's Second State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, con...</td>\n",
       "      <td>[(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2019</td>\n",
       "      <td>Donald Trump's Third State of the Union Address</td>\n",
       "      <td>madam speaker  mr  vice president  members o...</td>\n",
       "      <td>[madam, speaker, mr, vice, president, member, ...</td>\n",
       "      <td>[madam, speaker, mr, vice, president, member, ...</td>\n",
       "      <td>[(madam, NN), (speaker, NN), (mr, JJ), (vice, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>Joe Biden</td>\n",
       "      <td>2022</td>\n",
       "      <td>Joe Biden's First State of The Union Address</td>\n",
       "      <td>madam speaker  madam vice president  our fir...</td>\n",
       "      <td>[madam, speaker, madam, vice, president, our, ...</td>\n",
       "      <td>[madam, speaker, madam, vice, president, first...</td>\n",
       "      <td>[(madam, NN), (speaker, NN), (madam, JJ), (vic...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>220 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             President  Year  \\\n",
       "0    George Washington  1790   \n",
       "1    George Washington  1790   \n",
       "2    George Washington  1791   \n",
       "3    George Washington  1792   \n",
       "4    George Washington  1793   \n",
       "..                 ...   ...   \n",
       "215       Barack Obama  2016   \n",
       "216       Donald Trump  2017   \n",
       "217       Donald Trump  2018   \n",
       "218       Donald Trump  2019   \n",
       "219          Joe Biden  2022   \n",
       "\n",
       "                                                Title  \\\n",
       "0                    First State of the Union Address   \n",
       "1                   Second State of the Union Address   \n",
       "2                    Third State of the Union Address   \n",
       "3                   Fourth State of the Union Address   \n",
       "4                    Fifth State of the Union Address   \n",
       "..                                                ...   \n",
       "215  Barack Obama's Eighth State of the Union Address   \n",
       "216   Donald Trump's First State of the Union Address   \n",
       "217  Donald Trump's Second State of the Union Address   \n",
       "218   Donald Trump's Third State of the Union Address   \n",
       "219      Joe Biden's First State of The Union Address   \n",
       "\n",
       "                                                  Text  \\\n",
       "0      i embrace with great satisfaction the opport...   \n",
       "1      fellow citizens of the senate and the house ...   \n",
       "2      fellow citizens of the senate and the house ...   \n",
       "3      fellow citizens of the senate and of the hou...   \n",
       "4      fellow citizens of the senate and of the hou...   \n",
       "..                                                 ...   \n",
       "215    mr  speaker  mr  vice president  members of ...   \n",
       "216    mr  speaker  mr  vice president  members of ...   \n",
       "217    mr  speaker  mr  vice president  members of ...   \n",
       "218    madam speaker  mr  vice president  members o...   \n",
       "219    madam speaker  madam vice president  our fir...   \n",
       "\n",
       "                                            Lemma_Text  \\\n",
       "0    [i, embrace, with, great, satisfaction, the, o...   \n",
       "1    [fellow, citizen, of, the, senate, and, the, h...   \n",
       "2    [fellow, citizen, of, the, senate, and, the, h...   \n",
       "3    [fellow, citizen, of, the, senate, and, of, th...   \n",
       "4    [fellow, citizen, of, the, senate, and, of, th...   \n",
       "..                                                 ...   \n",
       "215  [mr, speaker, mr, vice, president, member, of,...   \n",
       "216  [mr, speaker, mr, vice, president, member, of,...   \n",
       "217  [mr, speaker, mr, vice, president, member, of,...   \n",
       "218  [madam, speaker, mr, vice, president, member, ...   \n",
       "219  [madam, speaker, madam, vice, president, our, ...   \n",
       "\n",
       "                                     text_no_stopwords  \\\n",
       "0    [embrace, great, satisfaction, opportunity, pr...   \n",
       "1    [fellow, citizen, senate, house, representativ...   \n",
       "2    [fellow, citizen, senate, house, representativ...   \n",
       "3    [fellow, citizen, senate, house, representativ...   \n",
       "4    [fellow, citizen, senate, house, representativ...   \n",
       "..                                                 ...   \n",
       "215  [mr, speaker, mr, vice, president, member, con...   \n",
       "216  [mr, speaker, mr, vice, president, member, con...   \n",
       "217  [mr, speaker, mr, vice, president, member, con...   \n",
       "218  [madam, speaker, mr, vice, president, member, ...   \n",
       "219  [madam, speaker, madam, vice, president, first...   \n",
       "\n",
       "                                           tagged_text  \n",
       "0    [(embrace, NN), (great, JJ), (satisfaction, NN...  \n",
       "1    [(fellow, JJ), (citizen, NN), (senate, NN), (h...  \n",
       "2    [(fellow, JJ), (citizen, NN), (senate, NN), (h...  \n",
       "3    [(fellow, JJ), (citizen, NN), (senate, NN), (h...  \n",
       "4    [(fellow, JJ), (citizen, NN), (senate, NN), (h...  \n",
       "..                                                 ...  \n",
       "215  [(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...  \n",
       "216  [(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...  \n",
       "217  [(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...  \n",
       "218  [(madam, NN), (speaker, NN), (mr, JJ), (vice, ...  \n",
       "219  [(madam, NN), (speaker, NN), (madam, JJ), (vic...  \n",
       "\n",
       "[220 rows x 7 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['tagged_text'] = df.text_no_stopwords.apply(lambda x: pos_tag(x))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>President</th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "      <th>Lemma_Text</th>\n",
       "      <th>text_no_stopwords</th>\n",
       "      <th>tagged_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>First State of the Union Address</td>\n",
       "      <td>i embrace with great satisfaction the opport...</td>\n",
       "      <td>[i, embrace, with, great, satisfaction, the, o...</td>\n",
       "      <td>[embrace, great, satisfaction, opportunity, pr...</td>\n",
       "      <td>[(embrace, NN), (great, JJ), (satisfaction, NN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1790</td>\n",
       "      <td>Second State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, the, h...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "      <td>[(fellow, JJ), (citizen, NN), (senate, NN), (h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1791</td>\n",
       "      <td>Third State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and the house ...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, the, h...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "      <td>[(fellow, JJ), (citizen, NN), (senate, NN), (h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1792</td>\n",
       "      <td>Fourth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, of, th...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "      <td>[(fellow, JJ), (citizen, NN), (senate, NN), (h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>1793</td>\n",
       "      <td>Fifth State of the Union Address</td>\n",
       "      <td>fellow citizens of the senate and of the hou...</td>\n",
       "      <td>[fellow, citizen, of, the, senate, and, of, th...</td>\n",
       "      <td>[fellow, citizen, senate, house, representativ...</td>\n",
       "      <td>[(fellow, JJ), (citizen, NN), (senate, NN), (h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>2016</td>\n",
       "      <td>Barack Obama's Eighth State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, con...</td>\n",
       "      <td>[(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2017</td>\n",
       "      <td>Donald Trump's First State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, con...</td>\n",
       "      <td>[(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2018</td>\n",
       "      <td>Donald Trump's Second State of the Union Address</td>\n",
       "      <td>mr  speaker  mr  vice president  members of ...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, of,...</td>\n",
       "      <td>[mr, speaker, mr, vice, president, member, con...</td>\n",
       "      <td>[(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>Donald Trump</td>\n",
       "      <td>2019</td>\n",
       "      <td>Donald Trump's Third State of the Union Address</td>\n",
       "      <td>madam speaker  mr  vice president  members o...</td>\n",
       "      <td>[madam, speaker, mr, vice, president, member, ...</td>\n",
       "      <td>[madam, speaker, mr, vice, president, member, ...</td>\n",
       "      <td>[(madam, NN), (speaker, NN), (mr, JJ), (vice, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>Joe Biden</td>\n",
       "      <td>2022</td>\n",
       "      <td>Joe Biden's First State of The Union Address</td>\n",
       "      <td>madam speaker  madam vice president  our fir...</td>\n",
       "      <td>[madam, speaker, madam, vice, president, our, ...</td>\n",
       "      <td>[madam, speaker, madam, vice, president, first...</td>\n",
       "      <td>[(madam, NN), (speaker, NN), (madam, JJ), (vic...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>220 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             President  Year  \\\n",
       "0    George Washington  1790   \n",
       "1    George Washington  1790   \n",
       "2    George Washington  1791   \n",
       "3    George Washington  1792   \n",
       "4    George Washington  1793   \n",
       "..                 ...   ...   \n",
       "215       Barack Obama  2016   \n",
       "216       Donald Trump  2017   \n",
       "217       Donald Trump  2018   \n",
       "218       Donald Trump  2019   \n",
       "219          Joe Biden  2022   \n",
       "\n",
       "                                                Title  \\\n",
       "0                    First State of the Union Address   \n",
       "1                   Second State of the Union Address   \n",
       "2                    Third State of the Union Address   \n",
       "3                   Fourth State of the Union Address   \n",
       "4                    Fifth State of the Union Address   \n",
       "..                                                ...   \n",
       "215  Barack Obama's Eighth State of the Union Address   \n",
       "216   Donald Trump's First State of the Union Address   \n",
       "217  Donald Trump's Second State of the Union Address   \n",
       "218   Donald Trump's Third State of the Union Address   \n",
       "219      Joe Biden's First State of The Union Address   \n",
       "\n",
       "                                                  Text  \\\n",
       "0      i embrace with great satisfaction the opport...   \n",
       "1      fellow citizens of the senate and the house ...   \n",
       "2      fellow citizens of the senate and the house ...   \n",
       "3      fellow citizens of the senate and of the hou...   \n",
       "4      fellow citizens of the senate and of the hou...   \n",
       "..                                                 ...   \n",
       "215    mr  speaker  mr  vice president  members of ...   \n",
       "216    mr  speaker  mr  vice president  members of ...   \n",
       "217    mr  speaker  mr  vice president  members of ...   \n",
       "218    madam speaker  mr  vice president  members o...   \n",
       "219    madam speaker  madam vice president  our fir...   \n",
       "\n",
       "                                            Lemma_Text  \\\n",
       "0    [i, embrace, with, great, satisfaction, the, o...   \n",
       "1    [fellow, citizen, of, the, senate, and, the, h...   \n",
       "2    [fellow, citizen, of, the, senate, and, the, h...   \n",
       "3    [fellow, citizen, of, the, senate, and, of, th...   \n",
       "4    [fellow, citizen, of, the, senate, and, of, th...   \n",
       "..                                                 ...   \n",
       "215  [mr, speaker, mr, vice, president, member, of,...   \n",
       "216  [mr, speaker, mr, vice, president, member, of,...   \n",
       "217  [mr, speaker, mr, vice, president, member, of,...   \n",
       "218  [madam, speaker, mr, vice, president, member, ...   \n",
       "219  [madam, speaker, madam, vice, president, our, ...   \n",
       "\n",
       "                                     text_no_stopwords  \\\n",
       "0    [embrace, great, satisfaction, opportunity, pr...   \n",
       "1    [fellow, citizen, senate, house, representativ...   \n",
       "2    [fellow, citizen, senate, house, representativ...   \n",
       "3    [fellow, citizen, senate, house, representativ...   \n",
       "4    [fellow, citizen, senate, house, representativ...   \n",
       "..                                                 ...   \n",
       "215  [mr, speaker, mr, vice, president, member, con...   \n",
       "216  [mr, speaker, mr, vice, president, member, con...   \n",
       "217  [mr, speaker, mr, vice, president, member, con...   \n",
       "218  [madam, speaker, mr, vice, president, member, ...   \n",
       "219  [madam, speaker, madam, vice, president, first...   \n",
       "\n",
       "                                           tagged_text  \n",
       "0    [(embrace, NN), (great, JJ), (satisfaction, NN...  \n",
       "1    [(fellow, JJ), (citizen, NN), (senate, NN), (h...  \n",
       "2    [(fellow, JJ), (citizen, NN), (senate, NN), (h...  \n",
       "3    [(fellow, JJ), (citizen, NN), (senate, NN), (h...  \n",
       "4    [(fellow, JJ), (citizen, NN), (senate, NN), (h...  \n",
       "..                                                 ...  \n",
       "215  [(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...  \n",
       "216  [(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...  \n",
       "217  [(mr, NN), (speaker, NN), (mr, JJ), (vice, NN)...  \n",
       "218  [(madam, NN), (speaker, NN), (mr, JJ), (vice, ...  \n",
       "219  [(madam, NN), (speaker, NN), (madam, JJ), (vic...  \n",
       "\n",
       "[220 rows x 7 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "docs = nlp.pipe(df.Text)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_clean = [[w.lemma_.lower() for w in doc if (not w.is_stop and not w.is_punct and not w.like_num)] for doc in docs]\n",
    "df['docs_clean'] = docs_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'   embrace great satisfaction opportunity   present   congratulate present favourable prospect public affair   recent accession important state north carolina constitution united states   official information receive rise credit respectability country general increase good government union concord   peace plenty   bless   circumstance   auspicious eminent degree national prosperity      resume consultation general good   derive encouragement reflection measure session satisfactory constituent   novelty difficulty work allow hope       expectation   secure blessing gracious providence place reach   course present important session   cool deliberate exertion patriotism   firmness wisdom      interesting object   engage attention   provide common defence merit particular regard prepare war effectual mean preserve peace     free people ought armed   discipline end uniform digest plan requisite   safety interest require promote manufactory   tend render independent   essential   particularly military supply       proper establishment troop deem indispensible   entitle mature consideration arrangement   respect       conciliate comfortable support officer soldier regard oeconomy       reason hope   pacific measure adopt regard certain hostile tribe indians relieve inhabitant southern western frontier depredation percieve   information contain paper   shall direct lay   comprehend communication commonwealth virginia   ought prepared afford protection part union     necessary punish aggressor      interest united states require   intercourse nation facilitate   provision enable fulfill duty respect   manner circumstance render conducive public good   end compensation person employ   accord nature appointment define law   competent fund      defray expense incident conduct foreign affair       consideration render expedient   term foreigner admit right citizen speedily ascertain uniform rule naturalization      uniformity currency   weight measure united states object great importance   persuade duly attend      advancement agriculture   commerce   manufacture proper mean   trust   need recommendation   forbear intimate expediency give effectual encouragement introduction new useful invention abroad   exertion skill genius produce home   facilitate intercourse distant part country attention post office post road           persuade   agree opinion     well deserve patronage   promotion science literature   knowledge country sure basis public happiness   measure government receive impression immediately sense community   proportionably essential   security free constitution contribute way   convince   entrust public administration   valuable end government well answer enlighten confidence people   teach people know value right   discern provide invasion   distinguish oppression necessary exercise lawful authority   burthen proceed disregard convenience result inevitable exigency society   discriminate spirit liberty licentiousness   cherish   avoid   unite speedy   temperate vigilence encroachment   inviolable respect law      desirable object well promote affording aid seminary learn establish   institution     national university   expedient worthy place deliberation legislature      see peculiar pleasure   close session   resolution enter expressive opinion   adequate provision support public credit matter high importance national honor prosperity   sentiment   entirely concur   perfect confidence good endeavor devise provision   truly consistent end   add equal reliance chearful co operation branch legislature   superfluous specify inducement measure character permanent interest united states obviously deeply concerned   receive explicit sanction declaration      direct proper officer lay respectively paper estimate regard affair   particularly recommend consideration necessary convey information state union   duty afford      welfare country great object care effort ought direct   shall derive great satisfaction co operation   pleasing tho arduous task ensure fellow citizen blessing right expect free   efficient equal government   '"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs_list_clean = [' '.join(doc) for doc in docs_clean]\n",
    "docs_list_clean[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aa</th>\n",
       "      <th>aaa</th>\n",
       "      <th>aana</th>\n",
       "      <th>aar</th>\n",
       "      <th>aaron</th>\n",
       "      <th>abandon</th>\n",
       "      <th>abandoning</th>\n",
       "      <th>abandonment</th>\n",
       "      <th>abate</th>\n",
       "      <th>abatement</th>\n",
       "      <th>...</th>\n",
       "      <th>zimbabwe</th>\n",
       "      <th>zimbabwean</th>\n",
       "      <th>zinc</th>\n",
       "      <th>zion</th>\n",
       "      <th>zollverein</th>\n",
       "      <th>zone</th>\n",
       "      <th>zoological</th>\n",
       "      <th>zuloaga</th>\n",
       "      <th>ôtil</th>\n",
       "      <th>ѕў</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 18613 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   aa  aaa  aana  aar  aaron  abandon  abandoning  abandonment  abate  \\\n",
       "0   0    0     0    0      0        0           0            0      0   \n",
       "1   0    0     0    0      0        0           0            0      0   \n",
       "2   0    0     0    0      0        0           0            0      0   \n",
       "3   0    0     0    0      0        0           0            0      0   \n",
       "4   0    0     0    0      0        0           0            0      0   \n",
       "\n",
       "   abatement  ...  zimbabwe  zimbabwean  zinc  zion  zollverein  zone  \\\n",
       "0          0  ...         0           0     0     0           0     0   \n",
       "1          0  ...         0           0     0     0           0     0   \n",
       "2          0  ...         0           0     0     0           0     0   \n",
       "3          1  ...         0           0     0     0           0     0   \n",
       "4          0  ...         0           0     0     0           0     0   \n",
       "\n",
       "   zoological  zuloaga  ôtil  ѕў  \n",
       "0           0        0     0   0  \n",
       "1           0        0     0   0  \n",
       "2           0        0     0   0  \n",
       "3           0        0     0   0  \n",
       "4           0        0     0   0  \n",
       "\n",
       "[5 rows x 18613 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "\n",
    "count_vec = CountVectorizer(stop_words='english')\n",
    "tfidf_vec = TfidfVectorizer(stop_words='english')\n",
    "X = count_vec.fit_transform(docs_list_clean)\n",
    "\n",
    "df_X = pd.DataFrame(X.toarray(), columns=count_vec.get_feature_names_out())\n",
    "df_X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import TruncatedSVD, NMF\n",
    "\n",
    "num_topics = 10\n",
    "topics = TruncatedSVD(num_topics)\n",
    "#topics = NMF(num_topics)\n",
    "doc_topic = topics.fit_transform(X)\n",
    "#topics.explained_variance_ratio_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aa</th>\n",
       "      <th>aaa</th>\n",
       "      <th>aana</th>\n",
       "      <th>aar</th>\n",
       "      <th>aaron</th>\n",
       "      <th>abandon</th>\n",
       "      <th>abandoning</th>\n",
       "      <th>abandonment</th>\n",
       "      <th>abate</th>\n",
       "      <th>abatement</th>\n",
       "      <th>...</th>\n",
       "      <th>zimbabwe</th>\n",
       "      <th>zimbabwean</th>\n",
       "      <th>zinc</th>\n",
       "      <th>zion</th>\n",
       "      <th>zollverein</th>\n",
       "      <th>zone</th>\n",
       "      <th>zoological</th>\n",
       "      <th>zuloaga</th>\n",
       "      <th>ôtil</th>\n",
       "      <th>ѕў</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.003</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.001</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.001</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.001</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>-0.008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.001</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>-0.003</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.002</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.002</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.005</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002</td>\n",
       "      <td>0.001</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.001</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.005</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.009</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.002</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.001</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.006</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.003</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.002</td>\n",
       "      <td>0.002</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>-0.004</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.006</td>\n",
       "      <td>0.002</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.002</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 18613 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    aa    aaa   aana  aar  aaron  abandon  abandoning  abandonment  abate  \\\n",
       "0  0.0  0.000  0.000  0.0  0.000    0.007         0.0        0.001  0.000   \n",
       "1 -0.0  0.000 -0.000  0.0  0.000   -0.003         0.0        0.000  0.000   \n",
       "2  0.0  0.001  0.000 -0.0 -0.000    0.001        -0.0        0.001  0.001   \n",
       "3 -0.0  0.001  0.000  0.0 -0.001   -0.008         0.0        0.001 -0.000   \n",
       "4  0.0  0.001 -0.000  0.0 -0.000   -0.003         0.0       -0.002 -0.001   \n",
       "5  0.0 -0.001 -0.000  0.0 -0.001    0.005         0.0        0.002  0.001   \n",
       "6  0.0  0.000 -0.000 -0.0 -0.000    0.009         0.0       -0.001  0.000   \n",
       "7  0.0 -0.000 -0.000  0.0  0.000    0.007         0.0       -0.000 -0.001   \n",
       "8 -0.0  0.001 -0.001 -0.0  0.000   -0.006        -0.0       -0.003 -0.001   \n",
       "9  0.0 -0.000 -0.000  0.0 -0.001   -0.004        -0.0       -0.006  0.002   \n",
       "\n",
       "   abatement  ...  zimbabwe  zimbabwean  zinc  zion  zollverein   zone  \\\n",
       "0        0.0  ...     0.000       0.000   0.0   0.0         0.0  0.003   \n",
       "1       -0.0  ...     0.001       0.001  -0.0   0.0        -0.0  0.001   \n",
       "2        0.0  ...     0.001       0.001   0.0  -0.0         0.0 -0.001   \n",
       "3       -0.0  ...    -0.000       0.000  -0.0  -0.0        -0.0  0.007   \n",
       "4       -0.0  ...    -0.002      -0.001   0.0  -0.0         0.0 -0.005   \n",
       "5       -0.0  ...     0.003       0.001  -0.0  -0.0         0.0 -0.005   \n",
       "6       -0.0  ...    -0.000      -0.000   0.0  -0.0         0.0  0.009   \n",
       "7       -0.0  ...     0.000       0.000  -0.0   0.0         0.0  0.007   \n",
       "8       -0.0  ...     0.000       0.000  -0.0   0.0        -0.0  0.015   \n",
       "9        0.0  ...    -0.002      -0.001   0.0  -0.0         0.0  0.011   \n",
       "\n",
       "   zoological  zuloaga  ôtil   ѕў  \n",
       "0       0.000    0.000   0.0  0.0  \n",
       "1      -0.000   -0.000  -0.0  0.0  \n",
       "2      -0.000    0.000  -0.0  0.0  \n",
       "3       0.001   -0.001  -0.0  0.0  \n",
       "4      -0.001    0.001   0.0  0.0  \n",
       "5      -0.000    0.000   0.0  0.0  \n",
       "6       0.000   -0.001  -0.0 -0.0  \n",
       "7       0.002   -0.000   0.0 -0.0  \n",
       "8       0.002    0.002  -0.0  0.0  \n",
       "9       0.001    0.000  -0.0  0.0  \n",
       "\n",
       "[10 rows x 18613 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_word = pd.DataFrame(topics.components_.round(3),\n",
    "             columns = count_vec.get_feature_names_out())\n",
    "topic_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_topics(model, feature_names, no_top_words, topic_names=None):\n",
    "    for ix, topic in enumerate(model.components_):\n",
    "        if not topic_names or not topic_names[ix]:\n",
    "            print(\"\\nTopic \", ix)\n",
    "        else:\n",
    "            print(\"\\nTopic: '\",topic_names[ix],\"'\")\n",
    "        print(\", \".join([feature_names[i]\n",
    "                        for i in topic.argsort()[:-no_top_words - 1:-1]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Topic  0\n",
      "government, year, congress, united, states, country, state, great, law, people\n",
      "\n",
      "Topic  1\n",
      "program, year, world, new, work, need, help, america, nation, federal\n",
      "\n",
      "Topic  2\n",
      "program, dollar, year, fiscal, united, war, expenditure, policy, administration, states\n",
      "\n",
      "Topic  3\n",
      "man, law, court, service, business, department, dollar, legislation, national, need\n",
      "\n",
      "Topic  4\n",
      "war, dollar, man, expenditure, power, people, great, peace, public, state\n",
      "\n",
      "Topic  5\n",
      "nation, administration, state, policy, man, energy, effort, continue, program, power\n",
      "\n",
      "Topic  6\n",
      "mexico, united, war, states, american, texas, mexican, man, peace, army\n",
      "\n",
      "Topic  7\n",
      "mexico, country, nof, nthe, texas, mexican, nto, nand, army, public\n",
      "\n",
      "Topic  8\n",
      "state, dollar, constitution, american, government, department, program, business, canal, united\n",
      "\n",
      "Topic  9\n",
      "world, government, nof, nthe, american, free, shall, nand, nto, great\n"
     ]
    }
   ],
   "source": [
    "display_topics(topics, count_vec.get_feature_names_out(), 10)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "eeda09e8682780dc1dbad7711462886786ede58877e15e2f55a2f4f20b8e3e86"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('metis')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}