03_preds.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Libraries\n",
    "import re\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import datetime as dt\n",
    "from tensorflow.keras import models, preprocessing #, layers, callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all .csv files from path\n",
    "# files with tweets collected with Twitter API\n",
    "path = 'data/clean_tweets/'\n",
    "files_path = path + '*.csv'\n",
    "files = glob.glob(files_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data/clean_tweets\\\\clean_tweets_2021-11-21.csv',\n",
       " 'data/clean_tweets\\\\clean_tweets_2021-11-22.csv',\n",
       " 'data/clean_tweets\\\\clean_tweets_2021-11-23.csv',\n",
       " 'data/clean_tweets\\\\clean_tweets_2021-11-24.csv',\n",
       " 'data/clean_tweets\\\\clean_tweets_2021-11-25.csv',\n",
       " 'data/clean_tweets\\\\clean_tweets_2021-11-26.csv',\n",
       " 'data/clean_tweets\\\\clean_tweets_2021-11-27.csv',\n",
       " 'data/clean_tweets\\\\clean_tweets_2021-11-28.csv',\n",
       " 'data/clean_tweets\\\\clean_tweets_2021-11-29.csv',\n",
       " 'data/clean_tweets\\\\clean_tweets_2021-11-30.csv',\n",
       " 'data/clean_tweets\\\\clean_tweets_2021-12-01.csv']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Get current date\n",
    "today = dt.date.today()\n",
    "today = today.strftime(\"%Y-%m-%d\")\n",
    "today"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "file_t = [i for i in files if re.search(today, i)]\n",
    "file_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8830, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>lang</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>possibly_sensitive</th>\n",
       "      <th>type</th>\n",
       "      <th>clean_tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-11-21 20:17:03+00:00</td>\n",
       "      <td>Symptomatic COVID-19 testing should be free an...</td>\n",
       "      <td>en</td>\n",
       "      <td>1466139300404375576</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['free', 'covid', 'drug', 'safe', 'ontarians',...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-11-21 20:17:03+00:00</td>\n",
       "      <td>https://t.co/e4JoW7aQGx\\nVaccines failed to st...</td>\n",
       "      <td>en</td>\n",
       "      <td>1466139299993301003</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['people', '4', '60+', '89', '3', '16', 'covid...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  created_at  \\\n",
       "0  2021-11-21 20:17:03+00:00   \n",
       "1  2021-11-21 20:17:03+00:00   \n",
       "\n",
       "                                                text lang  \\\n",
       "0  Symptomatic COVID-19 testing should be free an...   en   \n",
       "1  https://t.co/e4JoW7aQGx\\nVaccines failed to st...   en   \n",
       "\n",
       "              tweet_id  possibly_sensitive type  \\\n",
       "0  1466139300404375576               False  NaN   \n",
       "1  1466139299993301003               False  NaN   \n",
       "\n",
       "                                         clean_tweet  \n",
       "0  ['free', 'covid', 'drug', 'safe', 'ontarians',...  \n",
       "1  ['people', '4', '60+', '89', '3', '16', 'covid...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# empty dataframe\n",
    "tweets_api = pd.DataFrame()\n",
    "\n",
    "# loop over files\n",
    "for file in files:\n",
    "    \n",
    "    # store file in a temporary dataframe\n",
    "    df = pd.read_csv(file)\n",
    "    \n",
    "    # append df into main dataframe\n",
    "    tweets_api = tweets_api.append(df)\n",
    "    \n",
    "# from int to str\n",
    "tweets_api[\"tweet_id\"] = tweets_api[\"tweet_id\"].astype(str)\n",
    "\n",
    "# reset index\n",
    "tweets_api = tweets_api.reset_index(drop = True)\n",
    "print(tweets_api.shape)\n",
    "tweets_api.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>lang</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>possibly_sensitive</th>\n",
       "      <th>type</th>\n",
       "      <th>clean_tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-11-21 20:17:03+00:00</td>\n",
       "      <td>Symptomatic COVID-19 testing should be free an...</td>\n",
       "      <td>en</td>\n",
       "      <td>1466139300404375576</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[free,  covid,  drug,  safe,  ontarians,  19, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-11-21 20:17:03+00:00</td>\n",
       "      <td>https://t.co/e4JoW7aQGx\\nVaccines failed to st...</td>\n",
       "      <td>en</td>\n",
       "      <td>1466139299993301003</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[people,  4,  60+,  89,  3,  16,  covid,  7,  ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  created_at  \\\n",
       "0  2021-11-21 20:17:03+00:00   \n",
       "1  2021-11-21 20:17:03+00:00   \n",
       "\n",
       "                                                text lang  \\\n",
       "0  Symptomatic COVID-19 testing should be free an...   en   \n",
       "1  https://t.co/e4JoW7aQGx\\nVaccines failed to st...   en   \n",
       "\n",
       "              tweet_id  possibly_sensitive type  \\\n",
       "0  1466139300404375576               False  NaN   \n",
       "1  1466139299993301003               False  NaN   \n",
       "\n",
       "                                         clean_tweet  \n",
       "0  [free,  covid,  drug,  safe,  ontarians,  19, ...  \n",
       "1  [people,  4,  60+,  89,  3,  16,  covid,  7,  ...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Clean column\n",
    "# remove [], commas and \"\".\n",
    "tweets_api[\"clean_tweet\"] = tweets_api.clean_tweet.apply(lambda x: x.lstrip(\"[\"))\n",
    "tweets_api[\"clean_tweet\"] = tweets_api.clean_tweet.apply(lambda x: x.rstrip(\"]\"))\n",
    "tweets_api[\"clean_tweet\"] = tweets_api.clean_tweet.apply(lambda x: x.split(\",\"))\n",
    "tweets_api[\"clean_tweet\"] = tweets_api.clean_tweet.apply(lambda x: [s.replace(\"'\",\"\") for s in x])\n",
    "tweets_api[\"clean_tweet\"] = tweets_api.clean_tweet.apply(lambda x: [s.replace('\"',\"\") for s in x])\n",
    "tweets_api.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create class object.\n",
    "tokenizer = preprocessing.text.Tokenizer(num_words = 500000, filters='!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~', \n",
    "                                               lower = True)\n",
    "\n",
    "# Updates internal vocabulary based on a list of texts.\n",
    "# In our case, since texts contains lists, it is assumed each entry of the lists to be a token.\n",
    "tokenizer.fit_on_texts(tweets_api['clean_tweet'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tokenizer.texts_to_sequences(tweets_api[\"clean_tweet\"].values)\n",
    "X = preprocessing.sequence.pad_sequences(X, maxlen = 392)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load model that was previously trained\n",
    "keras_model = models.load_model(\"trained_model/tf_keras_model.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding (Embedding)       (None, 392, 10)           5000000   \n",
      "                                                                 \n",
      " spatial_dropout1d (SpatialD  (None, 392, 10)          0         \n",
      " ropout1D)                                                       \n",
      "                                                                 \n",
      " lstm (LSTM)                 (None, 100)               44400     \n",
      "                                                                 \n",
      " dense (Dense)               (None, 1)                 101       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 5,044,501\n",
      "Trainable params: 5,044,501\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "keras_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predict\n",
    "y_pred_scores = keras_model.predict(X)\n",
    "y_pred = np.round(y_pred_scores).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8830, 9)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>lang</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>possibly_sensitive</th>\n",
       "      <th>type</th>\n",
       "      <th>clean_tweet</th>\n",
       "      <th>toxic</th>\n",
       "      <th>pred_scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2021-11-21 20:17:03+00:00</td>\n",
       "      <td>Symptomatic COVID-19 testing should be free an...</td>\n",
       "      <td>en</td>\n",
       "      <td>1466139300404375576</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[free,  covid,  drug,  safe,  ontarians,  19, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.201645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-11-21 20:17:03+00:00</td>\n",
       "      <td>https://t.co/e4JoW7aQGx\\nVaccines failed to st...</td>\n",
       "      <td>en</td>\n",
       "      <td>1466139299993301003</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[people,  4,  60+,  89,  3,  16,  covid,  7,  ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.006692</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  created_at  \\\n",
       "0  2021-11-21 20:17:03+00:00   \n",
       "1  2021-11-21 20:17:03+00:00   \n",
       "\n",
       "                                                text lang  \\\n",
       "0  Symptomatic COVID-19 testing should be free an...   en   \n",
       "1  https://t.co/e4JoW7aQGx\\nVaccines failed to st...   en   \n",
       "\n",
       "              tweet_id  possibly_sensitive type  \\\n",
       "0  1466139300404375576               False  NaN   \n",
       "1  1466139299993301003               False  NaN   \n",
       "\n",
       "                                         clean_tweet  toxic  pred_scores  \n",
       "0  [free,  covid,  drug,  safe,  ontarians,  19, ...      0     0.201645  \n",
       "1  [people,  4,  60+,  89,  3,  16,  covid,  7,  ...      0     0.006692  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create columns with the result from the model\n",
    "tweets_api[\"toxic\"] = y_pred\n",
    "tweets_api[\"pred_scores\"] = y_pred_scores\n",
    "print(tweets_api.shape)\n",
    "tweets_api.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>lang</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>possibly_sensitive</th>\n",
       "      <th>type</th>\n",
       "      <th>clean_tweet</th>\n",
       "      <th>toxic</th>\n",
       "      <th>pred_scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-11-21 20:17:03+00:00</td>\n",
       "      <td>An Israeli doctor says he believes he caught t...</td>\n",
       "      <td>en</td>\n",
       "      <td>1466139299263533060</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[people,  say,  belief,  doctor,  london,  var...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.895560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2021-11-21 20:17:03+00:00</td>\n",
       "      <td>meu deus eu preciso fazer processos de emenda ...</td>\n",
       "      <td>pt</td>\n",
       "      <td>1466139299137695747</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[de,  emenda,  meu,  pra,  e,  fazer,  prefeit...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2021-11-21 20:17:02+00:00</td>\n",
       "      <td>Omarion Covid strand just hit the US. First ca...</td>\n",
       "      <td>en</td>\n",
       "      <td>1466139298239901701</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[hit,  first,  case,  omarion,  cali,  covid, ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.673810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2021-11-21 20:17:02+00:00</td>\n",
       "      <td>Camas mal ocupadas, espero que pronto queden l...</td>\n",
       "      <td>es</td>\n",
       "      <td>1466139297569021961</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[camas,  queden,  que,  madrid,  una,  miembro...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.630690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2021-11-21 20:17:01+00:00</td>\n",
       "      <td>@thearsenalhelen @jazz_sian COVID itself has a...</td>\n",
       "      <td>en</td>\n",
       "      <td>1466139291013160963</td>\n",
       "      <td>False</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>[cardiac,  likelihood,  treatment,  sian,  rec...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.931584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8798</th>\n",
       "      <td>2021-12-01 06:16:43+00:00</td>\n",
       "      <td>◤全球大流行◢在抗议现场，还有理发师为未接种疫苗的抗议者义剪……\\nhttps://t.co...</td>\n",
       "      <td>zh</td>\n",
       "      <td>1465927824208891905</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Xin,  Zhou,  Yi,  Fei,  Guan,  Zhong,  Zai,  ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.943636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8808</th>\n",
       "      <td>2021-12-01 06:16:38+00:00</td>\n",
       "      <td>La justice namuroise constate l'illégalité du ...</td>\n",
       "      <td>fr</td>\n",
       "      <td>1465927800959946753</td>\n",
       "      <td>False</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>[la,  lillegalite,  en,  appel,  ticket,  covi...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.887801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8810</th>\n",
       "      <td>2021-12-01 06:16:37+00:00</td>\n",
       "      <td>Schweizer Spitäler müssen dringliche Krebsoper...</td>\n",
       "      <td>de</td>\n",
       "      <td>1465927797893963778</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[patienten,  ein,  !,  wovon,  leute,  ungeimp...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.976746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8811</th>\n",
       "      <td>2021-12-01 06:16:37+00:00</td>\n",
       "      <td>@HealthMinistryM Our children are wanting to c...</td>\n",
       "      <td>en</td>\n",
       "      <td>1465927797466025986</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[child,  take,  wanting,  please,  restricted,...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.903671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8812</th>\n",
       "      <td>2021-12-01 06:16:36+00:00</td>\n",
       "      <td>Anti-vaccine Christian broadcaster Marcus Lamb...</td>\n",
       "      <td>en</td>\n",
       "      <td>1465927796518227968</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[anti,  dy,  broadcaster,  lamb,  marcus,  cov...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.616503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2075 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     created_at  \\\n",
       "3     2021-11-21 20:17:03+00:00   \n",
       "6     2021-11-21 20:17:03+00:00   \n",
       "9     2021-11-21 20:17:02+00:00   \n",
       "11    2021-11-21 20:17:02+00:00   \n",
       "23    2021-11-21 20:17:01+00:00   \n",
       "...                         ...   \n",
       "8798  2021-12-01 06:16:43+00:00   \n",
       "8808  2021-12-01 06:16:38+00:00   \n",
       "8810  2021-12-01 06:16:37+00:00   \n",
       "8811  2021-12-01 06:16:37+00:00   \n",
       "8812  2021-12-01 06:16:36+00:00   \n",
       "\n",
       "                                                   text lang  \\\n",
       "3     An Israeli doctor says he believes he caught t...   en   \n",
       "6     meu deus eu preciso fazer processos de emenda ...   pt   \n",
       "9     Omarion Covid strand just hit the US. First ca...   en   \n",
       "11    Camas mal ocupadas, espero que pronto queden l...   es   \n",
       "23    @thearsenalhelen @jazz_sian COVID itself has a...   en   \n",
       "...                                                 ...  ...   \n",
       "8798  ◤全球大流行◢在抗议现场，还有理发师为未接种疫苗的抗议者义剪……\\nhttps://t.co...   zh   \n",
       "8808  La justice namuroise constate l'illégalité du ...   fr   \n",
       "8810  Schweizer Spitäler müssen dringliche Krebsoper...   de   \n",
       "8811  @HealthMinistryM Our children are wanting to c...   en   \n",
       "8812  Anti-vaccine Christian broadcaster Marcus Lamb...   en   \n",
       "\n",
       "                 tweet_id  possibly_sensitive        type  \\\n",
       "3     1466139299263533060               False         NaN   \n",
       "6     1466139299137695747               False         NaN   \n",
       "9     1466139298239901701               False         NaN   \n",
       "11    1466139297569021961               False         NaN   \n",
       "23    1466139291013160963               False  replied_to   \n",
       "...                   ...                 ...         ...   \n",
       "8798  1465927824208891905               False         NaN   \n",
       "8808  1465927800959946753               False  replied_to   \n",
       "8810  1465927797893963778               False         NaN   \n",
       "8811  1465927797466025986               False         NaN   \n",
       "8812  1465927796518227968               False         NaN   \n",
       "\n",
       "                                            clean_tweet  toxic  pred_scores  \n",
       "3     [people,  say,  belief,  doctor,  london,  var...      1     0.895560  \n",
       "6     [de,  emenda,  meu,  pra,  e,  fazer,  prefeit...      1     0.998193  \n",
       "9     [hit,  first,  case,  omarion,  cali,  covid, ...      1     0.673810  \n",
       "11    [camas,  queden,  que,  madrid,  una,  miembro...      1     0.630690  \n",
       "23    [cardiac,  likelihood,  treatment,  sian,  rec...      1     0.931584  \n",
       "...                                                 ...    ...          ...  \n",
       "8798  [Xin,  Zhou,  Yi,  Fei,  Guan,  Zhong,  Zai,  ...      1     0.943636  \n",
       "8808  [la,  lillegalite,  en,  appel,  ticket,  covi...      1     0.887801  \n",
       "8810  [patienten,  ein,  !,  wovon,  leute,  ungeimp...      1     0.976746  \n",
       "8811  [child,  take,  wanting,  please,  restricted,...      1     0.903671  \n",
       "8812  [anti,  dy,  broadcaster,  lamb,  marcus,  cov...      1     0.616503  \n",
       "\n",
       "[2075 rows x 9 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find toxic tweets\n",
    "tweets_api[tweets_api[\"toxic\"] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "@dgodin1234 You're free to get and spread covid and get shot in gun violence...🤷‍♀️\n"
     ]
    }
   ],
   "source": [
    "# Seems toxic\n",
    "print(tweets_api.iloc[8060].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Covid documentaries and movies will come, and I hate that xD\n"
     ]
    }
   ],
   "source": [
    "print(tweets_api.iloc[8066].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_api.to_csv(\"trained_model/preds/tweets_preds.csv\", index = False)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}