05_dashboard.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import dash\n",
    "import dash_core_components as dcc\n",
    "import dash_html_components as html\n",
    "import dash_bootstrap_components as dbc\n",
    "import pandas as pd\n",
    "from tensorflow.keras import models, preprocessing #, layers, callbacks\n",
    "import plotly.express as px\n",
    "from geopy.geocoders import Nominatim\n",
    "import plotly.graph_objects as go\n",
    "import numpy as np\n",
    "from dash import dcc, html, Input, Output, State\n",
    "# Nuevos tweets\n",
    "import re\n",
    "import os\n",
    "import nltk\n",
    "import requests\n",
    "import unidecode\n",
    "import unicodedata\n",
    "import contractions\n",
    "from dotenv import load_dotenv\n",
    "from collections import Counter\n",
    "from nltk.corpus import stopwords\n",
    "import collecting_tweets\n",
    "from geopy.geocoders import Nominatim\n",
    "from datetime import date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to get city coordinates\n",
    "geolocator = Nominatim(user_agent = 'bmartin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prueba para nuevos tweets\n",
    "load_dotenv('data/envs/kafka.env', override = True)\n",
    "\n",
    "# getting twitter credentials\n",
    "twitter_key = os.environ.get('api_key')\n",
    "twitter_secret_key = os.environ.get('secret_key')\n",
    "bearer_token = os.environ.get('bearer_token')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PreProcessor:\n",
    "    \n",
    "    def __init__(self, regex_dict = None):\n",
    "        \n",
    "        # creating classes\n",
    "        # stem\n",
    "        self.sb = nltk.stem.SnowballStemmer('english')\n",
    "        \n",
    "        # lemmatize\n",
    "        self.lemmatizer = nltk.stem.wordnet.WordNetLemmatizer()\n",
    "        \n",
    "        # translate\n",
    "        #self.translator = Translator()\n",
    "        \n",
    "        # declare a default regex dict\n",
    "        self.default_regex_dict = {'goo[o]*d':'good', '2morrow':'tomorrow', 'b4':'before', 'otw':'on the way',\n",
    "                                   'idk':\"i don't know\", ':)':'smile', 'bc':'because', '2nite':'tonight',\n",
    "                                   'yeah':'yes', 'yeshhhhhhhh':'yes', ' yeeeee':'yes', 'btw':'by the way', \n",
    "                                   'fyi':'for your information', 'gr8':'great', 'asap':'as soon as possible', \n",
    "                                   'yummmmmy':'yummy', 'gf':'girlfriend', 'thx':'thanks','nowwwwwww':'now', \n",
    "                                   ' ppl ':' people ', 'yeiii':'yes'}\n",
    "        \n",
    "        # if no regex_dict defined by user, then use \n",
    "        # one by default. Else, concat two regex dicts\n",
    "        if regex_dict:            \n",
    "            self.regex_dict = {**regex_dict, **default_regex_dict}\n",
    "            \n",
    "        else:\n",
    "            self.regex_dict = self.default_regex_dict\n",
    "\n",
    "    def removeNoise(self, pdf):\n",
    "        \n",
    "        \"\"\"\n",
    "        Function to remove noise from strings. \n",
    "        \n",
    "        Inputs: A pandas dataframe with raw strings of length n.\n",
    "        \n",
    "        Output: A clean string where elements such as accented \n",
    "        words, html tags, punctuation marks, and extra white \n",
    "        spaces will be removed (or transform) if it's the case.\n",
    "        \"\"\"\n",
    "        \n",
    "        # to lower case\n",
    "        pdf[\"clean_tweet\"] = pdf.text.apply(lambda x: x.lower())\n",
    "        \n",
    "        # remove accented characters from string\n",
    "        # e.g. canción --> cancion\n",
    "        pdf[\"clean_tweet\"] = pdf.clean_tweet.apply(lambda x: unidecode.unidecode(x))\n",
    "        \n",
    "        # remove html tags \n",
    "        pdf[\"clean_tweet\"] = pdf.clean_tweet.str.replace(r'<[^<>]*>', '', regex=True)\n",
    "        \n",
    "        # remove (match with) usernames | hashtags | punct marks | links\n",
    "        # punct marks = \",.':!?;\n",
    "        # do not remove: ' \n",
    "        # but remove: \"\n",
    "        pdf[\"clean_tweet\"] = pdf.clean_tweet.apply(lambda x:' '.join(re.sub(\"(@[A-Za-z0-9]+)|(#[A-Za-z0-9]+)|([-.,:_;])|(https?:\\/\\/.*[\\r\\n]*)\",\n",
    "                                                                            \" \", x).split()).replace('\"',''))\n",
    "                \n",
    "        # remove white spaces at the begining and at \n",
    "        # the end of a string\n",
    "        pdf['clean_tweet'] = pdf.clean_tweet.apply(lambda x: x.lstrip(' '))\n",
    "        pdf['clean_tweet'] = pdf.clean_tweet.apply(lambda x: x.rstrip(' '))\n",
    "        \n",
    "        # Translate tweet\n",
    "        #pdf[\"clean_tweet\"] = pdf.apply(lambda x: self.translate_twt(x) if pd.isnull(x.clean_tweet) == False else x, axis = 1)\n",
    "        \n",
    "        # normalize string\n",
    "        # normalize accented charcaters and other strange characters\n",
    "        # NFKD if there are accented characters (????\n",
    "        pdf[\"clean_tweet\"] = pdf.clean_tweet.apply(lambda x: unicodedata.normalize('NFKC', x).encode('ASCII', 'ignore').decode(\"utf-8\"))\n",
    "        \n",
    "        return pdf\n",
    "    \n",
    "    \n",
    "    def textNormalization(self, pdf):\n",
    "        \n",
    "        \"\"\"\n",
    "        Function to normalize a string. \n",
    "        \n",
    "        Inputs: A pandas dataframe with strings (of length n) that \n",
    "        will be normalized. \n",
    "        \n",
    "        Outputs: A normalized string whitout noise, words in their\n",
    "        (expected) correct form and with no stopwords.\n",
    "        \"\"\"\n",
    "        \n",
    "        # remove noise first\n",
    "        pdf = self.removeNoise(pdf)\n",
    "\n",
    "        # expand contractions\n",
    "        # e.g. don't --> do not\n",
    "        pdf['clean_tweet'] = pdf.clean_tweet.apply(lambda x: contractions.fix(x))\n",
    " \n",
    "        # Normalize words\n",
    "        pdf['clean_tweet'] = pdf.clean_tweet.replace(self.regex_dict)\n",
    "                \n",
    "        # get English stopwords    \n",
    "        stop_words = stopwords.words('english')\n",
    "        stopwords_dict = Counter(stop_words)\n",
    "        \n",
    "        # remove stopwords from string\n",
    "        pdf[\"clean_tweet\"] = pdf.clean_tweet.apply(lambda x: ' '.join([word for word in x.split()\n",
    "                                                                       if word not in stopwords_dict]))\n",
    "            \n",
    "        return pdf\n",
    "    \n",
    "    \n",
    "    def wordTokenize(self, pdf):\n",
    "        \"\"\"\n",
    "        Function to tokenize a string into words. Tokenization is a way \n",
    "        of separating a piece of text into smaller units called tokens.\n",
    "        In this case tokens are words (but can also be characters or \n",
    "        subwords).\n",
    "        \n",
    "        Inputs: A pandas dataframe with strings (of length n) that will be tokenized. \n",
    "        \n",
    "        Outputs: A list of tokenized words.\n",
    "        \"\"\"\n",
    "        # string normalized\n",
    "        #normalized = self.textNormalization(string)\n",
    "        pdf = self.textNormalization(pdf)\n",
    "        \n",
    "        # Use word_tokenize method to split the string\n",
    "        # into individual words. By default it returns\n",
    "        # a list.\n",
    "        pdf[\"clean_tweet\"] = pdf.clean_tweet.apply(lambda x: nltk.word_tokenize(x))        \n",
    "        \n",
    "        # Using isalpha() will help us to only keep\n",
    "        # items from the alphabet (no punctuation\n",
    "        # marks). \n",
    "        #pdf[\"clean_tweet\"] = pdf.clean_tweet.apply(lambda x: [word for word in x if word.isalpha()])\n",
    "        \n",
    "        # Keep only unique elements\n",
    "        pdf[\"clean_tweet\"] = pdf.clean_tweet.apply(lambda x: list(set(x)))\n",
    "\n",
    "        # return list of tokenized words by row\n",
    "        return pdf\n",
    "    \n",
    "    def phraseTokenize(self, pdf):\n",
    "        \n",
    "        \"\"\"\n",
    "        Function to tokenize a string into sentences. Tokenization is\n",
    "        a way of separating a piece of text into smaller units called\n",
    "        tokens. In this case tokens are phrases (but can also be words,\n",
    "        characters or subwords).\n",
    "        \n",
    "        Inputs: A string (of length n) that will be tokenized. \n",
    "        \n",
    "        Outputs: A list of tokenized sentences.\n",
    "        \"\"\"\n",
    "        \n",
    "        # pandas dataframe with strings normalized\n",
    "        pdf = self.textNormalization(pdf)\n",
    "        \n",
    "        # Use sent_tokenize method to split the string\n",
    "        # into sentences. By default it returns a list.\n",
    "        pdf[\"clean_tweet\"] = pdf.clean_tweet.apply(lambda x: nltk.sent_tokenize(x))   \n",
    "        \n",
    "        return pdf \n",
    "    \n",
    "    \n",
    "    def stemWords(self, pdf):\n",
    "        \n",
    "        \"\"\"\n",
    "        Function to stem strings. Stemming is the process of reducing\n",
    "        a word to its word stem that affixes to suffixes and prefixes \n",
    "        or to the roots of words (known as a lemma).\n",
    "        \n",
    "        Inputs: A raw string of length n.\n",
    "        \n",
    "        Output: Roots of each word of a given string.\n",
    "        \"\"\"\n",
    "        \n",
    "        # pandas dataframe with strings normalized\n",
    "        pdf = self.textNormalization(pdf)\n",
    "        \n",
    "        # tokenized string (into words)\n",
    "        pdf = self.wordTokenize(data)\n",
    "            \n",
    "        # reduct words to its root    \n",
    "        pdf[\"clean_tweet\"] = pdf.clean_tweet.apply(lambda x: [self.sb.stem(word) for word in x])\n",
    "        \n",
    "        return pdf\n",
    "    \n",
    "    \n",
    "    def lemmatizeWords(self, pdf):\n",
    "        \n",
    "        \"\"\"\n",
    "        Function to lemmatize strings. Lemmatization is a method \n",
    "        responsible for grouping different inflected forms of \n",
    "        words into the root form, having the same meaning. It is \n",
    "        similar to stemming.\n",
    "        \n",
    "        Inputs: A raw string of length n.\n",
    "        \n",
    "        Output: Roots of each word of a given string (with better\n",
    "        performance than in stemming).\n",
    "        \"\"\"\n",
    "        \n",
    "        # pandas dataframe with strings normalized\n",
    "        pdf = self.textNormalization(pdf)\n",
    "        \n",
    "        # list of tokenized words (from string)\n",
    "        # Here it was decided to tokenize by words\n",
    "        # rather than by sentences due to we thought\n",
    "        # it would be easier to find the correct roots\n",
    "        # of each word.\n",
    "        pdf = self.wordTokenize(pdf)\n",
    "        \n",
    "        # lematize word from list of tokenized words\n",
    "        #lematized = [self.lemmatizer.lemmatize(word) for word in tokenized]\n",
    "        pdf[\"clean_tweet\"] = pdf.clean_tweet.apply(lambda x: [self.lemmatizer.lemmatize(word) for word in x])\n",
    "        \n",
    "        return pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mapbox token\n",
    "px.set_mapbox_access_token(open(\"data/mapbox_token/.mapbox_token\").read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plotly dark template\n",
    "template='plotly_dark'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read historical data and model from folder 'trained_model'\n",
    "hist = pd.read_csv('trained_model/preds/tweets_preds.csv')\n",
    "keras_model = models.load_model(\"trained_model/tf_keras_model.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read saved locations from historical data\n",
    "geoplaces = pd.read_csv('data/places/geoplaces/geoplaces.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Graph 1 - Tweeter score distribution\n",
    "\n",
    "# Generate dist plot\n",
    "fig_1 = px.histogram(hist, x=\"pred_scores\", \n",
    "                   color=\"toxic\",\n",
    "                   labels={\n",
    "                     \"pred_scores\": \"Score\",\n",
    "                   },\n",
    "                   marginal=\"box\", # or violin, rug\n",
    "                   template=template,\n",
    "                   #hover_name='class',\n",
    "                   color_discrete_sequence=px.colors.qualitative.G10,\n",
    "                   nbins=50,\n",
    "                   opacity=0.8\n",
    "                   )\n",
    "\n",
    "fig_1.update_layout(\n",
    "    title={\"text\": \"Tweets Classification Scores\", \"x\": 0.5}, \n",
    "    yaxis_title=\"Frequency\",\n",
    "    barmode='overlay', \n",
    "    legend_title_text='Toxic', \n",
    "    #paper_bgcolor='rgba(0,0,0,0)',\n",
    "    #plot_bgcolor='rgba(0,0,0,0)'\n",
    ")\n",
    "#fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To geodataframe\n",
    "#geo_df = gpd.GeoDataFrame(geoplaces, geometry=gpd.points_from_xy(geoplaces.lon, geoplaces.lat))\n",
    "\n",
    "# Tweet location distribution\n",
    "fig_2 = px.density_mapbox(geoplaces,\n",
    "                        lat=geoplaces.lat,\n",
    "                        lon=geoplaces.lon, \n",
    "                        #z = 'counts',\n",
    "                        radius=15,\n",
    "                        color_continuous_scale=px.colors.sequential.Jet,\n",
    "                        #center=dict(lat=geo_df.geometry.y, lon=geo_df.geometry.x), \n",
    "                        hover_name='full_name',\n",
    "                        hover_data={'counts':True},\n",
    "                        template=template,\n",
    "                        zoom=1.5,\n",
    "                        )\n",
    "fig_2.update_layout(\n",
    "    title={\"text\": \"Tweets Density\", \"x\": 0.5},\n",
    "    coloraxis_showscale=False, \n",
    "    #paper_bgcolor='rgba(0,0,0,0)',\n",
    "    #plot_bgcolor='rgba(0,0,0,0)'\n",
    "    margin={'b':20, 't':45}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "geoplaces['random_color']=np.random.rand(len(geoplaces))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_3 = px.scatter_mapbox(geoplaces,\n",
    "                        lat=geoplaces.lat,\n",
    "                        lon=geoplaces.lon,\n",
    "                        hover_name=\"full_name\",\n",
    "                        hover_data={'random_color':False, 'counts':True},\n",
    "                        template=template,\n",
    "                        color='random_color',\n",
    "                        color_continuous_scale=px.colors.cyclical.HSV,\n",
    "                        #size='counts',\n",
    "                        #size_max=20,\n",
    "                        #color_discrete_sequence=px.colors.qualitative.G10,\n",
    "                        zoom=1.5)\n",
    "fig_3.update_layout(\n",
    "    title={\"text\": \"Tweets Location\", \"x\": 0.5},\n",
    "    coloraxis_colorbar={'title':'Tweets<br>Number'},\n",
    "    coloraxis_showscale=False,\n",
    "    #paper_bgcolor='rgba(0,0,0,0)',\n",
    "    #plot_bgcolor='rgba(0,0,0,0)'\n",
    "    margin={'b':20,'t':45}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fig_3.data[0]['lat'], fig_3.data[0]['lon'], fig_3.data[0]['marker']['color'], fig_3.data[0]['hovertext'], fig_3.data[0]['customdata']=  np.append(fig_3.data[0]['lat'],20.400417), np.append(fig_3.data[0]['lon'],-89.134857), np.append(fig_3.data[0]['marker']['color'],np.random.rand()), np.append(fig_3.data[0]['hovertext'],'Yucatán'), np.vstack([fig_3.data[0]['customdata'],np.array([0,1])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Style application\n",
    "external_stylesheets = [dbc.themes.CYBORG]\n",
    "app = dash.Dash(external_stylesheets=[external_stylesheets[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to datetime format\n",
    "hist['created_at']= pd.to_datetime(hist['created_at'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tweets by Hour\n",
    "a = hist.groupby([pd.Grouper(freq='D', key='created_at')]).tweet_id.count().reset_index()\n",
    "\n",
    "# Tweets by class and hour\n",
    "b = hist.groupby([pd.Grouper(freq='D', key='created_at'), 'toxic']).tweet_id.count().reset_index()\n",
    "\n",
    "# Merge both\n",
    "c = b.merge(a, right_on='created_at',left_on='created_at',how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter tweets per class\n",
    "toxic = c[c['toxic']==1]\n",
    "no_toxic = c[c['toxic']==0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_4 = go.Figure()\n",
    "\n",
    "fig_4.add_trace(go.Scatter(\n",
    "        x=c.created_at,\n",
    "        y=c.tweet_id_y,\n",
    "        name='Total Tweets',\n",
    "        mode='markers+lines',\n",
    "        line=dict(color='white', width=3),\n",
    "        ))\n",
    "fig_4.add_trace(go.Bar(\n",
    "        x=no_toxic.created_at,\n",
    "        y=no_toxic.tweet_id_x,\n",
    "        name='Non Toxic',\n",
    "        opacity=.5,\n",
    "        #visible=\"legendonly\"\n",
    "        marker=dict(color=px.colors.qualitative.G10[0])\n",
    "        ))\n",
    "fig_4.add_trace(go.Bar(\n",
    "        x=toxic.created_at,\n",
    "        y=toxic.tweet_id_x,\n",
    "        name='Toxic',\n",
    "        #visible=True\n",
    "        opacity=.5,\n",
    "        marker=dict(color=px.colors.qualitative.G10[1])\n",
    "        ))\n",
    "\n",
    "fig_4.update_layout(barmode='stack',template=template,title={\"text\": \"Tweets Frequency Timeline\", \"x\": 0.5},\n",
    "        yaxis_title=\"Number of Tweets\",\n",
    "        #paper_bgcolor='rgba(0,0,0,0)',\n",
    "        #plot_bgcolor='rgba(0,0,0,0)'\n",
    "        #xaxis_title=\"Date\"\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate random timestamps\n",
    "from random import randrange\n",
    "import datetime \n",
    "\n",
    "random_dates = []\n",
    "def random_date(start,l):\n",
    "   current = start\n",
    "   while l >= 0:\n",
    "    current = current + datetime.timedelta(minutes=randrange(10))\n",
    "    yield current\n",
    "    l-=1\n",
    "\n",
    "startDate = datetime.datetime(2021, 12, 3, 00, 00)\n",
    "\n",
    "\n",
    "for x in reversed(list(random_date(startDate,len(geoplaces)-1))):\n",
    "    random_dates.append(x.strftime(\"%d/%m/%y %H:%M\"))\n",
    "\n",
    "geoplaces['created_at']=random_dates\n",
    "geoplaces['created_at']=pd.to_datetime(geoplaces['created_at'])\n",
    "geoplaces['dt_str'] = geoplaces['created_at'].apply(lambda x: x.strftime(\"%d/%m/%y %H\"))\n",
    "\n",
    "geoplaces = geoplaces.sort_values('created_at')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_5 = px.scatter_mapbox(geoplaces,\n",
    "                        lat=geoplaces.lat,\n",
    "                        lon=geoplaces.lon,\n",
    "                        hover_name=\"full_name\",\n",
    "                        #hover_data={'score':True},\n",
    "                        template=template,\n",
    "                        color='random_color',\n",
    "                        color_continuous_scale=px.colors.cyclical.HSV,\n",
    "                        zoom=1.5,\n",
    "                        animation_frame=\"dt_str\",\n",
    "                        )\n",
    "fig_5.update_layout(\n",
    "    title={\"text\": \"Today's Tweets Location per hour\", \"x\": 0.5},\n",
    "    #coloraxis_colorbar={'title':'Tweets<br>Number'},\n",
    "    coloraxis_showscale=False,\n",
    "    #paper_bgcolor='rgba(0,0,0,0)',\n",
    "    #plot_bgcolor='rgba(0,0,0,0)'\n",
    "    margin={'t':45},\n",
    "    height=750\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"fig_6 = go.Figure(data=[go.Table(\n",
    "    header=dict(values=list(hist.columns),\n",
    "                #fill_color='paleturquoise',\n",
    "                align='left'),\n",
    "    cells=dict(values=[hist[column] for column in hist.columns],\n",
    "               #fill_color='lavender',\n",
    "               align='left', height=30))\n",
    "], )\n",
    "\n",
    "fig_6.update_layout(height=750,template=template)\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "card = html.Div(\n",
    "    [\n",
    "        dbc.Card(\n",
    "            [\n",
    "                dbc.CardBody(\n",
    "                    [   \n",
    "                        html.H4(\"Tweets from nov 21st to dec 1st: {}\".format(len(hist)), className=\"card-title\", id='title-counter'),\n",
    "                        #dcc.Store(id='original_df', children=hist.to_json(date_format='iso', orient='split')),\n",
    "\n",
    "                        dbc.Row(\n",
    "                            [\n",
    "                                dbc.Col(\n",
    "                                    [   \n",
    "                                        dcc.DatePickerRange(\n",
    "                                            id='my-date-picker-range',\n",
    "                                            min_date_allowed=date(2021, 11, 21),\n",
    "                                            max_date_allowed=date(2021,12, 4),\n",
    "                                            start_date=date(2021, 11, 21),\n",
    "                                            end_date=date(2021,12, 2),\n",
    "                                            #initial_visible_month=date(2017, 8, 5),\n",
    "                                        ),\n",
    "                                        html.Br(),\n",
    "                                        html.Br(),\n",
    "                                        html.P(\"Click the button to update the dashboard with today's tweets\", className=\"card-title\"),\n",
    "                                        dbc.Button('Update', id='submit-val', n_clicks=0),\n",
    "                                        dbc.Spinner(html.Div(id='container-button-basic',\n",
    "                                        children='')),\n",
    "                                        dcc.Store(id='intermediate-value'),\n",
    "                                        dcc.Store(id='intermediate-value-places'),\n",
    "                                        html.Br(),\n",
    "                                        html.Div(id='test-output',\n",
    "                                        children='output prueba places'),\n",
    "                                        html.Br(),\n",
    "                                        html.Div(id='test-output-places',\n",
    "                                        children='lista places added'),\n",
    "                                        html.Br(),\n",
    "                                    ], width=6\n",
    "                                ),\n",
    "                                dbc.Col(\n",
    "                                    [\n",
    "                                        html.H6(\"Add a new Tweet location to the map\", className=\"card-title\"),\n",
    "                                        dbc.Input(id='input-1-state', type='number', placeholder='Insert latitude', style={'width':'30%'}),\n",
    "                                        dbc.Input(id='input-2-state', type='number', placeholder='Insert longitude', style={'width':'30%'}),\n",
    "                                        dbc.Input(id='input-3-state', type='text', placeholder='Insert label name', style={'width':'30%'}),\n",
    "                                        dbc.Button(id='submit-button-state', n_clicks=0, children='Submit'),\n",
    "                                        html.Div(id='output-state'),\n",
    "                                        html.Br(),\n",
    "                                    ], width=6\n",
    "                                ),\n",
    "                            ]\n",
    "                        ),\n",
    "\n",
    "                        \n",
    "                        dbc.Row(\n",
    "                            [\n",
    "                                dbc.Col(\n",
    "                                    [ \n",
    "                                        dcc.Graph(id='fig_2', figure=fig_2)\n",
    "                                    ], width=6\n",
    "                                ),\n",
    "                                dbc.Col(\n",
    "                                    [ \n",
    "                                        dcc.Graph(id='fig_3', figure=fig_3)\n",
    "                                    ], width=6\n",
    "                                )\n",
    "                            ], align='center'\n",
    "                        ),\n",
    "                        html.Br(),\n",
    "                        dbc.Row(\n",
    "                            [\n",
    "                                dbc.Col(\n",
    "                                    [\n",
    "                                        dcc.Graph(id='fig_1', figure=fig_1)\n",
    "                                    ], width=6\n",
    "                                ),\n",
    "                                dbc.Col(\n",
    "                                    [\n",
    "                                        dcc.Graph(id='fig_4', figure=fig_4)\n",
    "                                    ], width=6\n",
    "                                )\n",
    "                            ], align='center', #style={'background-color':'#060606'}\n",
    "                        )\n",
    "                    ]\n",
    "                )\n",
    "            ]\n",
    "        )\n",
    "    ]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "card_2 = html.Div(\n",
    "    [\n",
    "        dbc.Card(\n",
    "            [\n",
    "                dbc.CardBody(\n",
    "                    [   \n",
    "                        #html.H5(\"Card title\", className=\"card-title\"),\n",
    "                        #html.P(\"This card has some text content, but not much else\"),\n",
    "                        dbc.Row(\n",
    "                            [\n",
    "                                dbc.Col(\n",
    "                                    [ \n",
    "                                        dcc.Graph(id='fig_5', figure=fig_5)\n",
    "                                    ], width=12\n",
    "                                )\n",
    "                            ], align='center',\n",
    "                        ),\n",
    "                    ]\n",
    "                )\n",
    "            ]\n",
    "        )\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"card_3 = html.Div(\n",
    "    [\n",
    "        dbc.Card(\n",
    "            [\n",
    "                dbc.CardBody(\n",
    "                    [   \n",
    "                        html.H5(\"Covid Tweets DataFrame\", className=\"card-title\"),\n",
    "                        #html.P(\"This card has some text content, but not much else\"),\n",
    "                        dbc.Row(\n",
    "                            [\n",
    "                                dbc.Col(\n",
    "                                    [ \n",
    "                                        dcc.Graph(id='fig_6', figure=fig_6)\n",
    "                                    ], width=12\n",
    "                                )\n",
    "                            ], align='center',\n",
    "                        ),\n",
    "                    ]\n",
    "                )\n",
    "            ]\n",
    "        )\n",
    "    ]\n",
    ")\"\"\"\n",
    "from dash import dash_table\n",
    "card_3 = dash_table.DataTable(\n",
    "    id=\"table\",\n",
    "    columns=[{\"name\": i, \"id\": i} for i in hist.columns],\n",
    "    data=hist.to_dict(\"records\"),\n",
    "    export_format=\"csv\",\n",
    "    style_as_list_view=True,\n",
    "    style_cell={'padding': '5px', \n",
    "        'overflow': 'hidden',\n",
    "        'textOverflow': 'ellipsis',\n",
    "        'maxWidth': 0,\n",
    "        'backgroundColor': 'gray',},\n",
    "    style_header={\n",
    "        'backgroundColor': 'black',\n",
    "        'fontWeight': 'bold'\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "card_4 = html.Div(\n",
    "    [\n",
    "        dbc.Card(\n",
    "            [\n",
    "                dbc.CardBody(\n",
    "                    [   \n",
    "                        html.H5(\"Tweet Generator\", className=\"card-title\"),\n",
    "                        html.P(\"Transform a toxic Tweet into a non-toxic one\"),\n",
    "                        dbc.Row(\n",
    "                            [\n",
    "                                dbc.Col(\n",
    "                                    [ \n",
    "                                        html.Div(\n",
    "                                            [\n",
    "                                                dbc.Input(id='input-4-state', type='text', placeholder=\"Insert a toxic tweet\"),\n",
    "                                                dbc.Button(id='submit-button-state-tweet', n_clicks=0, children='Submit'),\n",
    "                                            ]\n",
    "                                        ),\n",
    "                                        html.Br(),\n",
    "                                        html.Div(\n",
    "                                            [\n",
    "                                                html.Div(id='output-state-tweet', children = 'The result will apear here')\n",
    "                                            ]\n",
    "                                        ),\n",
    "                                    ], width=8\n",
    "                                )\n",
    "                            ], align='center',\n",
    "                        ),\n",
    "                    ]\n",
    "                )\n",
    "            ]\n",
    "        )\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab_1 = dbc.Tab(label=\"Overview\",children=[card])\n",
    "tab_2 = dbc.Tab(label=\"Tweets Animation Frame\",children=[card_2])\n",
    "tab_3 = dbc.Tab(label=\"Raw Data\",children=[card_3])\n",
    "tab_4 = dbc.Tab(label=\"Tweet Generator\",children=[card_4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Layout\n",
    "app.layout = html.Div([html.H1('COVID-Tweets Dashboard'),\n",
    "                       #html.P('Holi SUbtitulo o historia mamon'),\n",
    "                       dbc.Tabs(\n",
    "                          [\n",
    "                             tab_1,\n",
    "                             tab_2,\n",
    "                             tab_3, \n",
    "                             tab_4\n",
    "                          ]\n",
    "                       )\n",
    "                       ]\n",
    "                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loc(x):\n",
    "    try:\n",
    "        return geolocator.geocode(x)[1]\n",
    "    except:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@app.callback([Output('output-state', 'children'), Output('fig_2', 'figure'), Output('fig_3', 'figure'), Output('test-output-places', 'children')],\n",
    "              [Input('submit-button-state', 'n_clicks'),Input('intermediate-value-places', 'data')],\n",
    "              State('input-1-state', 'value'),\n",
    "              State('input-2-state', 'value'),\n",
    "              State('input-3-state', 'value'),)\n",
    "def update_output(n_clicks, new_places, input1, input2, input3):\n",
    "\n",
    "    # interval\n",
    "    #df_date = hist[(hist.created_at>=start_date)&(hist.created_at<=end_date)]\n",
    "    \n",
    "    # Single location by hand\n",
    "    fig_2.data[0]['lat'], fig_2.data[0]['lon'], fig_2.data[0]['hovertext'], fig_2.data[0]['customdata']=  np.append(fig_2.data[0]['lat'],input1), np.append(fig_2.data[0]['lon'],input2), np.append(fig_2.data[0]['hovertext'],input3), np.vstack([fig_2.data[0]['customdata'],np.array([1])])\n",
    "    fig_3.data[0]['lat'], fig_3.data[0]['lon'], fig_3.data[0]['marker']['color'], fig_3.data[0]['hovertext'], fig_3.data[0]['customdata']=  np.append(fig_3.data[0]['lat'],input1), np.append(fig_3.data[0]['lon'],input2), np.append(fig_3.data[0]['marker']['color'],np.random.rand()), np.append(fig_3.data[0]['hovertext'],input3), np.vstack([fig_3.data[0]['customdata'],np.array([0,1])])\n",
    "    \n",
    "    # Many new locations with button\n",
    "    new_places = pd.read_json(new_places, orient='split')\n",
    "    \n",
    "    if new_places.shape[0]>0:\n",
    "        places_names = new_places.full_name.values\n",
    "        fig_2.data[0]['lat'], fig_2.data[0]['lon'], fig_2.data[0]['hovertext'], fig_2.data[0]['customdata']=  np.append(fig_2.data[0]['lat'],new_places.lat.values), np.append(fig_2.data[0]['lon'],new_places.lon.values), np.append(fig_2.data[0]['hovertext'],new_places.full_name.values), np.vstack([fig_2.data[0]['customdata'],np.array([[1] for i in range(len(new_places))])])\n",
    "        fig_3.data[0]['lat'], fig_3.data[0]['lon'], fig_3.data[0]['marker']['color'], fig_3.data[0]['hovertext'], fig_3.data[0]['customdata']=  np.append(fig_3.data[0]['lat'],new_places.lat.values), np.append(fig_3.data[0]['lon'],new_places.lon.values), np.append(fig_3.data[0]['marker']['color'],np.random.rand(len(new_places))), np.append(fig_3.data[0]['hovertext'],new_places.full_name.values), np.vstack([fig_3.data[0]['customdata'],np.array([[0,1] for i in range(len(new_places))])])\n",
    "    else:\n",
    "        places_names = 'None yet'\n",
    "        pass\n",
    "    \n",
    "    return u'''\n",
    "        The Button has been pressed {} times,\n",
    "        new place \"{}\" added, \n",
    "        latitude is \"{}\",\n",
    "        and longitude is \"{}\"\n",
    "    '''.format(n_clicks, input3, input1, input2), fig_2, fig_3, 'PLACES ADDED: {}'.format(places_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Callbacks for real-time\n",
    "@app.callback(\n",
    "    [Output('container-button-basic', 'children'),Output('intermediate-value', 'data'),Output('intermediate-value-places', 'data')],\n",
    "    Input('submit-val', 'n_clicks')\n",
    ")\n",
    "def update_output(n_clicks):\n",
    "    if n_clicks>0:\n",
    "\n",
    "        # main places df\n",
    "        main_places = pd.DataFrame(columns=geoplaces.columns)\n",
    "        # Get daa\n",
    "        search_tweet = collecting_tweets.search_tweets(query='covid', bearer_token = bearer_token, next_token = None)\n",
    "\n",
    "        # Check if we have tweet's location\n",
    "        if \"places\" in search_tweet['includes'].keys():\n",
    "            main_tweets, main_users, main_places = collecting_tweets.create_dataframes(search_tweet)\n",
    "            \n",
    "        else:\n",
    "            main_tweets, main_users = collecting_tweets.create_dataframes(search_tweet)\n",
    "            main_places = pd.DataFrame()\n",
    "\n",
    "        # Generate dataframes\n",
    "        try:\n",
    "            main_tweets, main_users, main_places = collecting_tweets.more_tweets(20, \"covid\", search_tweet,  main_tweets, \n",
    "                                                                                main_users, main_places)\n",
    "\n",
    "        except ValueError:\n",
    "            main_tweets, main_users = collecting_tweets.more_tweets(20, \"covid\", search_tweet, main_tweets, \n",
    "                                                                    main_users, main_places)\n",
    "\n",
    "        if main_places.empty:\n",
    "            pass\n",
    "\n",
    "        else:\n",
    "            main_places[\"coords\"] = main_places.full_name.apply(lambda x: loc(x))\n",
    "            main_places=main_places[main_places.coords.isna()==False]\n",
    "            \n",
    "            if main_places.shape[0]!=0:\n",
    "                main_places[\"lat\"] = main_places.coords.apply(lambda x: x[0])\n",
    "                main_places[\"lon\"] = main_places.coords.apply(lambda x: x[1])\n",
    "            else:\n",
    "                pass\n",
    "\n",
    "        #print(main_places.full_name.values)\n",
    "\n",
    "        # Create class object\n",
    "        pre_processor = PreProcessor()\n",
    "        # Clean data and only keep \n",
    "        # the roots of each word.\n",
    "        #tweets['clean_tweet'] = tweets.text.apply(pre_processor.removeNoise)\n",
    "        main_tweets = pre_processor.lemmatizeWords(main_tweets)\n",
    "\n",
    "        tokenizer = preprocessing.text.Tokenizer(num_words = 500000, filters='!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~', \n",
    "                                                lower = True)\n",
    "\n",
    "        # Updates internal vocabulary based on a list of texts.\n",
    "        # In our case, since texts contains lists, it is assumed each entry of the lists to be a token.\n",
    "        tokenizer.fit_on_texts(main_tweets['clean_tweet'].values)\n",
    "\n",
    "        X = tokenizer.texts_to_sequences(main_tweets[\"clean_tweet\"].values)\n",
    "        X = preprocessing.sequence.pad_sequences(X, maxlen = 392)\n",
    "\n",
    "        # Predict\n",
    "        y_pred_scores = keras_model.predict(X)\n",
    "        y_pred = np.round(y_pred_scores).astype(int)\n",
    "\n",
    "        # Create columns with the result from the model\n",
    "        main_tweets[\"toxic\"] = y_pred\n",
    "        main_tweets[\"pred_scores\"] = y_pred_scores\n",
    "        \n",
    "        cleaned_df= hist.append(main_tweets, ignore_index=True)\n",
    "        #hist.append(main_tweets, ignore_index=True, inplace=True)\n",
    "        #hist = cleaned_df\n",
    "\n",
    "        # aqui el hist es el que se tiene que cambiar\n",
    "        #hist = hist.append(main_tweets, ignore_index=True)\n",
    "\n",
    "        return 'You have updated the dashboard {} times'.format(\n",
    "            n_clicks\n",
    "        ), cleaned_df.to_json(date_format='iso', orient='split'), main_places.to_json(date_format='iso', orient='split')\n",
    "    else:\n",
    "        main_places=pd.DataFrame()\n",
    "        return 'You have updated the dashboard {} times'.format(\n",
    "            n_clicks\n",
    "        ), hist.to_json(date_format='iso', orient='split'), main_places.to_json(date_format='iso', orient='split')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@app.callback(\n",
    "    Output('test-output', 'children'),\n",
    "    [Input('intermediate-value', 'data'),Input('submit-val', 'n_clicks')]\n",
    ")\n",
    "def update_output(data, n_clicks):\n",
    "\n",
    "    if n_clicks > 0:\n",
    "        df = pd.read_json(data, orient='split')\n",
    "        return 'The new df has {} tweets'.format(len(df))\n",
    "    else:\n",
    "        return ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# figures callbacks\n",
    "@app.callback(\n",
    "        [   \n",
    "        Output('fig_1', \"figure\"),\n",
    "        #Output('fig_2', \"figure\"),\n",
    "        #Output('fig_3', \"figure\"),\n",
    "        Output('fig_4', \"figure\"),\n",
    "        #Output('fig_5', \"figure\"),\n",
    "        ],\n",
    "    [Input('intermediate-value', 'data'),\n",
    "    Input('my-date-picker-range', \"start_date\"),\n",
    "    Input('my-date-picker-range', \"end_date\"),]\n",
    ")\n",
    "\n",
    "def update_output(data, start_date, end_date):\n",
    "\n",
    "    new_hist = pd.read_json(data, orient='split')\n",
    "    mask = (\n",
    "        (new_hist.created_at >= start_date)\n",
    "        & (new_hist.created_at <= end_date)\n",
    "    )\n",
    "    \n",
    "    new_hist =  new_hist.loc[mask, :]\n",
    "\n",
    "    # FIGURA 1\n",
    "    fig_1 = px.histogram(new_hist, x=\"pred_scores\", \n",
    "                   color=\"toxic\",\n",
    "                   labels={\n",
    "                     \"pred_scores\": \"Score\",\n",
    "                   },\n",
    "                   marginal=\"box\", # or violin, rug\n",
    "                   template=template,\n",
    "                   #hover_name='class',\n",
    "                   color_discrete_sequence=px.colors.qualitative.G10,\n",
    "                   nbins=50,\n",
    "                   opacity=0.8\n",
    "                   )\n",
    "\n",
    "    fig_1.update_layout(\n",
    "        title={\"text\": \"Tweets Classification Scores\", \"x\": 0.5}, \n",
    "        yaxis_title=\"Frequency\",\n",
    "        barmode='overlay', \n",
    "        legend_title_text='Toxic', \n",
    "        #paper_bgcolor='rgba(0,0,0,0)',\n",
    "        #plot_bgcolor='rgba(0,0,0,0)'\n",
    "    )\n",
    "\n",
    "    # FIGURA 4 \n",
    "    \n",
    "    # Tweets by Hour\n",
    "    a = new_hist.groupby([pd.Grouper(freq='D', key='created_at')]).tweet_id.count().reset_index()\n",
    "\n",
    "    # Tweets by class and hour\n",
    "    b = new_hist.groupby([pd.Grouper(freq='D', key='created_at'), 'toxic']).tweet_id.count().reset_index()\n",
    "\n",
    "    # Merge both\n",
    "    c = b.merge(a, right_on='created_at',left_on='created_at',how='left')\n",
    "\n",
    "    # Filter tweets per class\n",
    "    toxic = c[c['toxic']==1]\n",
    "    no_toxic = c[c['toxic']==0]\n",
    "\n",
    "    fig_4 = go.Figure()\n",
    "\n",
    "    fig_4.add_trace(go.Scatter(\n",
    "            x=c.created_at,\n",
    "            y=c.tweet_id_y,\n",
    "            name='Total Tweets',\n",
    "            mode='markers+lines',\n",
    "            line=dict(color='white', width=3),\n",
    "            ))\n",
    "    fig_4.add_trace(go.Bar(\n",
    "            x=no_toxic.created_at,\n",
    "            y=no_toxic.tweet_id_x,\n",
    "            name='Non Toxic',\n",
    "            opacity=.5,\n",
    "            #visible=\"legendonly\"\n",
    "            marker=dict(color=px.colors.qualitative.G10[0])\n",
    "            ))\n",
    "    fig_4.add_trace(go.Bar(\n",
    "            x=toxic.created_at,\n",
    "            y=toxic.tweet_id_x,\n",
    "            name='Toxic',\n",
    "            #visible=True\n",
    "            opacity=.5,\n",
    "            marker=dict(color=px.colors.qualitative.G10[1])\n",
    "            ))\n",
    "\n",
    "    fig_4.update_layout(barmode='stack',template=template,title={\"text\": \"Tweets Frequency Timeline\", \"x\": 0.5},\n",
    "            yaxis_title=\"Number of Tweets\",\n",
    "            #paper_bgcolor='rgba(0,0,0,0)',\n",
    "            #plot_bgcolor='rgba(0,0,0,0)'\n",
    "            #xaxis_title=\"Date\"\n",
    "            )\n",
    "\n",
    "    return fig_1, fig_4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    app.run_server(debug=True, use_reloader=False)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "76e3749c4d8f143b85e17554ce3ae58a0fcf4212a15be7e0134aaba6048266e3"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 64-bit ('base': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}