Dashboard.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyhive import hive\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib.animation import FuncAnimation\n",
    "#%matplotlib inline\n",
    "%matplotlib notebook\n",
    "import random\n",
    "from itertools import count\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "from pandas._libs.tslibs.timestamps import Timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common_vars import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hive_cnx = hive.Connection(\n",
    "  host = hdfs_host, \n",
    "  port = hive_port, \n",
    "  username = hive_username,\n",
    "  password = hive_password,\n",
    "  auth = hive_mode,\n",
    "  database=hive_database)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_queries(queries):\n",
    "    with hive_cnx.cursor() as cursor:   \n",
    "        for q in queries:\n",
    "            cursor.execute(q)    \n",
    "        if cursor.poll().hasResultSet:\n",
    "            return cursor.fetchall()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_keys = ['id', 'text','created_at', 'geo', 'coordinates', 'place',\n",
    "              'quote_count', 'reply_count', 'retweet_count', 'favorite_count',\n",
    "              'user_id', 'user_followers','Sentiment','n_words', 'key']\n",
    "\n",
    "cols = ', '.join(tweet_keys)\n",
    "\n",
    "q = [f\"select {cols} from tweets where user_followers is not NULL\"]\n",
    "q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = run_queries(q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweetsDF = pd.DataFrame(tweets, columns= tweet_keys)\n",
    "tweetsDF.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(13,9))\n",
    "plt.scatter(tweetsDF['user_followers'], tweetsDF['Sentiment'] )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "t = tweetsDF[['created_at','user_followers','Sentiment']]\n",
    "\n",
    "log10  = lambda x : np.log10(x +1)\n",
    "\n",
    "t.loc[:,'user_followers'] = tweetsDF['user_followers'].apply(log10)\n",
    "\n",
    "timestp = lambda x: datetime.utcfromtimestamp(x.timestamp()) \n",
    "\n",
    "t.loc[:,'created_at'] = tweetsDF['created_at'].apply(timestp)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(13,9))\n",
    "plt.scatter(t['user_followers'], t['Sentiment'])\n",
    "plt.xlabel('log10(user_followers)')\n",
    "plt.ylabel('Sentiment')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=[13,9])\n",
    "hst = plt.hist2d(t.user_followers[t.Sentiment!=0], t.Sentiment[t.Sentiment!=0], bins=[100,100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q = .7\n",
    "qS = abs(t.Sentiment).quantile(q)\n",
    "qS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=[13,9])\n",
    "plt.hist(t.Sentiment[t.Sentiment!=0], bins=200)\n",
    "plt.vlines([-qS,qS],ymin= 0, ymax = 10,colors='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "quf = t.user_followers.quantile(q)\n",
    "quf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=[9,9])\n",
    "plt.hist(t.user_followers[t.Sentiment!=0], bins=200)\n",
    "plt.vlines([quf],ymin= 0, ymax = 5,colors='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tt = (t.user_followers>quf) & (t.user_followers<5) & ((t.Sentiment) < -qS)\n",
    "tt =  ((t.Sentiment) < -qS)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tt = tweetsDF[tt]\n",
    "len(tt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "badboys = tt.sort_values(by=['Sentiment','user_followers'], ascending = [True, False] )\n",
    "cols = \"text created_at user_followers Sentiment\".split(\" \")\n",
    "\n",
    "print(\"\\n-----------------------------\\n\".join(badboys['text']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(13,9))\n",
    "plt.scatter(np.log10(badboys['user_followers']), badboys['Sentiment'],)\n",
    "plt.xlabel('log10(user_followers)')\n",
    "plt.ylabel('Sentiment')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "badboys.index = range(len(badboys))\n",
    "i=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# print(badboys.text[i])\n",
    "# i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a,b = min(t['created_at']), max(t['created_at'])\n",
    "# a,b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyhive import hive\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib.animation import FuncAnimation\n",
    "from matplotlib import dates as mdates\n",
    "\n",
    "from itertools import count\n",
    "import numpy as np\n",
    "%matplotlib notebook\n",
    "n = len(t)\n",
    "\n",
    "plt.style.use('fivethirtyeight')\n",
    "\n",
    "\n",
    "# Parameters\n",
    "x_len = 200       # Number of points to display\n",
    "y_range = [-5,5]  # Range of possible Y values to display\n",
    "\n",
    "# Create figure for plotting\n",
    "fig = plt.figure(figsize = [9,9])\n",
    "ax = fig.add_subplot(1, 1, 1)\n",
    "xs = list(range(0, 200))\n",
    "ys = [0] * x_len\n",
    "ax.set_ylim([-100,100])\n",
    "#ax.set_xlim([a.timestamp(),b.timestamp()])\n",
    "\n",
    "ax.xaxis.set_major_formatter(mdates.DateFormatter(\"%H:%M\"))\n",
    "ax.xaxis_date()\n",
    "x = []\n",
    "y1 = []\n",
    "y2 = []\n",
    "wtf = []\n",
    "#index = count()\n",
    "\n",
    "#plt.plot([], [], label='Channel 1')\n",
    "#plt.plot([], [], label='Channel 2')\n",
    "\n",
    "\n",
    "def animate(i):\n",
    "    data = t[:i]\n",
    "    x = data['created_at']\n",
    "    y1 = data['user_followers']\n",
    "    y2 = data['Sentiment']\n",
    "    \n",
    "    #wtf.append(ax.xlim)\n",
    "    \n",
    "    ax.clear()\n",
    "    \n",
    "    #ax.scatter(x,y1, label='user_followers')\n",
    "    ax.scatter(x,y2, label='Sentiment')\n",
    "    ax.set_xlim([a,b])\n",
    "    plt.xticks(rotation=45, ha='right')\n",
    "    plt.legend(loc='upper left')\n",
    "    plt.subplots_adjust(bottom=0.30)\n",
    "    #plt.xlim = [0,24]\n",
    "#     plt.ylim = [-100,100]    \n",
    "#     plt.tight_layout()\n",
    "\n",
    "\n",
    "ani = FuncAnimation(fig, animate, frames=range(4000,4800), interval=100)#4000\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "fig = plt.figure(figsize=[13,9])\n",
    "ax = fig.add_subplot(1, 1, 1)\n",
    "xs = list(range(0, 200))\n",
    "ys = [0] * x_len\n",
    "ax.set_ylim([-100,100])\n",
    "ax.set_xlim([a,b])\n",
    "animate(4800)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}