4.2 Exercicio Autocodificadores.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Análise de sentimentos no Twitter ###\n",
    "\n",
    "Para este exercício, usaremos a seguinte base de dados: http://thinknook.com/wp-content/uploads/2012/09/Sentiment-Analysis-Dataset.zip\n",
    "\n",
    "Ela possui diversos Tweets e sua polaridade. Ela tem bastante ruído, então não espere precisões gigantescas.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras.backend as K\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Flatten\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "from nltk.stem.lancaster import LancasterStemmer\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "import numpy as np\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Vamos ler nossos dados e tokenizar (sempre é bom fazer isso com texto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tamanho do Corpus: 1578627\n",
      "0\n",
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n",
      "26000\n",
      "27000\n",
      "28000\n",
      "29000\n",
      "30000\n",
      "31000\n",
      "32000\n",
      "33000\n",
      "34000\n",
      "35000\n",
      "36000\n",
      "37000\n",
      "38000\n",
      "39000\n",
      "40000\n",
      "41000\n",
      "42000\n",
      "43000\n",
      "44000\n",
      "45000\n",
      "46000\n",
      "47000\n",
      "48000\n",
      "49000\n",
      "50000\n",
      "51000\n",
      "52000\n",
      "53000\n",
      "54000\n",
      "55000\n",
      "56000\n",
      "57000\n",
      "58000\n",
      "59000\n",
      "60000\n",
      "61000\n",
      "62000\n",
      "63000\n",
      "64000\n",
      "65000\n",
      "66000\n",
      "67000\n",
      "68000\n",
      "69000\n",
      "70000\n",
      "71000\n",
      "72000\n",
      "73000\n",
      "74000\n",
      "75000\n",
      "76000\n",
      "77000\n",
      "78000\n",
      "79000\n",
      "80000\n",
      "81000\n",
      "82000\n",
      "83000\n",
      "84000\n",
      "85000\n",
      "86000\n",
      "87000\n",
      "88000\n",
      "89000\n",
      "90000\n",
      "91000\n",
      "92000\n",
      "93000\n",
      "94000\n",
      "95000\n",
      "96000\n",
      "97000\n",
      "98000\n",
      "99000\n",
      "100000\n",
      "101000\n",
      "102000\n",
      "103000\n",
      "104000\n",
      "105000\n",
      "106000\n",
      "107000\n",
      "108000\n",
      "109000\n",
      "110000\n",
      "111000\n",
      "112000\n",
      "113000\n",
      "114000\n",
      "115000\n",
      "116000\n",
      "117000\n",
      "118000\n",
      "119000\n",
      "120000\n",
      "121000\n",
      "122000\n",
      "123000\n",
      "124000\n",
      "125000\n",
      "126000\n",
      "127000\n",
      "128000\n",
      "129000\n",
      "130000\n",
      "131000\n",
      "132000\n",
      "133000\n",
      "134000\n",
      "135000\n",
      "136000\n",
      "137000\n",
      "138000\n",
      "139000\n",
      "140000\n",
      "141000\n",
      "142000\n",
      "143000\n",
      "144000\n",
      "145000\n",
      "146000\n",
      "147000\n",
      "148000\n",
      "149000\n",
      "150000\n",
      "151000\n",
      "152000\n",
      "153000\n",
      "154000\n",
      "155000\n",
      "156000\n",
      "157000\n",
      "158000\n",
      "159000\n",
      "160000\n",
      "161000\n",
      "162000\n",
      "163000\n",
      "164000\n",
      "165000\n",
      "166000\n",
      "167000\n",
      "168000\n",
      "169000\n",
      "170000\n",
      "171000\n",
      "172000\n",
      "173000\n",
      "174000\n",
      "175000\n",
      "176000\n",
      "177000\n",
      "178000\n",
      "179000\n",
      "180000\n",
      "181000\n",
      "182000\n",
      "183000\n",
      "184000\n",
      "185000\n",
      "186000\n",
      "187000\n",
      "188000\n",
      "189000\n",
      "190000\n",
      "191000\n",
      "192000\n",
      "193000\n",
      "194000\n",
      "195000\n",
      "196000\n",
      "197000\n",
      "198000\n",
      "199000\n",
      "200000\n",
      "201000\n",
      "202000\n",
      "203000\n",
      "204000\n",
      "205000\n",
      "206000\n",
      "207000\n",
      "208000\n",
      "209000\n",
      "210000\n",
      "211000\n",
      "212000\n",
      "213000\n",
      "214000\n",
      "215000\n",
      "216000\n",
      "217000\n",
      "218000\n",
      "219000\n",
      "220000\n",
      "221000\n",
      "222000\n",
      "223000\n",
      "224000\n",
      "225000\n",
      "226000\n",
      "227000\n",
      "228000\n",
      "229000\n",
      "230000\n",
      "231000\n",
      "232000\n",
      "233000\n",
      "234000\n",
      "235000\n",
      "236000\n",
      "237000\n",
      "238000\n",
      "239000\n",
      "240000\n",
      "241000\n",
      "242000\n",
      "243000\n",
      "244000\n",
      "245000\n",
      "246000\n",
      "247000\n",
      "248000\n",
      "249000\n",
      "250000\n",
      "251000\n",
      "252000\n",
      "253000\n",
      "254000\n",
      "255000\n",
      "256000\n",
      "257000\n",
      "258000\n",
      "259000\n",
      "260000\n",
      "261000\n",
      "262000\n",
      "263000\n",
      "264000\n",
      "265000\n",
      "266000\n",
      "267000\n",
      "268000\n",
      "269000\n",
      "270000\n",
      "271000\n",
      "272000\n",
      "273000\n",
      "274000\n",
      "275000\n",
      "276000\n",
      "277000\n",
      "278000\n",
      "279000\n",
      "280000\n",
      "281000\n",
      "282000\n",
      "283000\n",
      "284000\n",
      "285000\n",
      "286000\n",
      "287000\n",
      "288000\n",
      "289000\n",
      "290000\n",
      "291000\n",
      "292000\n",
      "293000\n",
      "294000\n",
      "295000\n",
      "296000\n",
      "297000\n",
      "298000\n",
      "299000\n",
      "300000\n",
      "301000\n",
      "302000\n",
      "303000\n",
      "304000\n",
      "305000\n",
      "306000\n",
      "307000\n",
      "308000\n",
      "309000\n",
      "310000\n",
      "311000\n",
      "312000\n",
      "313000\n",
      "314000\n",
      "315000\n",
      "316000\n",
      "317000\n",
      "318000\n",
      "319000\n",
      "320000\n",
      "321000\n",
      "322000\n",
      "323000\n",
      "324000\n",
      "325000\n",
      "326000\n",
      "327000\n",
      "328000\n",
      "329000\n",
      "330000\n",
      "331000\n",
      "332000\n",
      "333000\n",
      "334000\n",
      "335000\n",
      "336000\n",
      "337000\n",
      "338000\n",
      "339000\n",
      "340000\n",
      "341000\n",
      "342000\n",
      "343000\n",
      "344000\n",
      "345000\n",
      "346000\n",
      "347000\n",
      "348000\n",
      "349000\n",
      "350000\n",
      "351000\n",
      "352000\n",
      "353000\n",
      "354000\n",
      "355000\n",
      "356000\n",
      "357000\n",
      "358000\n",
      "359000\n",
      "360000\n",
      "361000\n",
      "362000\n",
      "363000\n",
      "364000\n",
      "365000\n",
      "366000\n",
      "367000\n",
      "368000\n",
      "369000\n",
      "370000\n",
      "371000\n",
      "372000\n",
      "373000\n",
      "374000\n",
      "375000\n",
      "376000\n",
      "377000\n",
      "378000\n",
      "379000\n",
      "380000\n",
      "381000\n",
      "382000\n",
      "383000\n",
      "384000\n",
      "385000\n",
      "386000\n",
      "387000\n",
      "388000\n",
      "389000\n",
      "390000\n",
      "391000\n",
      "392000\n",
      "393000\n",
      "394000\n",
      "395000\n",
      "396000\n",
      "397000\n",
      "398000\n",
      "399000\n",
      "400000\n",
      "401000\n",
      "402000\n",
      "403000\n",
      "404000\n",
      "405000\n",
      "406000\n",
      "407000\n",
      "408000\n",
      "409000\n",
      "410000\n",
      "411000\n",
      "412000\n",
      "413000\n",
      "414000\n",
      "415000\n",
      "416000\n",
      "417000\n",
      "418000\n",
      "419000\n",
      "420000\n",
      "421000\n",
      "422000\n",
      "423000\n",
      "424000\n",
      "425000\n",
      "426000\n",
      "427000\n",
      "428000\n",
      "429000\n",
      "430000\n",
      "431000\n",
      "432000\n",
      "433000\n",
      "434000\n",
      "435000\n",
      "436000\n",
      "437000\n",
      "438000\n",
      "439000\n",
      "440000\n",
      "441000\n",
      "442000\n",
      "443000\n",
      "444000\n",
      "445000\n",
      "446000\n",
      "447000\n",
      "448000\n",
      "449000\n",
      "450000\n",
      "451000\n",
      "452000\n",
      "453000\n",
      "454000\n",
      "455000\n",
      "456000\n",
      "457000\n",
      "458000\n",
      "459000\n",
      "460000\n",
      "461000\n",
      "462000\n",
      "463000\n",
      "464000\n",
      "465000\n",
      "466000\n",
      "467000\n",
      "468000\n",
      "469000\n",
      "470000\n",
      "471000\n",
      "472000\n",
      "473000\n",
      "474000\n",
      "475000\n",
      "476000\n",
      "477000\n",
      "478000\n",
      "479000\n",
      "480000\n",
      "481000\n",
      "482000\n",
      "483000\n",
      "484000\n",
      "485000\n",
      "486000\n",
      "487000\n",
      "488000\n",
      "489000\n",
      "490000\n",
      "491000\n",
      "492000\n",
      "493000\n",
      "494000\n",
      "495000\n",
      "496000\n",
      "497000\n",
      "498000\n",
      "499000\n",
      "500000\n",
      "501000\n",
      "502000\n",
      "503000\n",
      "504000\n",
      "505000\n",
      "506000\n",
      "507000\n",
      "508000\n",
      "509000\n",
      "510000\n",
      "511000\n",
      "512000\n",
      "513000\n",
      "514000\n",
      "515000\n",
      "516000\n",
      "517000\n",
      "518000\n",
      "519000\n",
      "520000\n",
      "521000\n",
      "522000\n",
      "523000\n",
      "524000\n",
      "525000\n",
      "526000\n",
      "527000\n",
      "528000\n",
      "529000\n",
      "530000\n",
      "531000\n",
      "532000\n",
      "533000\n",
      "534000\n",
      "535000\n",
      "536000\n",
      "537000\n",
      "538000\n",
      "539000\n",
      "540000\n",
      "541000\n",
      "542000\n",
      "543000\n",
      "544000\n",
      "545000\n",
      "546000\n",
      "547000\n",
      "548000\n",
      "549000\n",
      "550000\n",
      "551000\n",
      "552000\n",
      "553000\n",
      "554000\n",
      "555000\n",
      "556000\n",
      "557000\n",
      "558000\n",
      "559000\n",
      "560000\n",
      "561000\n",
      "562000\n",
      "563000\n",
      "564000\n",
      "565000\n",
      "566000\n",
      "567000\n",
      "568000\n",
      "569000\n",
      "570000\n",
      "571000\n",
      "572000\n",
      "573000\n",
      "574000\n",
      "575000\n",
      "576000\n",
      "577000\n",
      "578000\n",
      "579000\n",
      "580000\n",
      "581000\n",
      "582000\n",
      "583000\n",
      "584000\n",
      "585000\n",
      "586000\n",
      "587000\n",
      "588000\n",
      "589000\n",
      "590000\n",
      "591000\n",
      "592000\n",
      "593000\n",
      "594000\n",
      "595000\n",
      "596000\n",
      "597000\n",
      "598000\n",
      "599000\n",
      "600000\n",
      "601000\n",
      "602000\n",
      "603000\n",
      "604000\n",
      "605000\n",
      "606000\n",
      "607000\n",
      "608000\n",
      "609000\n",
      "610000\n",
      "611000\n",
      "612000\n",
      "613000\n",
      "614000\n",
      "615000\n",
      "616000\n",
      "617000\n",
      "618000\n",
      "619000\n",
      "620000\n",
      "621000\n",
      "622000\n",
      "623000\n",
      "624000\n",
      "625000\n",
      "626000\n",
      "627000\n",
      "628000\n",
      "629000\n",
      "630000\n",
      "631000\n",
      "632000\n",
      "633000\n",
      "634000\n",
      "635000\n",
      "636000\n",
      "637000\n",
      "638000\n",
      "639000\n",
      "640000\n",
      "641000\n",
      "642000\n",
      "643000\n",
      "644000\n",
      "645000\n",
      "646000\n",
      "647000\n",
      "648000\n",
      "649000\n",
      "650000\n",
      "651000\n",
      "652000\n",
      "653000\n",
      "654000\n",
      "655000\n",
      "656000\n",
      "657000\n",
      "658000\n",
      "659000\n",
      "660000\n",
      "661000\n",
      "662000\n",
      "663000\n",
      "664000\n",
      "665000\n",
      "666000\n",
      "667000\n",
      "668000\n",
      "669000\n",
      "670000\n",
      "671000\n",
      "672000\n",
      "673000\n",
      "674000\n",
      "675000\n",
      "676000\n",
      "677000\n",
      "678000\n",
      "679000\n",
      "680000\n",
      "681000\n",
      "682000\n",
      "683000\n",
      "684000\n",
      "685000\n",
      "686000\n",
      "687000\n",
      "688000\n",
      "689000\n",
      "690000\n",
      "691000\n",
      "692000\n",
      "693000\n",
      "694000\n",
      "695000\n",
      "696000\n",
      "697000\n",
      "698000\n",
      "699000\n",
      "700000\n",
      "701000\n",
      "702000\n",
      "703000\n",
      "704000\n",
      "705000\n",
      "706000\n",
      "707000\n",
      "708000\n",
      "709000\n",
      "710000\n",
      "711000\n",
      "712000\n",
      "713000\n",
      "714000\n",
      "715000\n",
      "716000\n",
      "717000\n",
      "718000\n",
      "719000\n",
      "720000\n",
      "721000\n",
      "722000\n",
      "723000\n",
      "724000\n",
      "725000\n",
      "726000\n",
      "727000\n",
      "728000\n",
      "729000\n",
      "730000\n",
      "731000\n",
      "732000\n",
      "733000\n",
      "734000\n",
      "735000\n",
      "736000\n",
      "737000\n",
      "738000\n",
      "739000\n",
      "740000\n",
      "741000\n",
      "742000\n",
      "743000\n",
      "744000\n",
      "745000\n",
      "746000\n",
      "747000\n",
      "748000\n",
      "749000\n",
      "750000\n",
      "751000\n",
      "752000\n",
      "753000\n",
      "754000\n",
      "755000\n",
      "756000\n",
      "757000\n",
      "758000\n",
      "759000\n",
      "760000\n",
      "761000\n",
      "762000\n",
      "763000\n",
      "764000\n",
      "765000\n",
      "766000\n",
      "767000\n",
      "768000\n",
      "769000\n",
      "770000\n",
      "771000\n",
      "772000\n",
      "773000\n",
      "774000\n",
      "775000\n",
      "776000\n",
      "777000\n",
      "778000\n",
      "779000\n",
      "780000\n",
      "781000\n",
      "782000\n",
      "783000\n",
      "784000\n",
      "785000\n",
      "786000\n",
      "787000\n",
      "788000\n",
      "789000\n",
      "790000\n",
      "791000\n",
      "792000\n",
      "793000\n",
      "794000\n",
      "795000\n",
      "796000\n",
      "797000\n",
      "798000\n",
      "799000\n",
      "800000\n",
      "801000\n",
      "802000\n",
      "803000\n",
      "804000\n",
      "805000\n",
      "806000\n",
      "807000\n",
      "808000\n",
      "809000\n",
      "810000\n",
      "811000\n",
      "812000\n",
      "813000\n",
      "814000\n",
      "815000\n",
      "816000\n",
      "817000\n",
      "818000\n",
      "819000\n",
      "820000\n",
      "821000\n",
      "822000\n",
      "823000\n",
      "824000\n",
      "825000\n",
      "826000\n",
      "827000\n",
      "828000\n",
      "829000\n",
      "830000\n",
      "831000\n",
      "832000\n",
      "833000\n",
      "834000\n",
      "835000\n",
      "836000\n",
      "837000\n",
      "838000\n",
      "839000\n",
      "840000\n",
      "841000\n",
      "842000\n",
      "843000\n",
      "844000\n",
      "845000\n",
      "846000\n",
      "847000\n",
      "848000\n",
      "849000\n",
      "850000\n",
      "851000\n",
      "852000\n",
      "853000\n",
      "854000\n",
      "855000\n",
      "856000\n",
      "857000\n",
      "858000\n",
      "859000\n",
      "860000\n",
      "861000\n",
      "862000\n",
      "863000\n",
      "864000\n",
      "865000\n",
      "866000\n",
      "867000\n",
      "868000\n",
      "869000\n",
      "870000\n",
      "871000\n",
      "872000\n",
      "873000\n",
      "874000\n",
      "875000\n",
      "876000\n",
      "877000\n",
      "878000\n",
      "879000\n",
      "880000\n",
      "881000\n",
      "882000\n",
      "883000\n",
      "884000\n",
      "885000\n",
      "886000\n",
      "887000\n",
      "888000\n",
      "889000\n",
      "890000\n",
      "891000\n",
      "892000\n",
      "893000\n",
      "894000\n",
      "895000\n",
      "896000\n",
      "897000\n",
      "898000\n",
      "899000\n",
      "900000\n",
      "901000\n",
      "902000\n",
      "903000\n",
      "904000\n",
      "905000\n",
      "906000\n",
      "907000\n",
      "908000\n",
      "909000\n",
      "910000\n",
      "911000\n",
      "912000\n",
      "913000\n",
      "914000\n",
      "915000\n",
      "916000\n",
      "917000\n",
      "918000\n",
      "919000\n",
      "920000\n",
      "921000\n",
      "922000\n",
      "923000\n",
      "924000\n",
      "925000\n",
      "926000\n",
      "927000\n",
      "928000\n",
      "929000\n",
      "930000\n",
      "931000\n",
      "932000\n",
      "933000\n",
      "934000\n",
      "935000\n",
      "936000\n",
      "937000\n",
      "938000\n",
      "939000\n",
      "940000\n",
      "941000\n",
      "942000\n",
      "943000\n",
      "944000\n",
      "945000\n",
      "946000\n",
      "947000\n",
      "948000\n",
      "949000\n",
      "950000\n",
      "951000\n",
      "952000\n",
      "953000\n",
      "954000\n",
      "955000\n",
      "956000\n",
      "957000\n",
      "958000\n",
      "959000\n",
      "960000\n",
      "961000\n",
      "962000\n",
      "963000\n",
      "964000\n",
      "965000\n",
      "966000\n",
      "967000\n",
      "968000\n",
      "969000\n",
      "970000\n",
      "971000\n",
      "972000\n",
      "973000\n",
      "974000\n",
      "975000\n",
      "976000\n",
      "977000\n",
      "978000\n",
      "979000\n",
      "980000\n",
      "981000\n",
      "982000\n",
      "983000\n",
      "984000\n",
      "985000\n",
      "986000\n",
      "987000\n",
      "988000\n",
      "989000\n",
      "990000\n",
      "991000\n",
      "992000\n",
      "993000\n",
      "994000\n",
      "995000\n",
      "996000\n",
      "997000\n",
      "998000\n",
      "999000\n",
      "1000000\n",
      "1001000\n",
      "1002000\n",
      "1003000\n",
      "1004000\n",
      "1005000\n",
      "1006000\n",
      "1007000\n",
      "1008000\n",
      "1009000\n",
      "1010000\n",
      "1011000\n",
      "1012000\n",
      "1013000\n",
      "1014000\n",
      "1015000\n",
      "1016000\n",
      "1017000\n",
      "1018000\n",
      "1019000\n",
      "1020000\n",
      "1021000\n",
      "1022000\n",
      "1023000\n",
      "1024000\n",
      "1025000\n",
      "1026000\n",
      "1027000\n",
      "1028000\n",
      "1029000\n",
      "1030000\n",
      "1031000\n",
      "1032000\n",
      "1033000\n",
      "1034000\n",
      "1035000\n",
      "1036000\n",
      "1037000\n",
      "1038000\n",
      "1039000\n",
      "1040000\n",
      "1041000\n",
      "1042000\n",
      "1043000\n",
      "1044000\n",
      "1045000\n",
      "1046000\n",
      "1047000\n",
      "1048000\n",
      "1049000\n",
      "1050000\n",
      "1051000\n",
      "1052000\n",
      "1053000\n",
      "1054000\n",
      "1055000\n",
      "1056000\n",
      "1057000\n",
      "1058000\n",
      "1059000\n",
      "1060000\n",
      "1061000\n",
      "1062000\n",
      "1063000\n",
      "1064000\n",
      "1065000\n",
      "1066000\n",
      "1067000\n",
      "1068000\n",
      "1069000\n",
      "1070000\n",
      "1071000\n",
      "1072000\n",
      "1073000\n",
      "1074000\n",
      "1075000\n",
      "1076000\n",
      "1077000\n",
      "1078000\n",
      "1079000\n",
      "1080000\n",
      "1081000\n",
      "1082000\n",
      "1083000\n",
      "1084000\n",
      "1085000\n",
      "1086000\n",
      "1087000\n",
      "1088000\n",
      "1089000\n",
      "1090000\n",
      "1091000\n",
      "1092000\n",
      "1093000\n",
      "1094000\n",
      "1095000\n",
      "1096000\n",
      "1097000\n",
      "1098000\n",
      "1099000\n",
      "1100000\n",
      "1101000\n",
      "1102000\n",
      "1103000\n",
      "1104000\n",
      "1105000\n",
      "1106000\n",
      "1107000\n",
      "1108000\n",
      "1109000\n",
      "1110000\n",
      "1111000\n",
      "1112000\n",
      "1113000\n",
      "1114000\n",
      "1115000\n",
      "1116000\n",
      "1117000\n",
      "1118000\n",
      "1119000\n",
      "1120000\n",
      "1121000\n",
      "1122000\n",
      "1123000\n",
      "1124000\n",
      "1125000\n",
      "1126000\n",
      "1127000\n",
      "1128000\n",
      "1129000\n",
      "1130000\n",
      "1131000\n",
      "1132000\n",
      "1133000\n",
      "1134000\n",
      "1135000\n",
      "1136000\n",
      "1137000\n",
      "1138000\n",
      "1139000\n",
      "1140000\n",
      "1141000\n",
      "1142000\n",
      "1143000\n",
      "1144000\n",
      "1145000\n",
      "1146000\n",
      "1147000\n",
      "1148000\n",
      "1149000\n",
      "1150000\n",
      "1151000\n",
      "1152000\n",
      "1153000\n",
      "1154000\n",
      "1155000\n",
      "1156000\n",
      "1157000\n",
      "1158000\n",
      "1159000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1160000\n",
      "1161000\n",
      "1162000\n",
      "1163000\n",
      "1164000\n",
      "1165000\n",
      "1166000\n",
      "1167000\n",
      "1168000\n",
      "1169000\n",
      "1170000\n",
      "1171000\n",
      "1172000\n",
      "1173000\n",
      "1174000\n",
      "1175000\n",
      "1176000\n",
      "1177000\n",
      "1178000\n",
      "1179000\n",
      "1180000\n",
      "1181000\n",
      "1182000\n",
      "1183000\n",
      "1184000\n",
      "1185000\n",
      "1186000\n",
      "1187000\n",
      "1188000\n",
      "1189000\n",
      "1190000\n",
      "1191000\n",
      "1192000\n",
      "1193000\n",
      "1194000\n",
      "1195000\n",
      "1196000\n",
      "1197000\n",
      "1198000\n",
      "1199000\n",
      "1200000\n",
      "1201000\n",
      "1202000\n",
      "1203000\n",
      "1204000\n",
      "1205000\n",
      "1206000\n",
      "1207000\n",
      "1208000\n",
      "1209000\n",
      "1210000\n",
      "1211000\n",
      "1212000\n",
      "1213000\n",
      "1214000\n",
      "1215000\n",
      "1216000\n",
      "1217000\n",
      "1218000\n",
      "1219000\n",
      "1220000\n",
      "1221000\n",
      "1222000\n",
      "1223000\n",
      "1224000\n",
      "1225000\n",
      "1226000\n",
      "1227000\n",
      "1228000\n",
      "1229000\n",
      "1230000\n",
      "1231000\n",
      "1232000\n",
      "1233000\n",
      "1234000\n",
      "1235000\n",
      "1236000\n",
      "1237000\n",
      "1238000\n",
      "1239000\n",
      "1240000\n",
      "1241000\n",
      "1242000\n",
      "1243000\n",
      "1244000\n",
      "1245000\n",
      "1246000\n",
      "1247000\n",
      "1248000\n",
      "1249000\n",
      "1250000\n",
      "1251000\n",
      "1252000\n",
      "1253000\n",
      "1254000\n",
      "1255000\n",
      "1256000\n",
      "1257000\n",
      "1258000\n",
      "1259000\n",
      "1260000\n",
      "1261000\n",
      "1262000\n",
      "1263000\n",
      "1264000\n",
      "1265000\n",
      "1266000\n",
      "1267000\n",
      "1268000\n",
      "1269000\n",
      "1270000\n",
      "1271000\n",
      "1272000\n",
      "1273000\n",
      "1274000\n",
      "1275000\n",
      "1276000\n",
      "1277000\n",
      "1278000\n",
      "1279000\n",
      "1280000\n",
      "1281000\n",
      "1282000\n",
      "1283000\n",
      "1284000\n",
      "1285000\n",
      "1286000\n",
      "1287000\n",
      "1288000\n",
      "1289000\n",
      "1290000\n",
      "1291000\n",
      "1292000\n",
      "1293000\n",
      "1294000\n",
      "1295000\n",
      "1296000\n",
      "1297000\n",
      "1298000\n",
      "1299000\n",
      "1300000\n",
      "1301000\n",
      "1302000\n",
      "1303000\n",
      "1304000\n",
      "1305000\n",
      "1306000\n",
      "1307000\n",
      "1308000\n",
      "1309000\n",
      "1310000\n",
      "1311000\n",
      "1312000\n",
      "1313000\n",
      "1314000\n",
      "1315000\n",
      "1316000\n",
      "1317000\n",
      "1318000\n",
      "1319000\n",
      "1320000\n",
      "1321000\n",
      "1322000\n",
      "1323000\n",
      "1324000\n",
      "1325000\n",
      "1326000\n",
      "1327000\n",
      "1328000\n",
      "1329000\n",
      "1330000\n",
      "1331000\n",
      "1332000\n",
      "1333000\n",
      "1334000\n",
      "1335000\n",
      "1336000\n",
      "1337000\n",
      "1338000\n",
      "1339000\n",
      "1340000\n",
      "1341000\n",
      "1342000\n",
      "1343000\n",
      "1344000\n",
      "1345000\n",
      "1346000\n",
      "1347000\n",
      "1348000\n",
      "1349000\n",
      "1350000\n",
      "1351000\n",
      "1352000\n",
      "1353000\n",
      "1354000\n",
      "1355000\n",
      "1356000\n",
      "1357000\n",
      "1358000\n",
      "1359000\n",
      "1360000\n",
      "1361000\n",
      "1362000\n",
      "1363000\n",
      "1364000\n",
      "1365000\n",
      "1366000\n",
      "1367000\n",
      "1368000\n",
      "1369000\n",
      "1370000\n",
      "1371000\n",
      "1372000\n",
      "1373000\n",
      "1374000\n",
      "1375000\n",
      "1376000\n",
      "1377000\n",
      "1378000\n",
      "1379000\n",
      "1380000\n",
      "1381000\n",
      "1382000\n",
      "1383000\n",
      "1384000\n",
      "1385000\n",
      "1386000\n",
      "1387000\n",
      "1388000\n",
      "1389000\n",
      "1390000\n",
      "1391000\n",
      "1392000\n",
      "1393000\n",
      "1394000\n",
      "1395000\n",
      "1396000\n",
      "1397000\n",
      "1398000\n",
      "1399000\n",
      "1400000\n",
      "1401000\n",
      "1402000\n",
      "1403000\n",
      "1404000\n",
      "1405000\n",
      "1406000\n",
      "1407000\n",
      "1408000\n",
      "1409000\n",
      "1410000\n",
      "1411000\n",
      "1412000\n",
      "1413000\n",
      "1414000\n",
      "1415000\n",
      "1416000\n",
      "1417000\n",
      "1418000\n",
      "1419000\n",
      "1420000\n",
      "1421000\n",
      "1422000\n",
      "1423000\n",
      "1424000\n",
      "1425000\n",
      "1426000\n",
      "1427000\n",
      "1428000\n",
      "1429000\n",
      "1430000\n",
      "1431000\n",
      "1432000\n",
      "1433000\n",
      "1434000\n",
      "1435000\n",
      "1436000\n",
      "1437000\n",
      "1438000\n",
      "1439000\n",
      "1440000\n",
      "1441000\n",
      "1442000\n",
      "1443000\n",
      "1444000\n",
      "1445000\n",
      "1446000\n",
      "1447000\n",
      "1448000\n",
      "1449000\n",
      "1450000\n",
      "1451000\n",
      "1452000\n",
      "1453000\n",
      "1454000\n",
      "1455000\n",
      "1456000\n",
      "1457000\n",
      "1458000\n",
      "1459000\n",
      "1460000\n",
      "1461000\n",
      "1462000\n",
      "1463000\n",
      "1464000\n",
      "1465000\n",
      "1466000\n",
      "1467000\n",
      "1468000\n",
      "1469000\n",
      "1470000\n",
      "1471000\n",
      "1472000\n",
      "1473000\n",
      "1474000\n",
      "1475000\n",
      "1476000\n",
      "1477000\n",
      "1478000\n",
      "1479000\n",
      "1480000\n",
      "1481000\n",
      "1482000\n",
      "1483000\n",
      "1484000\n",
      "1485000\n",
      "1486000\n",
      "1487000\n",
      "1488000\n",
      "1489000\n",
      "1490000\n",
      "1491000\n",
      "1492000\n",
      "1493000\n",
      "1494000\n",
      "1495000\n",
      "1496000\n",
      "1497000\n",
      "1498000\n",
      "1499000\n",
      "1500000\n",
      "1501000\n",
      "1502000\n",
      "1503000\n",
      "1504000\n",
      "1505000\n",
      "1506000\n",
      "1507000\n",
      "1508000\n",
      "1509000\n",
      "1510000\n",
      "1511000\n",
      "1512000\n",
      "1513000\n",
      "1514000\n",
      "1515000\n",
      "1516000\n",
      "1517000\n",
      "1518000\n",
      "1519000\n",
      "1520000\n",
      "1521000\n",
      "1522000\n",
      "1523000\n",
      "1524000\n",
      "1525000\n",
      "1526000\n",
      "1527000\n",
      "1528000\n",
      "1529000\n",
      "1530000\n",
      "1531000\n",
      "1532000\n",
      "1533000\n",
      "1534000\n",
      "1535000\n",
      "1536000\n",
      "1537000\n",
      "1538000\n",
      "1539000\n",
      "1540000\n",
      "1541000\n",
      "1542000\n",
      "1543000\n",
      "1544000\n",
      "1545000\n",
      "1546000\n",
      "1547000\n",
      "1548000\n",
      "1549000\n",
      "1550000\n",
      "1551000\n",
      "1552000\n",
      "1553000\n",
      "1554000\n",
      "1555000\n",
      "1556000\n",
      "1557000\n",
      "1558000\n",
      "1559000\n",
      "1560000\n",
      "1561000\n",
      "1562000\n",
      "1563000\n",
      "1564000\n",
      "1565000\n",
      "1566000\n",
      "1567000\n",
      "1568000\n",
      "1569000\n",
      "1570000\n",
      "1571000\n",
      "1572000\n",
      "1573000\n",
      "1574000\n",
      "1575000\n",
      "1576000\n",
      "1577000\n",
      "1578000\n"
     ]
    }
   ],
   "source": [
    "dataset_location = 'data/tweets.csv'\n",
    "model_location = './'\n",
    "\n",
    "corpus = []\n",
    "labels = []\n",
    "\n",
    "# Parse tweets and sentiments\n",
    "with open(dataset_location, 'r', encoding='utf-8') as df:\n",
    "    for i, line in enumerate(df):\n",
    "        if i == 0:\n",
    "            # Skip the header\n",
    "            continue\n",
    "\n",
    "        parts = line.strip().split(',')\n",
    "        \n",
    "        # Sentiment (0 = Negative, 1 = Positive)\n",
    "        labels.append(int(parts[1].strip()))\n",
    "        \n",
    "        # Tweet\n",
    "        tweet = parts[3].strip()\n",
    "        if tweet.startswith('\"'):\n",
    "            tweet = tweet[1:]\n",
    "        if tweet.endswith('\"'):\n",
    "            tweet = tweet[::-1]\n",
    "        \n",
    "        corpus.append(tweet.strip().lower())\n",
    "        \n",
    "print('Tamanho do Corpus: {}'.format(len(corpus)))\n",
    "\n",
    "tkr = RegexpTokenizer('[a-zA-Z0-9@]+')\n",
    "stemmer = LancasterStemmer()\n",
    "\n",
    "tokenized_corpus = []\n",
    "\n",
    "for i, tweet in enumerate(corpus):\n",
    "    if(i%1000==0):\n",
    "        print(i)\n",
    "    tokens = [stemmer.stem(t) for t in tkr.tokenize(tweet) if not t.startswith('@')]\n",
    "    tokenized_corpus.append(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Como a base é relativamente grande, poderíamos treinar nosso próprio Word2Vec:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vector_size = 100\n",
    "window_size = 5\n",
    "\n",
    "# Create Word2Vec\n",
    "word2vec = Word2Vec(sentences=tokenized_corpus,\n",
    "                    size=vector_size, \n",
    "                    window=window_size, \n",
    "                    negative=20,\n",
    "                    iter=50,\n",
    "                    seed=1000)\n",
    "\n",
    "# Copy word vectors and delete Word2Vec model  and original corpus to save memory\n",
    "X_vecs = word2vec.wv\n",
    "del word2vec\n",
    "del corpus\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Contudo, como a aula é curta, vamos usar os vetores pré treinados da GloVe (https://nlp.stanford.edu/projects/glove/):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "glove = 'data/glove.twitter.27B/glove.twitter.27B.100d.txt'\n",
    "import pandas as pd\n",
    "import csv\n",
    "\n",
    "words = pd.read_table(glove, sep=\" \", index_col=0, header=None, quoting=csv.QUOTE_NONE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_vecs={}\n",
    "for t in tokenized_corpus:\n",
    "    for i in t:\n",
    "        if i not in X_vecs and i in words.index:\n",
    "            X_vecs[i]=words.loc[i].as_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X_vecs['cool'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average tweet length: 11.02887762593697\n",
      "Max tweet length: 53\n"
     ]
    }
   ],
   "source": [
    "train_size = 1000000\n",
    "\n",
    "# Test subset size (0 < size < len(tokenized_corpus) - train_size)\n",
    "test_size = 100000\n",
    "\n",
    "avg_length = 0.0\n",
    "max_length = 0\n",
    "\n",
    "for tweet in tokenized_corpus:\n",
    "    if len(tweet) > max_length:\n",
    "        max_length = len(tweet)\n",
    "    avg_length += float(len(tweet))\n",
    "    \n",
    "print('Average tweet length: {}'.format(avg_length / float(len(tokenized_corpus))))\n",
    "print('Max tweet length: {}'.format(max_length))\n",
    "\n",
    "# Tweet max length (number of tokens)\n",
    "max_tweet_length = 15\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = 1000000\n",
    "vector_size=100\n",
    "\n",
    "# Test subset size (0 < size < len(tokenized_corpus) - train_size)\n",
    "test_size = 100000\n",
    "\n",
    "indexes = set(np.random.choice(len(tokenized_corpus), train_size + test_size, replace=False))\n",
    "\n",
    "X_train = np.zeros((train_size, max_tweet_length, vector_size), dtype=K.floatx())\n",
    "Y_train = np.zeros((train_size, 2), dtype=np.int32)\n",
    "X_test = np.zeros((test_size, max_tweet_length, vector_size), dtype=K.floatx())\n",
    "Y_test = np.zeros((test_size, 2), dtype=np.int32)\n",
    "\n",
    "for i, index in enumerate(indexes):\n",
    "    for t, token in enumerate(tokenized_corpus[index]):\n",
    "        if t >= max_tweet_length:\n",
    "            break\n",
    "        \n",
    "        if token not in X_vecs:\n",
    "            continue\n",
    "    \n",
    "        if i < train_size:\n",
    "            X_train[i, t, :] = X_vecs[token]\n",
    "        else:\n",
    "            X_test[i - train_size, t, :] = X_vecs[token]\n",
    "            \n",
    "    if i < train_size:\n",
    "        Y_train[i, :] = [1.0, 0.0] if labels[index] == 0 else [0.0, 1.0]\n",
    "    else:\n",
    "        Y_test[i - train_size, :] = [1.0, 0.0] if labels[index] == 0 else [0.0, 1.0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Com isto, temos a nossa entrada como um tensor 1000000 x 15 x 100, e a saída como labels one-hot.\n",
    "\n",
    "### Exercício ###\n",
    "\n",
    "Façam em Keras um modelo de predição para estes dados."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_16 (Dense)             (None, 100)               150100    \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 100)               10100     \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 2)                 202       \n",
      "=================================================================\n",
      "Total params: 160,802\n",
      "Trainable params: 160,602\n",
      "Non-trainable params: 200\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "input_dimension = 100*15\n",
    "hidden1=100\n",
    "output=2\n",
    "dropout=0.2\n",
    "batch_size=128\n",
    "epochs=3\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "model = Sequential()\n",
    "model.add(Dense(hidden1,input_dim=input_dimension, activation='relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(dropout))\n",
    "model.add(Dense(hidden1,input_dim=input_dimension, activation='relu'))\n",
    "model.add(Dropout(dropout))\n",
    "model.add(Dense(2, activation='softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000000"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 800000 samples, validate on 200000 samples\n",
      "Epoch 1/3\n",
      "800000/800000 [==============================] - 45s 56us/step - loss: 0.5854 - acc: 0.6870 - val_loss: 0.5711 - val_acc: 0.7000\n",
      "Epoch 2/3\n",
      "800000/800000 [==============================] - 44s 55us/step - loss: 0.5583 - acc: 0.7104 - val_loss: 0.5573 - val_acc: 0.7095\n",
      "Epoch 3/3\n",
      "800000/800000 [==============================] - 44s 55us/step - loss: 0.5451 - acc: 0.7209 - val_loss: 0.5563 - val_acc: 0.7091\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x23c185ed278>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tboard = keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=batch_size, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names='embedding_1', embeddings_metadata=None, embeddings_data=None)\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.fit(X_train.reshape(X_train.shape[0],-1), Y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_split=0.2, callbacks=[tboard])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}