02_criar_modelo_fasttext_musicas.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/daniel/dev/venvs/p36/lib/python3.6/site-packages/sklearn/utils/fixes.py:55: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()\n",
      "  if 'order' in inspect.getargspec(np.copy)[0]:\n"
     ]
    }
   ],
   "source": [
    "import re \n",
    "from bs4 import BeautifulSoup \n",
    "import os\n",
    "import nltk \n",
    "from sklearn import feature_extraction\n",
    "from gensim import models\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "documentos = []\n",
    "diretorio = \"dados/letras-musicas/vagalume/sertanejo/\"\n",
    "arquivos = os.listdir(diretorio)\n",
    "for arquivo in arquivos:\n",
    "    caminho = diretorio + arquivo\n",
    "    if os.path.isfile(caminho) and \".html\" in arquivo:\n",
    "        with open(caminho,\"r\") as f:\n",
    "            html = f.read()\n",
    "        html = feature_extraction.text.strip_accents_ascii(html.lower())\n",
    "        html = html.replace(\"</br>\",\"\")\n",
    "        html = html.replace(\"<br>\",\" NLINHA \")\n",
    "        html = html.replace(\"<p>\",\" NLINHA \")\n",
    "        soup = BeautifulSoup(html,\"html.parser\")\n",
    "        documentos.append(soup.text.strip().lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs_tokenizados = []\n",
    "tokenizador = nltk.TweetTokenizer()\n",
    "for doc in documentos:\n",
    "    tokens = tokenizador.tokenize(doc)\n",
    "    docs_tokenizados.append(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Salva os documentos tokenizados em formato binario\n",
    "import pickle\n",
    "\n",
    "with open('modelos/docs_tokenizados_fasttext_vagalume.pkl', 'wb') as fp:\n",
    "    pickle.dump(docs_tokenizados, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47128"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "palavras = []\n",
    "for doc in docs_tokenizados:\n",
    "    palavras.extend(doc)\n",
    "unicas = set(palavras)\n",
    "len(unicas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.FastText(docs_tokenizados,size=350,window=15,min_count=10,workers=os.cpu_count(),iter=300,min_n=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Treinando um modelo Word2vec nos documentos\n",
    "w2v_model = models.Word2Vec(docs_tokenizados, size=350, window=15, min_count=10, workers=os.cpu_count(),iter=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "len(w2v_model.wv.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cria um array com o vetor das duas mil primeiras palavras\n",
    "vocab = list(w2v_model.wv.vocab)[:2000]\n",
    "vetores_w2v = w2v_model.wv[vocab]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vetores_w2v = np.asfarray(vetores_w2v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_model.most_similar(positive=[\"amor\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reduzindo a dimensionalidade dos vetores das palavras\n",
    "from sklearn.manifold import TSNE\n",
    "tsne = TSNE(n_components=2)\n",
    "vetores_reduzidos = tsne.fit_transform(vetores_w2v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cria um dataframe com as colunas x e y. O objetivo é transformar cada representação de uma palavra em uma coordenada.\n",
    "import pandas as pd\n",
    "df = pd.DataFrame(vetores_reduzidos, index=vocab, columns=['x', 'y'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Utiliza o matplotlib com a chave de interação ligada.\n",
    "%matplotlib notebook\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(1, 1, 1)\n",
    "\n",
    "for word, pos in df.iterrows():\n",
    "    ax.annotate(word, pos) # Coloca a palavra em cada ponto\n",
    "\n",
    "ax.scatter(df['x'], df['y']) # Cria um scatterPlot com todos os pontos x e y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_model.save(\"modelos/w2v_sertanejo_28kmusicas_tweet_tknzr_window_15_mincount_10.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_model.wv.vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}