03_criar_sentencas_treinamento.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re \n",
    "import os\n",
    "import nltk \n",
    "from sklearn import feature_extraction\n",
    "from gensim import models\n",
    "import numpy as np\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('modelos/docs_tokenizados_vagalume.pkl', 'rb') as fp:\n",
    "    docs_tokenizados = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47128"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "palavras = []\n",
    "for doc in docs_tokenizados:\n",
    "    palavras.extend(doc)\n",
    "unicas = set(palavras)\n",
    "len(unicas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "caminho_modelo_w2v = \"modelos/w2v_sertanejo_28kmusicas_tweet_tknzr_window_15_mincount_10.model\"\n",
    "if not os.path.isfile(caminho_modelo_w2v):\n",
    "    raise FileExistsError(\"Modelo w2v nao encontrado.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12082"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v_model = models.Word2Vec.load(caminho_modelo_w2v)\n",
    "vocab = list(w2v_model.wv.vocab)\n",
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Funcoes que retornam o indice da palavra no vocabulario criado pelo word2vec e vice-versa\n",
    "def word2idx(word):\n",
    "    return w2v_model.wv.vocab[word].index\n",
    "\n",
    "\n",
    "def idx2word(idx):\n",
    "    return w2v_model.wv.index2word[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3764843"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxlen = 15\n",
    "sentencas = []\n",
    "next_tokens = []\n",
    "\n",
    "for doc in docs_tokenizados:  # Cria as \"sentencas\" para cada documento do corpus\n",
    "    for i in range(0, len(doc) - maxlen):\n",
    "        \n",
    "        lista_palavras = doc[i:i + maxlen]\n",
    "        \n",
    "        excluir_sentenca = False\n",
    "        for palavra in lista_palavras:\n",
    "            if palavra not in w2v_model.wv.vocab:\n",
    "                excluir_sentenca = True\n",
    "                break\n",
    "                \n",
    "        proxima_palavra = doc[i + maxlen]\n",
    "        if proxima_palavra not in w2v_model.wv.vocab:\n",
    "            excluir_sentenca = True\n",
    "                \n",
    "        if excluir_sentenca:\n",
    "            continue\n",
    "        \n",
    "        sentencas.append(lista_palavras)\n",
    "        next_tokens.append(proxima_palavra)\n",
    "\n",
    "len(sentencas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Criando os inputs(x) e targets(y)\n",
    "x = np.zeros((len(sentencas), maxlen), dtype=np.int32)\n",
    "y = np.zeros((len(sentencas)), dtype=np.int32)\n",
    "for i, sentenca in enumerate(sentencas):\n",
    "    for t, token in enumerate(sentenca):\n",
    "        x[i, t] = word2idx(token)\n",
    "    y[i] = word2idx(next_tokens[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "dados_treino = [x,y]\n",
    "with open('modelos/dados_treino_mincount_10_vagalume.pkl', 'wb') as fp:\n",
    "    pickle.dump(dados_treino, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}