RNN_char.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "LSTM.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.2"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "R8FnhxiWY_Y1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "e553b587-eaf8-4624-9351-392963aba19a"
      },
      "source": [
        "import time\n",
        "import re\n",
        "import os\n",
        "import math\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from matplotlib import pyplot as plt\n",
        "import requests\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import (\n",
        "    Input, Reshape, BatchNormalization, Dense, Dropout,       # General\n",
        "    Embedding, LSTM, Dense,                              # RNN\n",
        ")\n",
        "from tensorflow.keras.activations import elu, relu, softmax, sigmoid\n",
        "\n",
        "print(tf.__version__)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.3.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fe1V83bpI8Tf",
        "colab_type": "text"
      },
      "source": [
        "## **Preprocessing**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "V5YiWU0daYkT",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "6850226a-0b93-426c-b35e-2617afc4e1d0"
      },
      "source": [
        "# Read the Divina Commedia\n",
        "url = \"https://raw.githubusercontent.com/DanieleVeri/deep_comedy/ea3c999bee1ea27700c61c36e00f9dc8a768e5ee/divina_commedia.txt\"\n",
        "response = requests.get(url)\n",
        "\n",
        "divina_commedia = response.text\n",
        "\n",
        "# Replace rare characters\n",
        "divina_commedia = divina_commedia.replace(\"ä\", \"a\")\n",
        "divina_commedia = divina_commedia.replace(\"é\", \"è\")\n",
        "divina_commedia = divina_commedia.replace(\"ë\", \"è\")\n",
        "divina_commedia = divina_commedia.replace(\"Ë\", \"E\")\n",
        "divina_commedia = divina_commedia.replace(\"ï\", \"i\")\n",
        "divina_commedia = divina_commedia.replace(\"Ï\", \"I\")\n",
        "divina_commedia = divina_commedia.replace(\"ó\", \"ò\")\n",
        "divina_commedia = divina_commedia.replace(\"ö\", \"o\")\n",
        "divina_commedia = divina_commedia.replace(\"ü\", \"u\")\n",
        "divina_commedia = divina_commedia.replace(\"(\", \"-\")\n",
        "divina_commedia = divina_commedia.replace(\")\", \"-\")\n",
        "divina_commedia = divina_commedia.replace(\"[\", \"\")\n",
        "divina_commedia = divina_commedia.replace(\"]\", \"\")\n",
        "divina_commedia = re.sub(r'[0-9]+', '', divina_commedia)\n",
        "divina_commedia = divina_commedia.replace(\" \\n\", \"\\n\")\n",
        "\n",
        "unique_chars = list(set(divina_commedia))\n",
        "unique_chars.sort()  # to make sure you get the same encoding at each run\n",
        "char2idx = { char[1]: char[0] for char in enumerate(unique_chars) }\n",
        "\n",
        "def numerical_encoding(text, char_dict):\n",
        "    \"\"\" Text to list of chars, to np.array of numerical idx \"\"\"\n",
        "    chars_list = [ char for char in text ]\n",
        "    chars_list = [ char_dict[char] for char in chars_list ]\n",
        "    chars_list = np.array(chars_list)\n",
        "    return chars_list\n",
        "\n",
        "encoded_text = numerical_encoding(divina_commedia, char2idx)\n",
        "\n",
        "def get_text_matrix(sequence, len_input):\n",
        "    X = np.empty((len(sequence)-len_input, len_input))\n",
        "    for i in range(X.shape[0]):\n",
        "        X[i,:] = sequence[i : i+len_input]\n",
        "    return X\n",
        "\n",
        "text_matrix = get_text_matrix(encoded_text, 100)\n",
        "\n",
        "print(text_matrix.shape)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(533803, 100)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8w9Pm22BI3yD",
        "colab_type": "text"
      },
      "source": [
        "## **LSTM training**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "mKysyBfpAtUS",
        "jupyter": {
          "outputs_hidden": true
        },
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "45ae5736-30de-4282-d1d2-bd2d0097d97d"
      },
      "source": [
        "# size of vocabulary\n",
        "vocab_size = len(char2idx)\n",
        "\n",
        "# size of mini batches during training\n",
        "batch_size = 1000\n",
        "# size of training subset at each epoch\n",
        "subset_size = batch_size * 100\n",
        "embedding_size = 256\n",
        "lstm_size = 1024\n",
        "hidden_size = 256\n",
        "\n",
        "n_epochs = 25\n",
        "learning_rate = 1e-3\n",
        "\n",
        "char_input = Input(shape=(batch_size,))\n",
        "RNN = Sequential([\n",
        "    Embedding(vocab_size, embedding_size,\n",
        "              batch_input_shape=(batch_size, None)),\n",
        "    \n",
        "    LSTM(lstm_size, return_sequences = True),\n",
        "    \n",
        "    Dense(hidden_size, activation = relu), \n",
        "    \n",
        "    Dense(vocab_size)\n",
        "])\n",
        "RNN.summary()\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate = learning_rate)\n",
        "\n",
        "# This is an Autograph function\n",
        "# its decorator makes it a TF op - i.e. much faster\n",
        "@tf.function\n",
        "def train_on_batch(x, y):\n",
        "    with tf.GradientTape() as tape:\n",
        "        current_loss = tf.reduce_mean(\n",
        "            tf.keras.losses.sparse_categorical_crossentropy(\n",
        "                y, RNN(x), from_logits = True))\n",
        "    gradients = tape.gradient(current_loss, RNN.trainable_variables)\n",
        "    optimizer.apply_gradients(zip(gradients, RNN.trainable_variables))\n",
        "    return current_loss\n",
        "\n",
        "loss_history = []\n",
        "\n",
        "for epoch in range(n_epochs):\n",
        "    start = time.time()\n",
        "    print(epoch)\n",
        "    \n",
        "    # Take subsets of train and target\n",
        "    sample = np.random.randint(0, text_matrix.shape[0]-1, subset_size)\n",
        "    sample_train = text_matrix[ sample , : ]\n",
        "    sample_target = text_matrix[ sample+1 , : ]\n",
        "    \n",
        "    for iteration in range(sample_train.shape[0] // batch_size):\n",
        "        take = iteration * batch_size\n",
        "        x = sample_train[ take:take+batch_size , : ]\n",
        "        y = sample_target[ take:take+batch_size , : ]\n",
        "\n",
        "        current_loss = train_on_batch(x, y)\n",
        "        loss_history.append(current_loss)\n",
        "    \n",
        "    print(\"{}.  \\t  Loss: {}  \\t  Time: {}ss\".format(\n",
        "        epoch+1, current_loss.numpy(), round(time.time()-start, 2)))\n",
        "    \n",
        "    \n",
        "plt.plot(loss_history)\n",
        "plt.title(\"Training Loss\")\n",
        "plt.show()\n",
        "\n",
        "RNN.save(\"/text_generator_RNN_00.h5\")"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_7\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_7 (Embedding)      (1000, None, 256)         15872     \n",
            "_________________________________________________________________\n",
            "lstm_7 (LSTM)                (1000, None, 1024)        5246976   \n",
            "_________________________________________________________________\n",
            "dense_14 (Dense)             (1000, None, 256)         262400    \n",
            "_________________________________________________________________\n",
            "dense_15 (Dense)             (1000, None, 62)          15934     \n",
            "=================================================================\n",
            "Total params: 5,541,182\n",
            "Trainable params: 5,541,182\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "0\n",
            "1.  \t  Loss: 2.392106294631958  \t  Time: 55.45ss\n",
            "1\n",
            "2.  \t  Loss: 2.0514543056488037  \t  Time: 53.66ss\n",
            "2\n",
            "3.  \t  Loss: 1.8689590692520142  \t  Time: 53.72ss\n",
            "3\n",
            "4.  \t  Loss: 1.7254834175109863  \t  Time: 53.78ss\n",
            "4\n",
            "5.  \t  Loss: 1.6298097372055054  \t  Time: 53.67ss\n",
            "5\n",
            "6.  \t  Loss: 1.5405349731445312  \t  Time: 53.74ss\n",
            "6\n",
            "7.  \t  Loss: 1.4623562097549438  \t  Time: 53.75ss\n",
            "7\n",
            "8.  \t  Loss: 1.3939487934112549  \t  Time: 53.76ss\n",
            "8\n",
            "9.  \t  Loss: 1.3404340744018555  \t  Time: 53.77ss\n",
            "9\n",
            "10.  \t  Loss: 1.2664786577224731  \t  Time: 53.7ss\n",
            "10\n",
            "11.  \t  Loss: 1.2040082216262817  \t  Time: 53.77ss\n",
            "11\n",
            "12.  \t  Loss: 1.1221280097961426  \t  Time: 53.77ss\n",
            "12\n",
            "13.  \t  Loss: 1.0357787609100342  \t  Time: 53.72ss\n",
            "13\n",
            "14.  \t  Loss: 0.9377641677856445  \t  Time: 53.72ss\n",
            "14\n",
            "15.  \t  Loss: 0.8344329595565796  \t  Time: 53.68ss\n",
            "15\n",
            "16.  \t  Loss: 0.729579508304596  \t  Time: 53.6ss\n",
            "16\n",
            "17.  \t  Loss: 0.6211807727813721  \t  Time: 53.67ss\n",
            "17\n",
            "18.  \t  Loss: 0.5205257534980774  \t  Time: 53.64ss\n",
            "18\n",
            "19.  \t  Loss: 0.44165125489234924  \t  Time: 53.63ss\n",
            "19\n",
            "20.  \t  Loss: 0.3801918625831604  \t  Time: 53.67ss\n",
            "20\n",
            "21.  \t  Loss: 0.32818886637687683  \t  Time: 53.75ss\n",
            "21\n",
            "22.  \t  Loss: 0.2961314022541046  \t  Time: 53.62ss\n",
            "22\n",
            "23.  \t  Loss: 0.2754611372947693  \t  Time: 53.62ss\n",
            "23\n",
            "24.  \t  Loss: 0.2575032413005829  \t  Time: 53.71ss\n",
            "24\n",
            "25.  \t  Loss: 0.24261178076267242  \t  Time: 53.65ss\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEICAYAAAB25L6yAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAa90lEQVR4nO3de3xV5Z3v8c8v9wAJEIjcIaCAIN6Qm9WCF4qCHT1tZzr1VW+0ll7sVI9aj7V1as9oT0dH2zPH2h6tt46OqIO13mpBBdEeEQIIiNyRAJGQCIYESEKS/Tt/7B2IkMsOZO+99s73/XrllZW1nr3372HFr0+edTN3R0REgist0QWIiEjbFNQiIgGnoBYRCTgFtYhIwCmoRUQCTkEtIhJwCmoJNDP7i5ld29ltRZKJ6Txq6Wxmtr/Zj92AOqAx8vN33f3p+Fd1/MzsAuApdx+c6Fqka8pIdAGSety9R9OymW0Drnf3N45uZ2YZ7t4Qz9pEkpGmPiRuzOwCM9tpZv/DzMqAx82st5m9YmYVZvZZZHlws9csMrPrI8vXmdm7ZvZvkbYfm9nM42w73MwWm1m1mb1hZr81s6eOo09jIp9baWZrzezyZttmmdlHkc8oNbNbI+v7RvpZaWZ7zewdM9N/i9Iq/XJIvPUHCoBhwBzCv4OPR34eCtQAD7bx+snABqAvcC/wqJnZcbT9T2Ap0Ae4C7i6ox0xs0zgZWA+cBLwT8DTZjY60uRRwlM9ecA44K3I+luAnUAh0A+4A9AcpLRKQS3xFgJ+7u517l7j7nvcfZ67H3T3auAeYFobry9x90fcvRF4EhhAOOyibmtmQ4GJwD+7+yF3fxd46Tj6MgXoAfwq8j5vAa8AV0a21wNjzSzf3T9z9xXN1g8Ahrl7vbu/4zpYJG1QUEu8Vbh7bdMPZtbNzP6vmZWYWRWwGOhlZumtvL6sacHdD0YWe3Sw7UBgb7N1ADs62A8i77PD3UPN1pUAgyLLXwNmASVm9raZnRtZfx+wGZhvZlvN7Pbj+GzpQhTUEm9HjxxvAUYDk909H5gaWd/adEZn2AUUmFm3ZuuGHMf7fAIMOWp+eShQCuDuy9z9CsLTIi8Cz0XWV7v7Le4+ArgcuNnMLj6Oz5cuQkEtiZZHeF660swKgJ/H+gPdvQQoBu4ys6zISPfv2nudmeU0/yI8x30QuM3MMiOn8f0dMDfyvt80s57uXg9UEZ72wcy+bGanRObL9xE+dTHU4oeKoKCWxPsNkAt8CiwBXo/T534TOBfYA9wNPEv4fO/WDCL8P5TmX0MIB/NMwvU/BFzj7usjr7ka2BaZ0vle5DMBRgJvAPuB94CH3H1hp/VMUo4ueBEBzOxZYL27x3xEL9JRGlFLl2RmE83sZDNLM7NLgSsIzyOLBI6uTJSuqj/wAuHzqHcC33f3lYktSaRlmvoQEQk4TX2IiARcTKY++vbt60VFRbF4axGRlLR8+fJP3b2wpW0xCeqioiKKi4tj8dYiIinJzEpa26apDxGRgFNQi4gEnIJaRCTgFNQiIgGnoBYRCTgFtYhIwCmoRUQCLlBBXbLnAO9u+jTRZYiIBEqgbso07b5FAGz71WWJLUREJEACNaIWEZFjKahFRAJOQS0iEnAKahGRgFNQi4gEnIJaRCTgFNQiIgGnoBYRCTgFtYhIwCmoRUQCTkEtIhJwCmoRkYBTUIuIBJyCWkQk4BTUIiIBp6AWEQk4BbWISMApqEVEAk5BLSIScFE9M9HMtgHVQCPQ4O4TYlmUiIgc0ZGH217o7npEuIhInGnqQ0Qk4KINagfmm9lyM5vTUgMzm2NmxWZWXFFR0XkVioh0cdEG9fnuPh6YCdxgZlOPbuDuD7v7BHefUFhY2KlFioh0ZVEFtbuXRr6XA38CJsWyKBEROaLdoDaz7maW17QMzAA+jHVhIiISFs1ZH/2AP5lZU/v/dPfXY1qViIgc1m5Qu/tW4Mw41CIiIi3Q6XkiIgGnoBYRCTgFtYhIwCmoRUQCTkEtIhJwCmoRkYBTUIuIBJyCWkQk4BTUIiIBp6AWEQk4BbWISMApqEVEAk5BLSIScApqEZGAU1CLiAScglpEJOAU1CIiAaegFhEJOAW1iEjAKahFRAJOQS0iEnAKahGRgFNQi4gEnIJaRCTgFNQiIgEXmKAOhTzRJYiIBFJggjotzRJdgohIIEUd1GaWbmYrzeyVWBYkIiKf15ER9Y3AulgV0py7pkFERJpEFdRmNhi4DPhDbMsREZGjRTui/g1wGxCKYS2HaUAtInJEu0FtZl8Gyt19eTvt5phZsZkVV1RUdFqBIiJdXTQj6vOAy81sGzAXuMjMnjq6kbs/7O4T3H1CYWFhJ5cpItJ1tRvU7v4Tdx/s7kXAN4C33P2qWBalmQ8RkSMCcx61iIi0LKMjjd19EbAoJpUAZuEDieHT83QBjIgIBGxEffP0UYkuQUQkcAIV1CIicqxABrUOJoqIHBGooDZNS4uIHCNQQd1EVyaKiBwRqKA2DalFRI4RqKBu4pqlFhE5LJBBLSIiRyioRUQCLlBBnRaZo65riMvdVEVEkkKggvrVNZ8A8MjirQmuREQkOAIV1P84YQgA/XvmJLgSEZHgCFRQn3tyXwB6ZHfoXlEiIiktUEGdnREuR3PUIiJHBCqoczLTAairb0xwJSIiwRGooM7O1IhaRORogQrqnIzwiLpWI2oRkcMCFdSZ6UaaaUQtItJcoILazMjOSNeIWkSkmUAFNUBOZppG1CIizQQuqDWiFhH5vMAFdU5mGrX1GlGLiDQJXFBnZ6RT16ARtYhIk8AFtUbUIiKfF7igzs5IZ19NfaLLEBEJjMAF9ch+Pdi4u5pQSI/jEhGBAAb1aQN7cvBQI6WVNYkuRUQkEAIX1GcM7gnA4k0VCa5ERCQY2g1qM8sxs6VmtsrM1prZL2JZ0GkD8xk7IJ8H39pMeXVtLD9KRCQpRDOirgMucvczgbOAS81sSqwKMjPu/fszKKuqZdI9b7Jnf12sPkpEJCm0G9Qetj/yY2bkK6ZH+sYN6sl3vjgCgHPufgN3HVgUka4rqjlqM0s3sw+AcmCBu7/fQps5ZlZsZsUVFSc+v3zHrDGH56ufL955wu8nIpKsogpqd29097OAwcAkMxvXQpuH3X2Cu08oLCzslOJe/MF5pBk8uHCzRtUi0mV16KwPd68EFgKXxqacz0tLM375ldPZvvcgH5ZWxeMjRUQCJ5qzPgrNrFdkORf4ErA+1oU1uXRcf7LS05i3QtMfItI1RTOiHgAsNLPVwDLCc9SvxLasI3p1y2LqqL4s3FAer48UEQmUjPYauPtq4Ow41NKqycP78Ma6csqrajkpPyeRpYiIxF3grkxsycThBQAs3bY3wZWIiMRfUgT1aQPzyc1MZ9nHCmoR6XqSIqgz09MYP6wXS7d9luhSRETiLimCGmBiUQHry6p0r2oR6XKSJqgnFRXgDitKNKoWka4laYL67KG9yUgzHVAUkS4naYI6NyudcYN66oCiiHQ5SRPUAJOGF7B65z5q6/WUchHpOpIqqCcWFXCoMcSqHZWJLkVEJG6SKqgnDOsNwP3zN3KgriHB1YiIxEdSBXXv7llkpocPKN755w8TXY6ISFwkVVBD+CnlAC+sKE1wJSIi8ZF0QX3/189MdAkiInGVdEF9cmGPw8s79h5MYCUiIvGRdEEN8NqPvgjAT1/UPLWIpL6kDOqxA/MZWtCNxRsr2HvgUKLLERGJqaQMajgyV/3vb25KcCUiIrGVtEE9saiAWaf355ml2ynbV5vockREYiZpgxrgJzPHEHLna7/7f7oARkRSVlIH9ZCCblz/xRGUVtbw6wUbE12OiEhMJHVQA9x2yWgA/vDux9w0dyXunuCKREQ6V9IHtZlx55fHAvDiB59w96vrElyRiEjnSvqgBvj2+cO5cHQhAI+++zHry6oSXJGISOdJiaAGeHz2JG6aPhKAS3/zDtv36KpFEUkNKRPUADdNH3V4eep9C6mq1YNwRST5pVRQA2z71WWHl8+4az4/eHp5AqsRETlxKRfUEA7rSUUFALy2pozfLdpCVW09oZDOCBGR5JOSQQ3w7Hen8KOLTgHgX19fzxl3zef3i7ckuCoRkY5rN6jNbIiZLTSzj8xsrZndGI/CTpSZcfOM0az6+YzD6+59fQML15cnsCoRkY6LZkTdANzi7mOBKcANZjY2tmV1np65mWy8eyZfGz8YgNlPLKPo9leZv7ZMZ4aISFJoN6jdfZe7r4gsVwPrgEGxLqwzZWWkcf/Xz2TJTy4+vG7Ofyxn6n0Lqa1vTGBlIiLt69ActZkVAWcD77ewbY6ZFZtZcUVFRedU18n698xhzV0zuHrKsMPrTr3zdb71xDI+qawBIBRyag4pvEUkOCzae2OYWQ/gbeAed3+hrbYTJkzw4uLiTigvdkIh5+x/WcC+miPnWg/qlcsVZw3koUVbWP8vl5KTmZ7ACkWkKzGz5e4+oaVtUY2ozSwTmAc83V5IJ4u0NGPVz2fw9PWTD68rrazhoUXhM0MeXrxVN3gSkUBod0RtZgY8Cex195uiedNkGFEfbXP5fqY/8PYx65/69mQmDS8gKyNlz2QUkQBoa0QdTVCfD7wDrAFCkdV3uPtrrb0mGYO6yebyatLMuOj+z4f20jsu5qT8nARVJSKp7oSC+ngkc1A3qa1v5NQ7Xz9m/b/9w5nMHNef7Iw0MtI1yhaRzqGgPgH1jSFueW4VL6365Jhtm++ZqbAWkU6hoO4E7s60+xaxfe+xF8k8MXsip/bPp39PTY2IyPFRUHeyuoZGbpr7AX/5sOyYbc/OmcLkEX0SUJWIJDMFdYzsq6nn8gffpaSVS9GLfzadvj2y41yViCQjBXUcvL91D1c+soSj76SamW48MXsSZw7pRY/sjMQUJyKBp6COo9r6Rkora7j4/mPPyQb4+oTB/PDCkQzt0y3OlYlIkCmoE6Qx5DyzdDuPvLO1xemRuXOmMEXz2SKCgjowHv/bx/zi5Y9a3DZzXH+umjKM807pG+eqRCQIFNQBs+9gPWf+z/mtbu/bI5tHrjmHcYN6kqnztEW6BAV1gG3fc5BH393Kk++VtNomM9244cJT+KeLRpKeZnGsTkTiRUGdRGrrG3m+eAd3/nltm+2mj+nH//rq6RTm6fQ/kVSgoE5S68uqqKpp4A/vbGX+R7vbbPv7q8YzfUw/0tOM8A0PRSSZKKhTxKf766ioruPyB9+lvrH9/XblpKH8+JLRFHTPikN1InIiFNQpatPuam5+bhVrSve127ZP9yy6ZaczbmBPfvmV08nPzdR8t0iAKKi7kAN1Dfz+7S38n7c2R9X+tIH5PHbdRPp0z9KdAEUSSEHdhTU0hli4oYL9dfUsXF/R4u1amzu1fx7ry6oBWPDfpzKisIdG3iJxoKCWw9ydg4caWbWjknkrSpm3YmfUr73uC0XcePFIcjLTGfPPrzOkIJd3brsohtWKdB0KamlXzaFG5i7bTkV13eEH/Ebrue+ey8Si3jrbROQEKKjluNU1NPLssh08tHALZVW1Ub3m8dkTGTewJ727ZWreWyRKCmrpNLv21fDiyk/4x4lD2F/bwK3Pr2Lptr3tvm7sgHyumjKMWaf3p1c3nS4ocjQFtcScu7NieyU//q9VDOqVyzubPm2z/ZCCXL4/7RQG9c5l2qjCOFUpElwKakmI+sYQ63ZVcfcr69i+92BUUyf3fGUcl5zWn565mbohlXQpCmoJlNLKGu59fT1//qDtUwUhfHVlfm4Gfbtn852pI+JQnUhiKKgl0OobQywv+YxvPLwkqva3zhjFtV8oIi8nM8aVicSPglqSSijk/G3Lpxyoa+Dl1bt4dfWuNtv/1/fOZdygnuRkpsepQpHOp6CWpOfulFfXMfmXb7bb9vHZE8nLzmBCUUEcKhPpHApqSUkHDzWwoqSSqx59v812F44u5LffHE+3LD0FXoJLQS1dxv9+YxO/fmNjq9unjSpkQM8czh/Zlx8/v5plP5tOj2wFuCTeCQW1mT0GfBkod/dx0XygglqCoLa+kblLtzN32Y7DN5pqyXe+OJw7Zo3RJfCSUCca1FOB/cAfFdSSrNydv67dzY1zV1LXEGq13aJbL6DRnYZGZ3T/vDhWKF3dCU99mFkR8IqCWlLNog3lXPf4sla33zpjFLNOH8CIwh5xrEq6orgEtZnNAeYADB069JySktafqi0SJIcaQmzfe5DpD7zdbts1d83Q+dsSExpRi3RAY8hJM/jWE8tYuKGixTZfGz+Y700bwch+mh6RztFWUOtwt8hRmp5o8/jsSQBUVNexu6qWax5byt4DhwCYt2Ln5x66cN0XivjpZWN0fxKJCY2oRTrg4KEGird9xgMLNvLBjspjtvfLz+aBr5/F2AH59NbT36UDTvSsj2eAC4C+wG7g5+7+aFuvUVBLV/Lelj1c+cix9ynJz8ngx5eMpk+PbGadPiABlUky0QUvInHwSWUN339qOat27mtx+3enjeDb5w2nMC9b52zLMRTUInFW19DIvOWlvP/xnlZv5zp3zhSmjOgT58okqBTUIgnU0BjihRWl3DZvdYvbH776HAb0zCUzwzi1f36cq5OgUFCLBMjOzw7ysxc/ZFELp/6dXNid+/7hTMYP7Z2AyiSRFNQiAfVh6T6+//RyduytaXH7I9dM4Etj+8W5KkkEBbVIEqhvDPHUkhJ+8fJHLW6fdXp/RvXL44YLT9H52ilIQS2SZBpDzi9eXssf32v5Vgw3f2kU151XRL4uZ08ZCmqRJHWgroHy6jrun7+BV1p4JFluZjpv3jKNgu5ZehRZklNQi6SQ97fu4bG/fcxf1+4+Ztv0Mf2YNrqQq6cMS0BlciIU1CIpqGxfLbfNW83ijS3fOCojzXj+e+eSmZ7GaQPzdZFNwCmoRbqAsn21/HrBRhZvqmDXvtpjtv/o4pH84IKTNUUSUApqkS7m408PcOPcleyrqadkz8Fjto8blM99f38mYwboApugUFCLCC+t+oQfPbOyxW3nndKHO2aNYUhBN51JkiAKahE5bNWOSh5cuJkFHx17MLK5Z+dMYbLuRRI3CmoRaZG7U1pZw4srS3n8b9vYE3kwwtF65mZy1ZShjBvYk0vH9deByRhQUItIhzy1pIQVJZ/xwsrSFrcX5mVz3sl9uPrcYZwzrCDO1aUmBbWIHJemfNj66QF++eo63lxffkyb/JwMxg/rzfC+3cnKSOPcEX04/5S+ZOgy9w5RUItIp1q3q4rni3fSEApRsucgu6tqWV9W3eZrCrpn8eTsSYwZkKcQb4GCWkRirmxfLc8V7+C1Nbtwhw272w7uJmkG131hOF8dP4hBvXLp1S2zS86BK6hFJCFCIae6roHfLdrCU0tKONQYIjsjjerahg69z0l52cw+bzhnD+1FQfcs+vbIpiDFHh6soBaRwNlf18DG3dV8sL2SvJwM5i7bQVZ6Gu9t3XNc7ze0oBvb9x5kzIB8Jgzrzf66BnIy0xh5Uh5jBuRzqDHEWYN70SMng/S04I3YFdQiknTcnZr6RhpDzvy1u+mWlc66smqWbNlDVW09melprClt+UHC0eiZm8m+mvrPrRvRtztTRxXSPTud3Mx0SitreHNdOV8dP5izh/YiNzOdA3UNpKcZI/vlMbh3LhXVdRTmZZORZic0ZaOgFpEuIRRyPtlXw4G6RjaX72fJ1j1sLt/PSfnZLPhoN2MG5FNzqJGSPQfokZPB7qq6Tv38ycMLeOr6ycf1YIe2gjrjhCsTEQmItDRjcO9uAIzun8dlZwzo0OubBq7uUFPfyP66Bv6yZhe9umXRIzuDiv11zF9bxrpd1ZRV1XL20F6s3F7JuSP60C0rncK87Jg8fUdBLSIS0TR1YQbdszPonp3BdecN/1ybKycNjXtdOplRRCTgFNQiIgGnoBYRCTgFtYhIwEUV1GZ2qZltMLPNZnZ7rIsSEZEj2g1qM0sHfgvMBMYCV5rZ2FgXJiIiYdGMqCcBm919q7sfAuYCV8S2LBERaRJNUA8CdjT7eWdk3eeY2RwzKzaz4oqKlh9fLyIiHddpF7y4+8PAwwBmVmFmJcf5Vn2BTzurriShPqe+rtZfUJ87alhrG6IJ6lJgSLOfB0fWtcrdC6Or61hmVtza9e6pSn1OfV2tv6A+d6Zopj6WASPNbLiZZQHfAF7q7EJERKRl7Y6o3b3BzH4I/BVIBx5z97Uxr0xERIAo56jd/TXgtRjX0uThOH1OkKjPqa+r9RfU504Tk/tRi4hI59El5CIiAaegFhEJuMAEdSrfT8TMtpnZGjP7wMyKI+sKzGyBmW2KfO8dWW9m9u+Rf4fVZjY+sdVHx8weM7NyM/uw2boO99HMro2032Rm1yaiL9Fqpc93mVlpZF9/YGazmm37SaTPG8zskmbrk+Z338yGmNlCM/vIzNaa2Y2R9Sm5r9vob3z3s7sn/Ivw2SRbgBFAFrAKGJvoujqxf9uAvketuxe4PbJ8O/CvkeVZwF8AA6YA7ye6/ij7OBUYD3x4vH0ECoCtke+9I8u9E923Dvb5LuDWFtqOjfxeZwPDI7/v6cn2uw8MAMZHlvOAjZG+peS+bqO/cd3PQRlRd8X7iVwBPBlZfhL4b83W/9HDlgC9zKxjD35LAHdfDOw9anVH+3gJsMDd97r7Z8AC4NLYV398Wulza64A5rp7nbt/DGwm/HufVL/77r7L3VdElquBdYRvKZGS+7qN/rYmJvs5KEEd1f1EkpgD881suZnNiazr5+67IstlQL/Icir9W3S0j6nS9x9G/sx/rGkKgBTss5kVAWcD79MF9vVR/YU47uegBHWqO9/dxxO+VewNZja1+UYP/82U0udJdoU+RvwOOBk4C9gF3J/YcmLDzHoA84Cb3L2q+bZU3Nct9Deu+zkoQd3h+4kkE3cvjXwvB/5E+M+g3U1TGpHv5ZHmqfRv0dE+Jn3f3X23uze6ewh4hPC+hhTqs5llEg6tp939hcjqlN3XLfU33vs5KEGdsvcTMbPuZpbXtAzMAD4k3L+mI93XAn+OLL8EXBM5Wj4F2NfsT8pk09E+/hWYYWa9I39KzoisSxpHHU/4CuF9DeE+f8PMss1sODASWEqS/e6bmQGPAuvc/YFmm1JyX7fW37jv50QfVW12tHQW4SOqW4CfJrqeTuzXCMJHeFcBa5v6BvQB3gQ2AW8ABZH1RviJOluANcCERPchyn4+Q/hPwHrC82/fPp4+At8ifABmMzA70f06jj7/R6RPqyP/IQ5o1v6nkT5vAGY2W580v/vA+YSnNVYDH0S+ZqXqvm6jv3Hdz7qEXEQk4IIy9SEiIq1QUIuIBJyCWkQk4BTUIiIBp6AWEQk4BbWISMApqEVEAu7/A0HZo6aSWbOZAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gawSIEf_Ix31",
        "colab_type": "text"
      },
      "source": [
        "## **LSTM generation:**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "vE2hYSqAAtkn",
        "jupyter": {
          "source_hidden": true
        },
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 862
        },
        "outputId": "ed84a727-c3d7-43a1-a0cb-575ca4c3ad4e"
      },
      "source": [
        "generator = Sequential([\n",
        "    Embedding(vocab_size, embedding_size,\n",
        "              batch_input_shape=(1, None)),\n",
        "    LSTM(lstm_size, return_sequences = True, stateful=True),\n",
        "    Dense(hidden_size, activation = relu), \n",
        "    Dense(vocab_size)\n",
        "])\n",
        "\n",
        "generator.summary()\n",
        "\n",
        "# Import trained weights from RNN to generator\n",
        "generator.set_weights(RNN.get_weights())\n",
        "\n",
        "def generate_text(start_string, num_generate = 1000, temperature = 1.0):\n",
        "    \n",
        "    # Vectorize input string\n",
        "    input_eval = [char2idx[s] for s in start_string]  \n",
        "    input_eval = tf.expand_dims(input_eval, 0)\n",
        "    \n",
        "    text_generated = [] # List to append predicted chars \n",
        "    \n",
        "    idx2char = { v: k for k, v in char2idx.items() }  # invert char-index mapping\n",
        "    \n",
        "    generator.reset_states()\n",
        "    \n",
        "    for i in range(num_generate):\n",
        "        predictions = generator(input_eval)\n",
        "        predictions = tf.squeeze(predictions, 0)\n",
        "        \n",
        "        # sample next char based on distribution and temperature\n",
        "        predictions = predictions / temperature\n",
        "        predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n",
        "        \n",
        "        input_eval = tf.expand_dims([predicted_id], 0)\n",
        "\n",
        "        text_generated.append(idx2char[predicted_id])\n",
        "        \n",
        "    return (start_string + ''.join(text_generated))\n",
        "\n",
        "init=\"\"\"\n",
        "Nel mezzo del cammin di nostra vita\n",
        "mi ritrovai per una selva oscura,\n",
        "chè la diritta via era smarrita.\n",
        "\"\"\"\n",
        "print(generate_text(init))\n"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_8\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_8 (Embedding)      (1, None, 256)            15872     \n",
            "_________________________________________________________________\n",
            "lstm_8 (LSTM)                (1, None, 1024)           5246976   \n",
            "_________________________________________________________________\n",
            "dense_16 (Dense)             (1, None, 256)            262400    \n",
            "_________________________________________________________________\n",
            "dense_17 (Dense)             (1, None, 62)             15934     \n",
            "=================================================================\n",
            "Total params: 5,541,182\n",
            "Trainable params: 5,541,182\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "\n",
            "Nel mezzo del cammin di nostra vita\n",
            "mi ritrovai per una selva oscura,\n",
            "chè la diritta via era smarrita.\n",
            "  Ahi quanto a dir de la destra scempie\n",
            "trovai pur sei le lettere che 'ncise\n",
            "quest'è quanti molti poco s'imbiancea:\n",
            "  \"Or dipinto giù per la mente c'innoviva.\n",
            "  Questi che guida in alto li occhi miei,\n",
            "è quel Virgilio dal qual tu togliesti\n",
            "fiange come da l'acqua e de la toglia,\n",
            "  comparata in mezzo del ciel che tu partito!\n",
            "  Ma vieni un gran per lonoria, giacea\n",
            "due buoi uscir, chè non fu più perso;\n",
            "e noi, quanto le nove spense perso\n",
            "quel che pragona de l'opera sicelta.\n",
            "  Così si mossere in vicanda uscili!\n",
            "  Così per viva, girondo tutto le viste\n",
            "di qua tenene lo bricadde e indi,\n",
            "cu' batte infetto da Essisore e gravi,\n",
            "oggiunti si spiri, non essa appario;\n",
            "  e s'a quel trasso moder li decissimai\n",
            "per la qual si feo; e s'io non l'esserdia;\n",
            "però in parte vita per nave,\n",
            "  che se' t'inse'misarato speso costai\n",
            "reglio ha volta, ed el trascende a' piante\n",
            "  al padre per lo ciel volo sperse,\n",
            "e 'l suo disiagge e fè sentitta intene.\n",
            "  E come dal figlio par le cintor letizia,\n",
            "se tem suoi disposti che vu\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}