Deep Learning/Intro to Natural Language Processing/Text_Generation_from_Shakespeare_Data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Text Generation from Shakespeare Data.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "files.upload()"
      ],
      "metadata": {
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 92
        },
        "id": "dr64gXnjKxf9",
        "outputId": "15693a52-c325-4fcf-f118-c28d464a5cc4"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-5984cc43-6f89-46f0-8f53-91fb42d2dcb7\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-5984cc43-6f89-46f0-8f53-91fb42d2dcb7\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving kaggle.json to kaggle.json\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'kaggle.json': b'{\"username\":\"marcellopoliti\",\"key\":\"810de577597fed0b321973b6c8894d21\"}'}"
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install kaggle\n",
        "!mkdir ~/.kaggle\n",
        "!cp kaggle.json ~/.kaggle/\n",
        "!chmod 600 ~/.kaggle/kaggle.json"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Uz7sdSWmK-J5",
        "outputId": "3db50718-addc-4541-cbc2-11098939e0e0"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: kaggle in /usr/local/lib/python3.7/dist-packages (1.5.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from kaggle) (2.23.0)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.7/dist-packages (from kaggle) (2.8.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from kaggle) (4.62.3)\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.7/dist-packages (from kaggle) (5.0.2)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from kaggle) (2021.10.8)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.7/dist-packages (from kaggle) (1.15.0)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.7/dist-packages (from kaggle) (1.24.3)\n",
            "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.7/dist-packages (from python-slugify->kaggle) (1.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->kaggle) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->kaggle) (2.10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!kaggle datasets download \"kingburrito666/shakespeare-plays\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KYXh2xWoLBab",
        "outputId": "16f734a3-56bd-49bd-a920-28bec5531642"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading shakespeare-plays.zip to /content\n",
            "\r  0% 0.00/4.55M [00:00<?, ?B/s]\r100% 4.55M/4.55M [00:00<00:00, 45.9MB/s]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "IkexlOPKGtN1"
      },
      "outputs": [],
      "source": [
        "import zipfile\n",
        "\n",
        "zip_ref = zipfile.ZipFile(\"/content/shakespeare-plays.zip\")\n",
        "zip_ref.extractall()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "size = 10_000\n",
        "\n",
        "df = pd.read_csv(\"/content/Shakespeare_data.csv\")\n",
        "df.head()\n",
        "df = df[:size]"
      ],
      "metadata": {
        "id": "QvPRqjqiLR4x"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YnQH5LiNUuux",
        "outputId": "743ceb74-8747-4033-87d9-c725172de3ec"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10000"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "corpus = df.PlayerLine"
      ],
      "metadata": {
        "id": "gIY6K-FBLaUU"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences"
      ],
      "metadata": {
        "id": "QplGpZIxLfbp"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "oov_token = \"<OOV>\"\n",
        "truncating_type = \"pre\""
      ],
      "metadata": {
        "id": "rseFoWmfLvXq"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = Tokenizer(oov_token = oov_token)\n",
        "tokenizer.fit_on_texts(corpus)\n",
        "word_index = tokenizer.word_index\n",
        "vocab_size = len(word_index)"
      ],
      "metadata": {
        "id": "cCNMvROzLtev"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sequences = tokenizer.texts_to_sequences(corpus)"
      ],
      "metadata": {
        "id": "Hud3YX-kMBXO"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_sequences = []\n",
        "\n",
        "for seq in sequences:\n",
        "  for i in range(len(seq)):\n",
        "    n_gram = seq[0:i+1]\n",
        "    all_sequences.append(n_gram)"
      ],
      "metadata": {
        "id": "cXFssalWMRCl"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "max_len = max([len(seq) for seq in all_sequences])\n",
        "max_len"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0TKtYL1gOBpm",
        "outputId": "d651cd92-99ed-4503-93fa-c53ba0902ee5"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "56"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "padded = pad_sequences(all_sequences, maxlen = max_len, padding= \"pre\")"
      ],
      "metadata": {
        "id": "Vs6dHOOUL2Br"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x = padded[:,:-1]\n",
        "y = padded[:,-1]"
      ],
      "metadata": {
        "id": "702xOLeROfza"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.layers import Embedding, Dense,LSTM,GRU,Bidirectional,Flatten,Conv1D\n",
        "from tensorflow.keras.models import Sequential"
      ],
      "metadata": {
        "id": "3XbHJX6mOqXA"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Let's import pre-trained embedding"
      ],
      "metadata": {
        "id": "NRudhWxQR7_R"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://huggingface.co/stanfordnlp/glove/resolve/main/glove.6B.zip ./glove.6B.100d.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MuNo_G-iSu6L",
        "outputId": "be8dedb7-edf5-4cec-c639-9e89ee77cac7"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-02-14 08:26:34--  https://huggingface.co/stanfordnlp/glove/resolve/main/glove.6B.zip\n",
            "Resolving huggingface.co (huggingface.co)... 34.204.221.201, 34.224.55.150, 54.205.126.93, ...\n",
            "Connecting to huggingface.co (huggingface.co)|34.204.221.201|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://cdn-lfs.huggingface.co/stanfordnlp/glove/6471382cdd837544bf3ac72497a38715e845897d265b2b424b4761832009c837 [following]\n",
            "--2022-02-14 08:26:34--  https://cdn-lfs.huggingface.co/stanfordnlp/glove/6471382cdd837544bf3ac72497a38715e845897d265b2b424b4761832009c837\n",
            "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 13.226.9.86, 13.226.9.100, 13.226.9.123, ...\n",
            "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|13.226.9.86|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 862182753 (822M) [application/zip]\n",
            "Saving to: ‘glove.6B.zip’\n",
            "\n",
            "glove.6B.zip        100%[===================>] 822.24M  41.6MB/s    in 19s     \n",
            "\n",
            "2022-02-14 08:26:54 (42.8 MB/s) - ‘glove.6B.zip’ saved [862182753/862182753]\n",
            "\n",
            "--2022-02-14 08:26:54--  http://./glove.6B.100d.txt\n",
            "Resolving . (.)... failed: No address associated with hostname.\n",
            "wget: unable to resolve host address ‘.’\n",
            "FINISHED --2022-02-14 08:26:54--\n",
            "Total wall clock time: 20s\n",
            "Downloaded: 1 files, 822M in 19s (42.8 MB/s)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "zip_ref = zipfile.ZipFile(\"/content/glove.6B.zip\")\n",
        "zip_ref.extractall()"
      ],
      "metadata": {
        "id": "hJp-Z-X5S03w"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "embeddings_index = {};\n",
        "embedding_dim = 100\n",
        "\n",
        "with open('/content/glove.6B.100d.txt', encoding=\"utf-8\",errors='ignore') as f:\n",
        "    for line in f:\n",
        "        values = line.split();\n",
        "        word = values[0];\n",
        "        coefs = np.asarray(values[1:], dtype='float32');\n",
        "        embeddings_index[word] = coefs;\n",
        "\n",
        "embeddings_matrix = np.zeros((vocab_size+1, embedding_dim));\n",
        "for word, i in word_index.items():\n",
        "    embedding_vector = embeddings_index.get(word);\n",
        "    if embedding_vector is not None:\n",
        "        embeddings_matrix[i] = embedding_vector;"
      ],
      "metadata": {
        "id": "hD5zjNbrS3Yj"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = Sequential([\n",
        "  Embedding(input_dim=vocab_size + 1, output_dim=100, input_length=max_len - 1, weights = [embeddings_matrix], trainable = False),\n",
        "  Conv1D(filters = 64, kernel_size = 5, activation = \"relu\"),\n",
        "  Bidirectional(LSTM(64)),\n",
        "  Dense(64,activation = \"relu\"),\n",
        "  Dense(vocab_size + 1,\"softmax\")\n",
        "])\n",
        "  \n",
        "model.summary()\n",
        "model.compile(loss = \"categorical_crossentropy\", metrics = [\"accuracy\"], optimizer = \"adam\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BAtYJBM6PO3c",
        "outputId": "af44a746-e417-4185-c0c4-ca12b49c0141"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding (Embedding)       (None, 55, 100)           785800    \n",
            "                                                                 \n",
            " conv1d (Conv1D)             (None, 51, 64)            32064     \n",
            "                                                                 \n",
            " bidirectional (Bidirectiona  (None, 128)              66048     \n",
            " l)                                                              \n",
            "                                                                 \n",
            " dense (Dense)               (None, 64)                8256      \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 7858)              510770    \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,402,938\n",
            "Trainable params: 617,138\n",
            "Non-trainable params: 785,800\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow import keras\n",
        "import tensorflow as tf"
      ],
      "metadata": {
        "id": "bsGfurb5QpsF"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "early_stopping = keras.callbacks.EarlyStopping(monitor = \"loss\",patience = 50)"
      ],
      "metadata": {
        "id": "dWMHqqxGQXHX"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y = tf.one_hot(y,vocab_size+1)"
      ],
      "metadata": {
        "id": "pBiXZTKZT9Qw"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "history = model.fit(x,y,epochs = 500, callbacks = [early_stopping])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R_8s3DDLQJ2B",
        "outputId": "52822851-a066-493f-9afd-744301c13f06"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/500\n",
            "2389/2389 [==============================] - 95s 34ms/step - loss: 6.8217 - accuracy: 0.0466\n",
            "Epoch 2/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 6.3800 - accuracy: 0.0648\n",
            "Epoch 3/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 6.1476 - accuracy: 0.0721\n",
            "Epoch 4/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 5.9896 - accuracy: 0.0775\n",
            "Epoch 5/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 5.8482 - accuracy: 0.0820\n",
            "Epoch 6/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 5.7241 - accuracy: 0.0867\n",
            "Epoch 7/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 5.6183 - accuracy: 0.0893\n",
            "Epoch 8/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 5.5309 - accuracy: 0.0914\n",
            "Epoch 9/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 5.4517 - accuracy: 0.0935\n",
            "Epoch 10/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 5.3776 - accuracy: 0.0956\n",
            "Epoch 11/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 5.3115 - accuracy: 0.0982\n",
            "Epoch 12/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 5.2459 - accuracy: 0.0998\n",
            "Epoch 13/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 5.1838 - accuracy: 0.1011\n",
            "Epoch 14/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 5.1258 - accuracy: 0.1023\n",
            "Epoch 15/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 5.0696 - accuracy: 0.1043\n",
            "Epoch 16/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 5.0161 - accuracy: 0.1062\n",
            "Epoch 17/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 4.9649 - accuracy: 0.1079\n",
            "Epoch 18/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 4.9170 - accuracy: 0.1092\n",
            "Epoch 19/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.8734 - accuracy: 0.1116\n",
            "Epoch 20/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.8263 - accuracy: 0.1141\n",
            "Epoch 21/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.7898 - accuracy: 0.1185\n",
            "Epoch 22/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.7551 - accuracy: 0.1205\n",
            "Epoch 23/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.7191 - accuracy: 0.1231\n",
            "Epoch 24/500\n",
            "2389/2389 [==============================] - 86s 36ms/step - loss: 4.6895 - accuracy: 0.1258\n",
            "Epoch 25/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 4.6555 - accuracy: 0.1281\n",
            "Epoch 26/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 4.6260 - accuracy: 0.1314\n",
            "Epoch 27/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 4.5967 - accuracy: 0.1339\n",
            "Epoch 28/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.5668 - accuracy: 0.1367\n",
            "Epoch 29/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 4.5389 - accuracy: 0.1388\n",
            "Epoch 30/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 4.5182 - accuracy: 0.1422\n",
            "Epoch 31/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.4886 - accuracy: 0.1460\n",
            "Epoch 32/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.4657 - accuracy: 0.1476\n",
            "Epoch 33/500\n",
            "2389/2389 [==============================] - 79s 33ms/step - loss: 4.4432 - accuracy: 0.1504\n",
            "Epoch 34/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.4224 - accuracy: 0.1528\n",
            "Epoch 35/500\n",
            "2389/2389 [==============================] - 79s 33ms/step - loss: 4.4020 - accuracy: 0.1548\n",
            "Epoch 36/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.3787 - accuracy: 0.1590\n",
            "Epoch 37/500\n",
            "2389/2389 [==============================] - 79s 33ms/step - loss: 4.3593 - accuracy: 0.1599\n",
            "Epoch 38/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.3427 - accuracy: 0.1625\n",
            "Epoch 39/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 4.3281 - accuracy: 0.1639\n",
            "Epoch 40/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.3128 - accuracy: 0.1651\n",
            "Epoch 41/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.2929 - accuracy: 0.1678\n",
            "Epoch 42/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 4.2778 - accuracy: 0.1701\n",
            "Epoch 43/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 4.2615 - accuracy: 0.1712\n",
            "Epoch 44/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.2426 - accuracy: 0.1735\n",
            "Epoch 45/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.2349 - accuracy: 0.1753\n",
            "Epoch 46/500\n",
            "2389/2389 [==============================] - 79s 33ms/step - loss: 4.2140 - accuracy: 0.1780\n",
            "Epoch 47/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.2037 - accuracy: 0.1788\n",
            "Epoch 48/500\n",
            "2389/2389 [==============================] - 79s 33ms/step - loss: 4.1868 - accuracy: 0.1817\n",
            "Epoch 49/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.1735 - accuracy: 0.1830\n",
            "Epoch 50/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.1628 - accuracy: 0.1848\n",
            "Epoch 51/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.1470 - accuracy: 0.1862\n",
            "Epoch 52/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 4.1393 - accuracy: 0.1861\n",
            "Epoch 53/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.1244 - accuracy: 0.1883\n",
            "Epoch 54/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 4.1141 - accuracy: 0.1896\n",
            "Epoch 55/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.1022 - accuracy: 0.1927\n",
            "Epoch 56/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 4.0878 - accuracy: 0.1919\n",
            "Epoch 57/500\n",
            "2389/2389 [==============================] - 79s 33ms/step - loss: 4.0774 - accuracy: 0.1950\n",
            "Epoch 58/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 4.0755 - accuracy: 0.1949\n",
            "Epoch 59/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 4.0579 - accuracy: 0.1982\n",
            "Epoch 60/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.0487 - accuracy: 0.1987\n",
            "Epoch 61/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 4.0419 - accuracy: 0.1989\n",
            "Epoch 62/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.0303 - accuracy: 0.2008\n",
            "Epoch 63/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.0239 - accuracy: 0.2025\n",
            "Epoch 64/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.0178 - accuracy: 0.2005\n",
            "Epoch 65/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 4.0076 - accuracy: 0.2029\n",
            "Epoch 66/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 4.0023 - accuracy: 0.2041\n",
            "Epoch 67/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.9960 - accuracy: 0.2053\n",
            "Epoch 68/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 3.9885 - accuracy: 0.2064\n",
            "Epoch 69/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 3.9768 - accuracy: 0.2066\n",
            "Epoch 70/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 3.9696 - accuracy: 0.2086\n",
            "Epoch 71/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.9628 - accuracy: 0.2078\n",
            "Epoch 72/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 3.9534 - accuracy: 0.2110\n",
            "Epoch 73/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.9479 - accuracy: 0.2109\n",
            "Epoch 74/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 3.9424 - accuracy: 0.2119\n",
            "Epoch 75/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.9319 - accuracy: 0.2134\n",
            "Epoch 76/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.9304 - accuracy: 0.2136\n",
            "Epoch 77/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 3.9314 - accuracy: 0.2153\n",
            "Epoch 78/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.9130 - accuracy: 0.2152\n",
            "Epoch 79/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.9063 - accuracy: 0.2168\n",
            "Epoch 80/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.9056 - accuracy: 0.2179\n",
            "Epoch 81/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8939 - accuracy: 0.2180\n",
            "Epoch 82/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.8968 - accuracy: 0.2184\n",
            "Epoch 83/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.8888 - accuracy: 0.2182\n",
            "Epoch 84/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.8833 - accuracy: 0.2216\n",
            "Epoch 85/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.8819 - accuracy: 0.2207\n",
            "Epoch 86/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 3.8794 - accuracy: 0.2203\n",
            "Epoch 87/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8659 - accuracy: 0.2211\n",
            "Epoch 88/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8661 - accuracy: 0.2229\n",
            "Epoch 89/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8585 - accuracy: 0.2246\n",
            "Epoch 90/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8604 - accuracy: 0.2221\n",
            "Epoch 91/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.8510 - accuracy: 0.2240\n",
            "Epoch 92/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8494 - accuracy: 0.2243\n",
            "Epoch 93/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8483 - accuracy: 0.2236\n",
            "Epoch 94/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 3.8360 - accuracy: 0.2260\n",
            "Epoch 95/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8268 - accuracy: 0.2269\n",
            "Epoch 96/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8349 - accuracy: 0.2269\n",
            "Epoch 97/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.8259 - accuracy: 0.2282\n",
            "Epoch 98/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8205 - accuracy: 0.2297\n",
            "Epoch 99/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 3.8168 - accuracy: 0.2280\n",
            "Epoch 100/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8166 - accuracy: 0.2293\n",
            "Epoch 101/500\n",
            "2389/2389 [==============================] - 82s 34ms/step - loss: 3.8051 - accuracy: 0.2301\n",
            "Epoch 102/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.8017 - accuracy: 0.2307\n",
            "Epoch 103/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7972 - accuracy: 0.2308\n",
            "Epoch 104/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7918 - accuracy: 0.2311\n",
            "Epoch 105/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7880 - accuracy: 0.2331\n",
            "Epoch 106/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7823 - accuracy: 0.2340\n",
            "Epoch 107/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.7762 - accuracy: 0.2341\n",
            "Epoch 108/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.7793 - accuracy: 0.2346\n",
            "Epoch 109/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.7734 - accuracy: 0.2359\n",
            "Epoch 110/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7664 - accuracy: 0.2344\n",
            "Epoch 111/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 3.7609 - accuracy: 0.2366\n",
            "Epoch 112/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7595 - accuracy: 0.2372\n",
            "Epoch 113/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 3.7515 - accuracy: 0.2374\n",
            "Epoch 114/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7567 - accuracy: 0.2373\n",
            "Epoch 115/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7451 - accuracy: 0.2384\n",
            "Epoch 116/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7445 - accuracy: 0.2385\n",
            "Epoch 117/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7354 - accuracy: 0.2392\n",
            "Epoch 118/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7318 - accuracy: 0.2417\n",
            "Epoch 119/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.7319 - accuracy: 0.2416\n",
            "Epoch 120/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.7287 - accuracy: 0.2408\n",
            "Epoch 121/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.7270 - accuracy: 0.2410\n",
            "Epoch 122/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.7172 - accuracy: 0.2431\n",
            "Epoch 123/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7136 - accuracy: 0.2425\n",
            "Epoch 124/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.7127 - accuracy: 0.2442\n",
            "Epoch 125/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.7057 - accuracy: 0.2446\n",
            "Epoch 126/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.7043 - accuracy: 0.2452\n",
            "Epoch 127/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.7036 - accuracy: 0.2444\n",
            "Epoch 128/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.7023 - accuracy: 0.2464\n",
            "Epoch 129/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6970 - accuracy: 0.2461\n",
            "Epoch 130/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6979 - accuracy: 0.2470\n",
            "Epoch 131/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6957 - accuracy: 0.2464\n",
            "Epoch 132/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6822 - accuracy: 0.2477\n",
            "Epoch 133/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6855 - accuracy: 0.2475\n",
            "Epoch 134/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6899 - accuracy: 0.2456\n",
            "Epoch 135/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 3.6772 - accuracy: 0.2482\n",
            "Epoch 136/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6800 - accuracy: 0.2488\n",
            "Epoch 137/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6750 - accuracy: 0.2489\n",
            "Epoch 138/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6723 - accuracy: 0.2484\n",
            "Epoch 139/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6733 - accuracy: 0.2485\n",
            "Epoch 140/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6655 - accuracy: 0.2514\n",
            "Epoch 141/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6672 - accuracy: 0.2507\n",
            "Epoch 142/500\n",
            "2389/2389 [==============================] - 80s 33ms/step - loss: 3.6620 - accuracy: 0.2498\n",
            "Epoch 143/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6599 - accuracy: 0.2509\n",
            "Epoch 144/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6549 - accuracy: 0.2517\n",
            "Epoch 145/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6543 - accuracy: 0.2513\n",
            "Epoch 146/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6508 - accuracy: 0.2525\n",
            "Epoch 147/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6602 - accuracy: 0.2499\n",
            "Epoch 148/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6433 - accuracy: 0.2524\n",
            "Epoch 149/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6419 - accuracy: 0.2542\n",
            "Epoch 150/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6382 - accuracy: 0.2536\n",
            "Epoch 151/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6375 - accuracy: 0.2534\n",
            "Epoch 152/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6291 - accuracy: 0.2558\n",
            "Epoch 153/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6417 - accuracy: 0.2528\n",
            "Epoch 154/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6325 - accuracy: 0.2540\n",
            "Epoch 155/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6222 - accuracy: 0.2563\n",
            "Epoch 156/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6298 - accuracy: 0.2554\n",
            "Epoch 157/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6244 - accuracy: 0.2559\n",
            "Epoch 158/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6149 - accuracy: 0.2569\n",
            "Epoch 159/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6187 - accuracy: 0.2570\n",
            "Epoch 160/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6241 - accuracy: 0.2566\n",
            "Epoch 161/500\n",
            "2389/2389 [==============================] - 80s 34ms/step - loss: 3.6107 - accuracy: 0.2582\n",
            "Epoch 162/500\n",
            "2389/2389 [==============================] - 81s 34ms/step - loss: 3.6149 - accuracy: 0.2581\n",
            "Epoch 163/500\n",
            "2211/2389 [==========================>...] - ETA: 6s - loss: 3.6069 - accuracy: 0.2588"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "wivczrW29eOn"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}