model_train_and_evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 80
    },
    "colab_type": "code",
    "id": "9C-HQH4QScyN",
    "outputId": "80e84019-d7ff-40a1-f4a7-898e9a547c94"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<p style=\"color: red;\">\n",
       "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
       "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
       "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
       "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Necessary Imports\n",
    "\n",
    "import ktrain\n",
    "from ktrain import text\n",
    "import pickle as p\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3AbCNySmSinl"
   },
   "outputs": [],
   "source": [
    "with open('Dataset/processed_data/data.pkl','rb') as f:\n",
    "  x_train = p.load(f)\n",
    "with open('/content/gdrive/My Drive/aclImdb/datatest.pkl','rb') as f:\n",
    "  x_test = p.load(f)\n",
    "y_train = np.load('/content/gdrive/My Drive/aclImdb/labels.npy')\n",
    "y_test = np.load('/content/gdrive/My Drive/aclImdb/labelstest.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VAbNomssSp3A"
   },
   "outputs": [],
   "source": [
    "classes = ['positive' , 'negative' , 'neutral']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create bert-ready data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "58xfnpaaSsaJ",
    "outputId": "1e094ad7-221c-45dc-b56a-3c34188e9e75"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preprocessing train...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "done."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preprocessing test...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "done."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "(x_train,  y_train), (x_test, y_test), preproc = text.texts_from_array(x_train=x_train, y_train=y_train,\n",
    "                                                                       x_test=x_test, y_test=y_test,\n",
    "                                                                       class_names=classes,\n",
    "                                                                       preprocess_mode='bert',\n",
    "                                                                       maxlen=64, \n",
    "                                                                       max_features=50000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Bert-base uncased model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ao0gqYEqSyqy"
   },
   "outputs": [],
   "source": [
    "filepath = \"model.h5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')\n",
    "callbacks_list = [checkpoint]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "CrBmlf4uS1W1",
    "outputId": "0d243a8d-e184-413a-a128-eb30ec80fb82"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Multi-Label? False\n",
      "maxlen is 64\n",
      "done.\n"
     ]
    }
   ],
   "source": [
    "model = text.text_classifier('bert', train_data=(x_train, y_train), preproc=preproc)\n",
    "learner = ktrain.get_learner(model, train_data=(x_train, y_train),val_data=(x_test, y_test) ,batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "P-VETkmjzdHg",
    "outputId": "597c51df-397a-4346-a9cb-7ebaede93d19"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_8\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Input-Token (InputLayer)        (None, 64)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Input-Segment (InputLayer)      (None, 64)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Token (TokenEmbedding [(None, 64, 768), (3 23440896    Input-Token[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Segment (Embedding)   (None, 64, 768)      1536        Input-Segment[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Token-Segment (Add)   (None, 64, 768)      0           Embedding-Token[0][0]            \n",
      "                                                                 Embedding-Segment[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Position (PositionEmb (None, 64, 768)      49152       Embedding-Token-Segment[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Dropout (Dropout)     (None, 64, 768)      0           Embedding-Position[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Norm (LayerNormalizat (None, 64, 768)      1536        Embedding-Dropout[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 64, 768)      2362368     Embedding-Norm[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-1-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 64, 768)      0           Embedding-Norm[0][0]             \n",
      "                                                                 Encoder-1-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 64, 768)      1536        Encoder-1-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward (FeedForw (None, 64, 768)      4722432     Encoder-1-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward-Dropout ( (None, 64, 768)      0           Encoder-1-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward-Add (Add) (None, 64, 768)      0           Encoder-1-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-1-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward-Norm (Lay (None, 64, 768)      1536        Encoder-1-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 64, 768)      2362368     Encoder-1-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-2-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-1-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-2-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 64, 768)      1536        Encoder-2-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward (FeedForw (None, 64, 768)      4722432     Encoder-2-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward-Dropout ( (None, 64, 768)      0           Encoder-2-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward-Add (Add) (None, 64, 768)      0           Encoder-2-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-2-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward-Norm (Lay (None, 64, 768)      1536        Encoder-2-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 64, 768)      2362368     Encoder-2-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-3-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-2-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-3-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 64, 768)      1536        Encoder-3-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward (FeedForw (None, 64, 768)      4722432     Encoder-3-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward-Dropout ( (None, 64, 768)      0           Encoder-3-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward-Add (Add) (None, 64, 768)      0           Encoder-3-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-3-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward-Norm (Lay (None, 64, 768)      1536        Encoder-3-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 64, 768)      2362368     Encoder-3-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-4-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-3-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-4-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 64, 768)      1536        Encoder-4-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward (FeedForw (None, 64, 768)      4722432     Encoder-4-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward-Dropout ( (None, 64, 768)      0           Encoder-4-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward-Add (Add) (None, 64, 768)      0           Encoder-4-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-4-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward-Norm (Lay (None, 64, 768)      1536        Encoder-4-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 64, 768)      2362368     Encoder-4-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-5-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-4-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-5-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 64, 768)      1536        Encoder-5-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward (FeedForw (None, 64, 768)      4722432     Encoder-5-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward-Dropout ( (None, 64, 768)      0           Encoder-5-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward-Add (Add) (None, 64, 768)      0           Encoder-5-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-5-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward-Norm (Lay (None, 64, 768)      1536        Encoder-5-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 64, 768)      2362368     Encoder-5-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-6-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-5-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-6-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 64, 768)      1536        Encoder-6-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward (FeedForw (None, 64, 768)      4722432     Encoder-6-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward-Dropout ( (None, 64, 768)      0           Encoder-6-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward-Add (Add) (None, 64, 768)      0           Encoder-6-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-6-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward-Norm (Lay (None, 64, 768)      1536        Encoder-6-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 64, 768)      2362368     Encoder-6-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-7-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-6-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-7-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 64, 768)      1536        Encoder-7-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward (FeedForw (None, 64, 768)      4722432     Encoder-7-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward-Dropout ( (None, 64, 768)      0           Encoder-7-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward-Add (Add) (None, 64, 768)      0           Encoder-7-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-7-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward-Norm (Lay (None, 64, 768)      1536        Encoder-7-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 64, 768)      2362368     Encoder-7-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-8-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-7-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-8-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 64, 768)      1536        Encoder-8-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward (FeedForw (None, 64, 768)      4722432     Encoder-8-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward-Dropout ( (None, 64, 768)      0           Encoder-8-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward-Add (Add) (None, 64, 768)      0           Encoder-8-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-8-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward-Norm (Lay (None, 64, 768)      1536        Encoder-8-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 64, 768)      2362368     Encoder-8-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-9-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 64, 768)      0           Encoder-8-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-9-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 64, 768)      1536        Encoder-9-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward (FeedForw (None, 64, 768)      4722432     Encoder-9-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward-Dropout ( (None, 64, 768)      0           Encoder-9-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward-Add (Add) (None, 64, 768)      0           Encoder-9-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-9-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward-Norm (Lay (None, 64, 768)      1536        Encoder-9-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 64, 768)      2362368     Encoder-9-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 64, 768)      0           Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 64, 768)      0           Encoder-9-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 64, 768)      1536        Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward (FeedFor (None, 64, 768)      4722432     Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward-Dropout  (None, 64, 768)      0           Encoder-10-FeedForward[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward-Add (Add (None, 64, 768)      0           Encoder-10-MultiHeadSelfAttention\n",
      "                                                                 Encoder-10-FeedForward-Dropout[0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward-Norm (La (None, 64, 768)      1536        Encoder-10-FeedForward-Add[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 64, 768)      2362368     Encoder-10-FeedForward-Norm[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 64, 768)      0           Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 64, 768)      0           Encoder-10-FeedForward-Norm[0][0]\n",
      "                                                                 Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 64, 768)      1536        Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward (FeedFor (None, 64, 768)      4722432     Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward-Dropout  (None, 64, 768)      0           Encoder-11-FeedForward[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward-Add (Add (None, 64, 768)      0           Encoder-11-MultiHeadSelfAttention\n",
      "                                                                 Encoder-11-FeedForward-Dropout[0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward-Norm (La (None, 64, 768)      1536        Encoder-11-FeedForward-Add[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 64, 768)      2362368     Encoder-11-FeedForward-Norm[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 64, 768)      0           Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 64, 768)      0           Encoder-11-FeedForward-Norm[0][0]\n",
      "                                                                 Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 64, 768)      1536        Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward (FeedFor (None, 64, 768)      4722432     Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward-Dropout  (None, 64, 768)      0           Encoder-12-FeedForward[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward-Add (Add (None, 64, 768)      0           Encoder-12-MultiHeadSelfAttention\n",
      "                                                                 Encoder-12-FeedForward-Dropout[0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward-Norm (La (None, 64, 768)      1536        Encoder-12-FeedForward-Add[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Extract (Extract)               (None, 768)          0           Encoder-12-FeedForward-Norm[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "NSP-Dense (Dense)               (None, 768)          590592      Extract[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 3)            2307        NSP-Dense[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 109,140,483\n",
      "Trainable params: 109,140,483\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 419
    },
    "colab_type": "code",
    "id": "eAU5vyqkaJMC",
    "outputId": "97229b0a-887d-4287-80e9-99d14ae2b81b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "simulating training for different learning rates... this may take a few moments...\n",
      "Epoch 1/1024\n",
      "13618/13618 [==============================] - 11s 811us/step - loss: 1.5291 - acc: 0.3407\n",
      "Epoch 2/1024\n",
      "12208/13618 [=========================>....] - ETA: 0s - loss: 1.4197 - acc: 0.3857\n",
      "\n",
      "done.\n",
      "Please invoke the Learner.lr_plot() method to visually inspect the loss plot to help identify the maximal learning rate associated with falling loss.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZzd493/8ddn9snMJJHJZJFEJ5FE\nEAQRexqKoiktarm1RZVWN9W6K1Sp4r7bX0tb3GiKFlVVa4kg1JqGMIkksokgRNbJnpnMnDnL5/fH\nOTOZjNkS8z3LnPfz8TgP3/37OcfkfM51Xd/ruszdERGR7JWT6gBERCS1lAhERLKcEoGISJZTIhAR\nyXJKBCIiWU6JQEQky+WlOoCd1bdvX6+srEx1GCIiGWXWrFnr3L2itX0ZlwgqKyupqqpKdRgiIhnF\nzD5qa5+qhkREspwSgYhIllMiEBHJckoEIiJZTolARCTLBZ4IzCzXzN42symt7Cs0s4fMbKmZzTSz\nyqDjERGRHSWjRHApsKiNfRcCG919OPB74DdJiEdEJOPMX7GZWCyYaQMCTQRmNhj4EnBXG4ecCtyb\nWH4E+IKZWZAxiYhkmo/W1zLx1ulc/vDcQK4fdIngD8DPgFgb+wcBywHcPQJsBspbHmRmF5tZlZlV\nVVdXBxWriEhaWlcTAmD33sWBXD+wRGBmE4G17j7rs17L3Se7+1h3H1tR0WoPaRGRbmtLfQSAY/fu\nF8j1gywRHAmcYmbLgH8Ax5rZ31ocswIYAmBmeUAvYH2AMYmIZJytiURQVhjMqECBJQJ3v9LdB7t7\nJXA28KK7f73FYU8C5yWWz0gco0mURUSaqWuIJ4IeASWCpA86Z2a/Aqrc/UngbuB+M1sKbCCeMERE\npJn6cLyZtSgvmN/uSUkE7v4y8HJi+Zpm2+uBryUjBhGRTFUXjgJQXJAbyPXVs1hEJM3VNcQTQVGe\nEoGISFaqj0QpyMshJyeYblZKBCIiaa6+IUpxfjClAVAiEBFJe/XhmBKBiEg2qwtHKcoP7utaiUBE\nJM3FE4FKBCIiWas+HA3s0VFQIhARSXv14Whgj46CEoGISNqrU4lARCS76akhEZEsV9cQpVBPDYmI\nZK+6sDqUiYhkrYZIjI3bGqgoKwzsHkoEIiJpbH1tCHfo37MosHsoEYiIpLFtiZFHe+ipIRGR7FSf\nmIugUP0IRESyU33Ak9KAEoGISFpbvTkEBDdNJSgRiIiktWfmrwKgd4+CwO6hRCAiksYG7VYMwMj+\npYHdQ4lARCSNRaNOj4JczIKZphKUCERE0lpDNEZBgO0DoEQgIpLWGiIxCnKVCEREslZDJINLBGZW\nZGZvmtlcM1tgZte1csz5ZlZtZnMSr28HFY+ISCYKJaFqKC/Aa4eAY929xszygelm9oy7v9HiuIfc\n/QcBxiEikrGSUTUUWCJwdwdqEqv5iZcHdT8Rke6oIRKjMFOrhgDMLNfM5gBrgefdfWYrh51uZvPM\n7BEzGxJkPCIimSaj2wgA3D3q7mOAwcA4Mxvd4pCngEp33x94Hri3teuY2cVmVmVmVdXV1UGGLCKS\nVhqisUAHnIMkPTXk7puAl4ATW2xf7+6hxOpdwMFtnD/Z3ce6+9iKiopggxURSSOhSDRzSwRmVmFm\nvRPLxcDxwOIWxwxstnoKsCioeEREMlFGNxYDA4F7zSyXeML5p7tPMbNfAVXu/iTwIzM7BYgAG4Dz\nA4xHRCTjJKONIMinhuYBB7ay/Zpmy1cCVwYVg4hIpsv4xmIREflsNNaQiEiWC2V6PwIREflsVDUk\nIpLF3D3ej0Cjj4qIZKdIzHFHJQIRkWy1tT4CKBGIiGStd1dvBWDPiuDmKwYlAhGRtBWOxgDoVZwf\n6H2UCERE0lQkFk8EeWosFhHJTuFofAqXvBwL9D5KBCIiaSqSSAT5KhGIiGSn7VVDKhGIiGSlxqqh\n/ByVCEREslIkqhKBiEhWi8TUWCwiktW2lwhUNSQikpWa2ghUNSQikp1CkSgAhXm5gd5HiUBEJE2F\nIjHMVCIQEclaoUiMorxczJQIRESyUkMkRmF+8F/TSgQiImkqFIkGPl8xKBGIiKSt2lCU4vxgG4pB\niUBEJG2tqwlRXloY+H2UCERE0tS6mhB9SwsCv09gicDMiszsTTOba2YLzOy6Vo4pNLOHzGypmc00\ns8qg4hERyTTrahrom+ElghBwrLsfAIwBTjSzw1occyGw0d2HA78HfhNgPCIiGSMSjbFxW4YnAo+r\nSazmJ17e4rBTgXsTy48AX7CgH5gVEckAG7Y14A59yzI4EQCYWa6ZzQHWAs+7+8wWhwwClgO4ewTY\nDJS3cp2LzazKzKqqq6uDDFlEJC2s29oAQN+SDG4jAHD3qLuPAQYD48xs9C5eZ7K7j3X3sRUVFV0b\npIhIGlpXEwK6QYmgkbtvAl4CTmyxawUwBMDM8oBewPpkxCQiks7W1yYSQSa3EZhZhZn1TiwXA8cD\ni1sc9iRwXmL5DOBFd2/ZjiAiknVqQvGRR0sL8wK/V5B3GAjca2a5xBPOP919ipn9Cqhy9yeBu4H7\nzWwpsAE4O8B4REQyRiicGII6CWMNBZYI3H0ecGAr269ptlwPfC2oGEREMlUoEp+drCjguQhAPYtF\nRNJSfTialLkIQIlARCQt1YejSZmLAJQIRETSUigSoygJ7QOgRCAikpbqw9HA5ypupEQgIpKG6sMq\nEYiIZLX47GQqEYiIZC2VCEREslx9OEphEqapBCUCEZG0VB+OUqREICKSvWpCEcqSMM4QKBGIiKSl\n2lCUkkKVCEREslZtKEKJSgQiItnJ3altiCRlCGpQIhARSTt14SgxRyUCEZFsVROKAEoEIiJZq7Zp\ndjI1FouIZKXaxhJBgUoEIiJZqbFqSI3FIiJZqjYd2wjM7FIz62lxd5vZbDM7IejgRESyUVOJoCiN\nEgHwLXffApwA7AZ8A/h1YFGJiGSxdK0aapw082Tgfndf0GybiIh0obSsGgJmmdk04ongOTMrA2LB\nhSUikr1qEo+P9kjS6KOdTTcXAmOAD9x9m5n1AS4ILiwRkewVCkcpzMshJyc5FS+dLREcDrzr7pvM\n7OvA1cDm9k4wsyFm9pKZLTSzBWZ2aSvHTDCzzWY2J/G6ZuffgohI9xKOOgW5yXuos7MlgjuAA8zs\nAOCnwF3AfcDn2zknAvzU3WcnqpJmmdnz7r6wxXGvufvEnQ1cRKS7isRi5OUmrxm2sykn4u4OnArc\n5u7/B5S1d4K7r3L32YnlrcAiYNBnCVZEJBuEozHyk1gi6OydtprZlcQfG33azHKA/M7exMwqgQOB\nma3sPtzM5prZM2a2b2evKSLSXYWjnpaJ4CwgRLw/wWpgMPDbzpxoZqXAo8CPE30RmpsNfM7dDwBu\nBZ5o4xoXm1mVmVVVV1d3MmQRkcwUjqZh1VDiy/8BoJeZTQTq3f2+js4zs3ziSeABd3+sletucfea\nxPJUIN/M+rZy3GR3H+vuYysqKjoTsohIxoqkY4nAzM4E3gS+BpwJzDSzMzo4x4C7gUXufnMbxwxI\nHIeZjUvEs77z4YuIdD/haIy8JD06Cp1/aujnwCHuvhbAzCqAF4BH2jnnSOJtCu+Y2ZzEtquAPQDc\n/U7gDOASM4sAdcDZiUZpEZGslezG4s4mgpzGJJCwng5KE+4+nQ6GoXD324DbOhmDiEhWiMSc/CS2\nEXQ2ETxrZs8BDybWzwKmBhOSiEh2a4jEyEu3EoG7/7eZnU68ugdgsrs/HlxYIiLZKxJzipM0zhB0\nvkSAuz9K/AkgEREJUDgaoyxJcxFAB4nAzLYCrTXeGuDu3jOQqEREslg46uTlpEnVkLu3O4yEiIh0\nvUg0RkFemnUoExGR5In3I0izDmUiIpI84ain3xATIiKSPOFoLKnzESgRiIikmVAkRkGeEoGISFZy\nd2pDkaRNXA9KBCIiaaUhGiMSc0qVCEREslNtKApASUHyehYrEYiIpJHaUASAHioRiIhkp9qGeCJQ\n1ZCISJZqLBGosVhEJEupjUBEJMupRCAikuVqGhNBgRKBiEhW2taQqBoqVNWQiEhWqlHVkIhIdqsN\nRcjNMQo11pCISHba1hClpCAXMw1DLSKSlWpCkaR2JgMlAhGRtFIbiiR1eAlQIhARSSu1DdGkNhRD\ngInAzIaY2UtmttDMFpjZpa0cY2Z2i5ktNbN5ZnZQUPGIiGSC2lCE0iQ+OgrBlggiwE/dfR/gMOD7\nZrZPi2NOAkYkXhcDdwQYj4hI2qsNReiRxM5kEGAicPdV7j47sbwVWAQManHYqcB9HvcG0NvMBgYV\nk4hIultX00Dv4vyk3jMpbQRmVgkcCMxssWsQsLzZ+id8OllgZhebWZWZVVVXVwcVpohISm1riLCu\nJsTQipKk3jfwRGBmpcCjwI/dfcuuXMPdJ7v7WHcfW1FR0bUBioikiYZIDIDi/O7TRoCZ5RNPAg+4\n+2OtHLICGNJsfXBim4hI1mmIxhNBXm5yH+gM8qkhA+4GFrn7zW0c9iTwzcTTQ4cBm919VVAxiYik\ns0jUASjITV6vYoAgm6aPBL4BvGNmcxLbrgL2AHD3O4GpwMnAUmAbcEGA8YiIpLVwY4kgJ7klgsAS\ngbtPB9pNa+7uwPeDikFEJJOEEyWC/CQOOAfqWSwikjYaSwT5OcmtGlIiEBFJE41tBN2msVhERHZO\nOJYoESS5sViJQEQkTTT2IyhQiUBEJDvVh+PzFRd2pw5lIiLSeY2JoFv1LBYRkc6rD8erhoryVTUk\nIpKVGksERSoRiIhkpzpVDYmIZLftVUNKBCIiWamxRFCoISZERLJTKBylMC+HHA0xEZyv3zWTyklP\nE0mM5yEikk7qw9GkVwtBliWC6UvXAXD8718lFImmOBoRkR3VhaNJbyiGLEoEL727tmn5w3W1PD5b\nE6GJSHqpD8eS3ocAsigRhMI7lgDuf+Mjlq2r5Zp/zW96dldEJJXqUlQ1FOQMZWllz4rSHdYXrNzC\nhN+9DMCYIb057aDBKYhKRGQ7tREEbET/Ml772TFMv+IYJp00aod9Mz/YkKKoRES2C6lqKHhD+vRg\n8G49uOjoYTtsf27hauKzZoqIpI4ai5MoN8f47uf3bFrftC3M7I83pTCi9BOLKTGKJJuqhpJs0kmj\nWHz9icy95gQATr9jBu+u3pqSWKYtWE3lpKeZ/fHGXSqZbKkP4+5Ubw11ScP32i31DLtqKpWTnmbW\nRxs/8/Uaba0PN83J2p6FK7dQvTUExCfqmP1xPIaNtQ0s37CtafIOke4mVSWCrGksbk1Rfi5F+bns\nPbAni1Zt4Yt/eJVfnbov3zy8stPXmL9iMxNvnc5Pjx/JD78wotPnrdxUx83PL+GRWZ80bTvt9hn0\nLS3k9SuPJb+TMxTNeH8d//XnmTtsW3LDSRTsRBf1UCTKS4vXMmbIbjw7fxU3TVvStO/0O2aQYzCw\nVzGv/PcE5q/cwtC+JfQqzu/09QGiMeeLv3+VlZvrAbj2y/tQ2beEY/bqx7qaED/+xxyKC3L59Wn7\ncfItrwFw0dFD2VwX5p9Vn7BnRQnvV9cCMLxfKSfs058fHDucHgVZ/Scs3Ux9OJb0SWkgyxNBoz99\n/WDG//YlAK751wIOH1bOiP5lxGLOi4vXcsyofuS20eW7sZPaTc8vIRJzLjt+JPXhKN+8501G9Cvl\ntIMGs2jVFo7buz/PL1rDQXv0pqwwv+l+La2rCfHS4rWcsO+ADuNeuHIL371/1qe2PzxrOece+rkd\nrjn1nVVMGNmPPcp7NG1viMSY/Or7PPjmclZsqvvUdX507HBueXEpMYcVm+oY/vNntr/vK45h8G49\nPnVOW6YtWN2UBACue2ohAL175NMQibGtIV6SeX7hmqZj/vzah03L71fXMqh3MSs21bF0bQ1L19Zw\n+8vvA3D9V0YzrrIPA3oVdTpBXfOv+YwfUcFx+/Tv9HsQCVooHE1JY7FlWiPp2LFjvaqqqsuv+/zC\nNVx03/brvnT5BK6fspAXF6/lR8cO5ycn7NXqeZc9NIfH3961zmnjR1bw9UP34IAhvelVnM+ClVs4\n/Y4ZAPTvWcjlJ+zF+JEV9O9ZRH04ysJVW5j8ygf89ISRfK68hJFXx7+Y/3DWGGZ+uIHK8h48u2A1\nb3+8iX5lhRw2rJwn565sul/vHvnMuvr4pqR224vv8btmv/4bmcGMSccysFcxj7/9CdMWrOGZ+as/\nddxrPzuGIX06lwwu+MubLFy1hQcvOoyqjzbys0fm7bD/hq+M5uon5jetz732BP4+82NeXLyGUw7Y\nnUOHlTOiXylvL9/EjU8varPK6uov7c23WzwM0NLN097llheXAjDnmuPp3aOgU+9BJGjDr5rKReOH\nccWJozo+eCeZ2Sx3H9vqPiWC7WpDER5882NueHrRp/bN++UJ9Cza/mtzzZZ6npq7khueXsS4oX34\n3oQ9Of8vb7V63dGDejJ/xRYAKst7UL01xLeOGspPW0kuh9z4QlP9eKNj9qrgpXerm9bN4qMT1odj\n7DeoF098/8imL/fX3qvmG3e/2e77LCvKY/zICp6et6pp218vOITSwjzGVvZp9Zw5yzdRnJ/LXgPK\nqJz0NADHjurHH88eQ1lR27/CF63awtzlm5j02Duce+ge3PjV/XbYv7G2gY82bGPMkN4AuDtmHQ+4\nNf29dcz8cD0T9qrg9Dte32Hf+UdUsmdFCceM6kefkgJ6FOQRizn/M3URd03/cIdj+5YW8sJPxisZ\nSMpFY86eV03lsuNGculxna9m7qyUJAIzuweYCKx199Gt7J8A/Ato/Jf5mLv/qqPrBpkIGv3Xn99g\nxvvrd9h285kH7NDp7Iw7ZlCV+FV63Sn7ct4RlUSiMVZuque2l97julNG89jbnzB+RAV9Sgq4+fkl\nfG/CnpSXFrZ77+qtIWZ9tJG3l2/kT6980O6x/coKmTHpWPKatSe4O3/5zzLWbg1x5yvvc9iwPtxy\nzoH0KMhjzHXTiLR4Gmj6FceQn5tD/55FnfpsAF5/fz1fv3tm/A+3ooR//3QCtaEID1ctZ2hFKeNH\n9KUhGmPmBxv45j3bk9KzPz6aUQN6dvo+nbWlPsziVVupC0c5755PJ8HfnL4fVzz6zg7bnvj+kbyw\ncA23vbSUUQPKePIHR+1Uu4pIV6sPRxn1i2f52Yl78b0Jw7v8+qlKBOOBGuC+dhLB5e4+cWeum4xE\nEI7GWL25nvpwlD0rShl21VQA5l/3Rdyd/Nwc9v/lNBqiMR7/3hGMGdK7U79id1Ys5ixYuYXLH55L\nZd8e3H7uwWyuC3PZQ3M4Z9wQThw9sN3zl62rpW9ZIaWF8aagpWtreGvZBirLS/jjv5dww1dGM7xf\n2S7F9s4nm/nybdMBOHRoH2Z+uL1TXvOG3UYtE2kQYjHn8ofn8n51De+s2ExrT8Duu3tPJp00iqNH\nVADwP1MXMfnVD/jhscNbLaGJJMuW+jD7/3Jap6o3d0XKqobMrBKYkmmJoKXvPzCbp9/ZXo3S+EV3\n/4Xjmr5QstF7a7Zy/O9fbfeYUw7YnVvOOTBJEX3alHkr+cHf3+bEfQdw5zcObvWYn/xzDv+as5K/\nf/tQDh1WnuQIReLW1YQYe8MLO/3kYme1lwhS/dTQ4WY2F1hJPCksaO0gM7sYuBhgjz32SGJ4cf93\n7kEcPP1DfjUl/qRL46/do4b3TXos6WRE/zIe/u7h3PLv97j6S/uw14B46eKtZRuY8/Emvn300EBK\nSjtj4v67c+jQcspL2m4DuHbivry6pJqzJr9Bbo5x85kHcOqYQUmMUoSmPjYFnXx0vCulslJ0NvA5\ndz8AuBV4oq0D3X2yu49197EVFan5Bf6to4by3o0nccxe8ftf/5XRKf+SSweHVPbh/gsPbUoCjdsu\nGj8sbT6firLCdmd86tUjn9vPjZcWojHn0n/M4b7Xl+3Q+e0XT8xn+FVTeW7Bp5+eEukKjR0lO9uH\nqCulrETg7luaLU81s9vNrK+7r0tVTB3Jz83hLxeMY9XmOvqXdb5xVdLfuKF9+OPZY9hnYE+O//2r\nXPOvBTw2ewVnjh1CNBbj/jc+AuA7989ixqRjKS8toCA3B3eSPq2gdE9NJYIUPLSQskRgZgOANe7u\nZjaOeOlkfQenpYWBvYpTHYIEoLE66KGLD+OsyW8wZ/km5izfPgbV+UdU8tcZyzji1y+SY9C/ZxGr\nNtdTWd6Dv1wwjqF9S1IVunQDoe5YIjCzB4EJQF8z+wS4FsgHcPc7gTOAS8wsAtQBZ3umdWqQbunQ\nYeW89fPjOP2OGewzsCe5OUZFWSFXnjyKaMy5/42PiDlNg4MtW7+NHz44m0cvOYLCvOQPDyDdQzga\n//orTEGJQB3KRHZS805vsZjzyKxP+Nmj8zikcjf+9u1DlQxkl7y1bANfu/N1/nbhoRw1ousfRGnv\nqSH1oBHZSc0bwXNyjDMPGcKVJ43irWUbmfzKByzfsI3VzcZVmrZgNd9/YDY1oQizP97IM80eRRZp\n1NhYnFVtBCLdyXc+vycz3l/PTc8v4abnl7Bbj3xmTPoCf/j3kqYe4s37ojz9o6PYd/deqQpX0lBD\ntLGNIPkPH6hEINJFLjt+ZNPyxm1h9r7m2aYkUJCXQ45t/7X3u+feJdqi63OmVdNK11KJQKQbGDOk\nNy9dPoHdexdx+cPzeGruSsYM6c3/nXsQg3oXUx+O4g5/evV9/vDCe4y78QUevPgw+vcsIjfHOO32\n/7B8Qx13nTeWI7O8s2I2SmWHMiUCkS7U+AjpreccyB/PGrNDH4PGp4x+fNxIyoryuX7KQk5oZYiO\nc++ayaDexfzpGwczepCqj7JFKksEqhoSCUh7Hc0uPGoopxywO3nNjjn/iEqqrj6O3XsVsWJTHRNv\nnc5/lrbevzIWc6YtWM19ry9TlVI3URuKAKRk1j09PiqSIu5OzPnU7Hd1DVH+NWcFkx6LD539n0nH\nMmXuSv73mcUM6FlEOBpjfW1D0/FHj+jLtV/el+H9SpMav3Stxnk+3rvxpEA6laXzoHMiWcvMaO0B\nkeKCXM4etwczP9zA42+v4Mhfv9i0b+O2hqYeqPsP7sXeA3ryUNVyjrv5FcpLCijIy+EvFxzCXv3L\n0masJ+lYpNm4Vt2qZ7GIfDa/PGVf+vcs4s5X3ueL+/bnJ8fvxcj+pZ/6gj9ieDn/M3URa7bEZ7Y7\n8Q+vAfF5Iq46eW8OSMz+JumrcSKsc8Ylf3RlUNWQSNpbu6Wefh3MINfY2/nqJ97hb298vMO+Q4f2\n4bpT9w1kdjjpGndP/5Drpyzk0UuO4ODP7RbIPdSzWCSDdZQEYHtv5+tPHc0LP/k83/n8MM4/opJR\nA8qY+eEGzr/nrabGSEk/62pC5OcaB+2RmtKbqoZEuhEzY3i/Uq48ae+mba8sqea8e97kkgdmM/kb\nBzc9xirpIxSOUZSXm7J2HZUIRLq5o4f35eT9BvDqkmpG/eJZbnvxvVSHJC2EIlEK81P3daxEINLN\n5eQYt55zEBceNRSA301bwk3T3k1xVNJcKBJL6ai1SgQiWSA3x/jFxH2YedUXGNa3hFtfXMqVj81j\na32YNVvqO76ABKo+HE3JPASN1EYgkkX69yxi2mXjOfiGF3jwzeU8+Obypn3H7d2fP549hpJCfS0k\nS304ygMzP2bKvFXs1b+s4xMCohKBSJbJy81h+hXH8KNjh3PR0UMZVhEfH+mFRWvY99rnuP/1ZWyt\nD1Ojp4wC99Bby7l+ykIAThw9IGVxqB+BSJZr/A749bOLm4bNbnTx+GEcObwvQ8tL2KO8RyrC69Ya\n+3388zuHM25on0Dv1V4/AiUCEWkSikS5Z/oyfvPs4k/tO2avCm4+cwx5uUZDJEav4nyc+JAIDZFY\nSkbNTDexmPPh+loqy0vYXBfm7Y83UheOUlqYR3F+LlvqI+zRpwfF+bk0RGNMenQe9ZEoU354dOCx\nKRGIyE5bsamOu177gEWrtrChtoEla2raPf60gwbxiy/tQ+8e+Sl5Ht7d2VDbQDTmlJcW4u7ktTJu\nT+N3XjTmLK2uoU9JAYV5uWyobSA/1ygvKWRzXZh1NSFWba6nIRJjaN8SYu58srGO2lAEM6htiLKh\npoHahgj5uUZtKMoLi9bwycY6CnJzmmYc68iFRw3lFxP36dLPojVKBCLymT02+xNueHoRG2obKMzL\nIZQoBUzcbyBrttbzn6Xrm47df3Avbj/3IHLM2K1HAcUFrT8aGY05OQbPLVjDw1XLGdG/jNWb67ji\npFEM7FUMwPqaEOtrGwiFYzREo6zcVE95aQFzlm/ipNEDeWz2Jzw2ewUQT14AZuAO5SUF9Ckp4L21\nNZSXxONYtbmeXsX5bGg2gutnUZAXLxE1Din+7aOHUdcQIT83h30H9WTPilJCkRibt4VZvHoL5aWF\n1IYiFObnsr4mxCUT9kzKo6NKBCISuFkfbeDKx95pteRw5tjBjOxfRs+ifPqWFXD14/PJz8vho/Xb\n2rxeRVkhn+vTg6qPNnbq/keP6Evf0kJG9i9j07b4L/VN28IsXVvD4tVbKSnI5YAhvRnUuxgHXlq8\nluH9Sjl5v4HUhaP071lITShK9ZZ6zIyyojx2711M39JC5i7fRFFBLmMG96ZncR4xh5KCXHYrKSAv\nx4g5RGIxNtaGGdCr4yFBUkGJQESS6sN1tVz6j7eZ98nmTh0/fmQFd5x7EA2RGK8tXcfr76/nwTfj\ng+ftu3tPRvYv4/G3V/DVAwcx++ONfLR+GyeNHkBFWSGjBvTkxNED6FNS0Oq13Z0tdRF69cjvsveX\niZQIRCRllq7dyvVTFlEfjjLzww0M6l3MJRP2pG9pAfsM7MWQPsWttinMWb4JgAMG99LcCl0gJRPT\nmNk9wERgrbuPbmW/AX8ETga2Aee7++yg4hGR1Bjer4x7vzVup88bo3kUkibI573+CpzYzv6TgBGJ\n18XAHQHGIiIibQgsEbj7q8CGdg45FbjP494AepvZwKDiERGR1qWyB8ggYHmz9U8S20REJIkyoiug\nmV1sZlVmVlVdXZ3qcEREupVUJoIVwJBm64MT2z7F3Se7+1h3H1tRUZGU4EREskUqE8GTwDct7jBg\ns7uvSmE8IiJZKcjHRx8EJsdCEG4AAAkQSURBVAB9zewT4FogH8Dd7wSmEn90dCnxx0cvCCoWERFp\nW2CJwN3P6WC/A98P6v4iItI5Gdez2Mw2A81n3+4FbG6x3Nq2vsC6nbxd8+t0dl/L7e2tt4z3s8Sq\neDM73vbi7uq/3V2Nt6NlxZve8fZ299YbWd09o17A5LbWG5fb2Fb1We/VmX3txddRvJ8lVsWb2fF2\nEHeX/u3uarwdLSvezI03Ix4fbeGpdtafamdbV9yrM/vai6/lest4P0usHZ2veNM73o7+jndWR+fu\nSrwdLSvejvenZbwZVzW0q8ysytsYcCndZFKsoHiDpniDpXgzpENZF5mc6gB2QibFCoo3aIo3WFkf\nb9aUCEREpHXZVCIQEZFWKBGIiGQ5JQIRkSyX9YnAzI42szvN7C4zm5HqeDpiZjlmdqOZ3Wpm56U6\nno6Y2QQzey3xGU9IdTydYWYlidFuJ6Y6lo6Y2d6Jz/YRM7sk1fF0xMy+YmZ/NrOHzOyEVMfTETMb\nZmZ3m9kjqY6lLYm/13sTn+u5u3KNjE4EZnaPma01s/kttp9oZu+a2VIzm9TeNdz9NXf/LjAFuDfd\n4yU+oc9gIEx8DofAdFG8DtQARWRGvABXAP8MJsod4uqKv99Fib/fM4EjMyDeJ9z9IuC7wFkZEO8H\n7n5hkHG2ZidjPw14JPG5nrJLN9zZHmrp9ALGAwcB85ttywXeB4YBBcBcYB9gP+Jf9s1f/Zqd90+g\nLN3jBSYB30mc+0gGxJuTOK8/8EAGxHs8cDZwPjAx3eNNnHMK8AzwX5kQb+K8m4CDMijeQP+tfcbY\nrwTGJI75+67cL7BB55LB3V81s8oWm8cBS939AwAz+wdwqrv/L9BqUd/M9iA+DPbWAMPtkngTI7k2\nJFajwUXbdZ9vwkagMIg4G3XR5zsBKCH+D6zOzKa6eyxd401c50ngSTN7Gvh7ELF2VbxmZsCvgWfc\nfXZQsXZVvKmyM7ETL2kPBuawi7U8GZ0I2tDaFJiHdnDOhcBfAouofTsb72PArWZ2NPBqkIG1Yafi\nNbPTgC8CvYHbgg2tVTsVr7v/HMDMzgfWBZUE2rGzn+8E4lUDhcSHdk+2nf37/SFwHNDLzIZ7fEj6\nZNrZz7ccuBE40MyuTCSMVGkr9luA28zsS+ziMBTdMRHsNHe/NtUxdJa7byOeuDKCuz9GPHllFHf/\na6pj6Ax3fxl4OcVhdJq730L8iysjuPt64u0Zacvda/mM87lkdGNxGzo9BWaaULzBUrzBUrzJE1js\n3TERvAWMMLOhZlZAvOHvyRTH1B7FGyzFGyzFmzzBxZ7MlvAAWtYfBFax/VHKCxPbTwaWEG9h/3mq\n41S8ilfxKt50jl2DzomIZLnuWDUkIiI7QYlARCTLKRGIiGQ5JQIRkSynRCAikuWUCEREspwSgQTO\nzGqScI9TOjnEdFfec4KZHbEL5x1oZncnls83s1SMwfQpZlbZctjjVo6pMLNnkxWTJIcSgWQMM8tt\na5+7P+nuvw7gnu2NxzUB2OlEAFxFBo2305y7VwOrzCzQuQ8kuZQIJKnM7L/N7C0zm2dm1zXb/oSZ\nzTKzBWZ2cbPtNWZ2k5nNBQ43s2Vmdp2ZzTazd8xsVOK4pl/WZvZXM7vFzGaY2QdmdkZie46Z3W5m\ni83seTOb2rivRYwvm9kfzKwKuNTMvmxmM83sbTN7wcz6J4YI/i5wmZnNsfhMdxVm9mji/b3V2pel\nmZUB+7v73Fb2VZrZi4nP5t+J4dExsz3N7I3E+72htRKWxWepetrM5prZfDM7K7H9kMTnMNfM3jSz\nssR9Xkt8hrNbK9WYWa6Z/bbZ/6vvNNv9BLBLM2FJmkp1V2q9uv8LqEn89wRgMmDEf4RMAcYn9vVJ\n/LcYmA+UJ9YdOLPZtZYBP0wsfw+4K7F8PnBbYvmvwMOJe+xDfAx3gDOID9WcAwwgPkfCGa3E+zJw\ne7P13aCpF/63gZsSy78ELm923N+BoxLLewCLWrn2McCjzdabx/0UcF5i+VvAE4nlKcA5ieXvNn6e\nLa57OvDnZuu9iE9e8gFwSGJbT+IjDvcAihLbRgBVieVKEhOhABcDVyeWC4EqYGhifRDwTqr/rvTq\nupeGoZZkOiHxejuxXkr8i+hV4Edm9tXE9iGJ7euJT77zaIvrNA5rPYv4WPytecLjcwksNLP+iW1H\nAQ8ntq82s5faifWhZsuDgYfMbCDxL9cP2zjnOGAfM2tc72lmpe7e/Bf8QKC6jfMPb/Z+7gf+X7Pt\nX0ks/x34XSvnvgPcZGa/Aaa4+2tmth+wyt3fAnD3LRAvPRAfv34M8c93ZCvXOwHYv1mJqRfx/ycf\nAmuB3dt4D5KBlAgkmQz4X3f/0w4b45OrHAcc7u7bzOxl4nMcA9S7e8uZ2EKJ/0Zp+2841GzZ2jim\nPbXNlm8Fbnb3JxOx/rKNc3KAw9y9vp3r1rH9vXUZd19iZgcRH5TsBjP7N/B4G4dfBqwBDiAec2vx\nGvGS13Ot7Csi/j6km1AbgSTTc8C3zKwUwMwGmVk/4r82NyaSwCjgsIDu/x/g9ERbQX/ijb2d0Yvt\n476f12z7VqCs2fo04jNwAZD4xd3SImB4G/eZQXxoYYjXwb+WWH6DeNUPzfbvwMx2B7a5+9+A3xKf\n7/ZdYKCZHZI4pizR+N2LeEkhBnyD+Fy4LT0HXGJm+YlzRyZKEhAvQbT7dJFkFiUCSRp3n0a8auN1\nM3sHeIT4F+mzQJ6ZLSI+n+0bAYXwKPEhfRcCfwNmA5s7cd4vgYfNbBawrtn2p4CvNjYWAz8CxiYa\nVxfSysxW7r6Y+DSNZS33EU8iF5jZPOJf0Jcmtv8Y+Eli+/A2Yt4PeNPM5gDXAje4ewNwFvGpTecC\nzxP/NX87cF5i2yh2LP00uov45zQ78Ujpn9he+joGeLqVcyRDaRhqySqNdfYWn4v2TeBId1+d5Bgu\nA7a6+12dPL4HUOfubmZnE284PjXQINuP51XiE75vTFUM0rXURiDZZoqZ9Sbe6Ht9spNAwh3A13bi\n+IOJN+4asIn4E0UpYWYVxNtLlAS6EZUIRESynNoIRESynBKBiEiWUyIQEclySgQiIllOiUBEJMsp\nEYiIZLn/D3iBZvgqxWHuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Try with different learning rates to determine optimal\n",
    "\n",
    "learner.lr_find()\n",
    "learner.lr_plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 306
    },
    "colab_type": "code",
    "id": "zOdR9ZTfS3rb",
    "outputId": "69abc2c8-c6a2-4fa8-982f-73797c2e6c4e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "begin training using triangular learning rate policy with max lr of 2e-05...\n",
      "Train on 15131 samples, validate on 1869 samples\n",
      "Epoch 1/3\n",
      "15131/15131 [==============================] - 201s 13ms/step - loss: 0.9211 - acc: 0.5425 - val_loss: 0.9707 - val_acc: 0.5484\n",
      "\n",
      "Epoch 00001: loss improved from inf to 0.92112, saving model to /content/gdrive/My Drive/aclImdb/model.h5\n",
      "Epoch 2/3\n",
      "15131/15131 [==============================] - 179s 12ms/step - loss: 0.7841 - acc: 0.6414 - val_loss: 0.9846 - val_acc: 0.5661\n",
      "\n",
      "Epoch 00002: loss improved from 0.92112 to 0.78408, saving model to /content/gdrive/My Drive/aclImdb/model.h5\n",
      "Epoch 3/3\n",
      "15131/15131 [==============================] - 178s 12ms/step - loss: 0.6803 - acc: 0.7063 - val_loss: 1.0440 - val_acc: 0.5548\n",
      "\n",
      "Epoch 00003: loss improved from 0.78408 to 0.68025, saving model to /content/gdrive/My Drive/aclImdb/model.h5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fe6949dbb38>"
      ]
     },
     "execution_count": 122,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.autofit(2e-5, 3, callbacks = callbacks_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 238
    },
    "colab_type": "code",
    "id": "rDiaz92LS_y5",
    "outputId": "0ddd1c34-ee18-49d0-ee51-36880dfc649f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    positive       0.61      0.56      0.59       582\n",
      "    negative       0.57      0.61      0.59       533\n",
      "     neutral       0.54      0.54      0.54       754\n",
      "\n",
      "    accuracy                           0.57      1869\n",
      "   macro avg       0.57      0.57      0.57      1869\n",
      "weighted avg       0.57      0.57      0.57      1869\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[328,  70, 184],\n",
       "       [ 43, 323, 167],\n",
       "       [168, 176, 410]])"
      ]
     },
     "execution_count": 111,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.validate(val_data=(x_test, y_test),class_names=classes)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "model_train_and_evaluate.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}