Neural_Machine_Translation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Machine Translation.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMHiS10k6lpyCqHWoB0Oqpf",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ShivanshuPurohit/Practice-models/blob/master/Neural_Machine_Translation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_az1fTLp-H1o",
        "colab_type": "code",
        "outputId": "7d0d05b3-d055-4fbd-dd53-9e8334598a3c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 752
        }
      },
      "source": [
        "!wget https://github.com/udacity/aind2-nlp-capstone/blob/master/data/small_vocab_en?raw=true\n",
        "!wget https://github.com/udacity/aind2-nlp-capstone/blob/master/data/small_vocab_fr?raw=true"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-05-17 09:24:09--  https://github.com/udacity/aind2-nlp-capstone/blob/master/data/small_vocab_en?raw=true\n",
            "Resolving github.com (github.com)... 140.82.113.4\n",
            "Connecting to github.com (github.com)|140.82.113.4|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://github.com/udacity/aind2-nlp-capstone/raw/master/data/small_vocab_en [following]\n",
            "--2020-05-17 09:24:09--  https://github.com/udacity/aind2-nlp-capstone/raw/master/data/small_vocab_en\n",
            "Reusing existing connection to github.com:443.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/udacity/aind2-nlp-capstone/master/data/small_vocab_en [following]\n",
            "--2020-05-17 09:24:10--  https://raw.githubusercontent.com/udacity/aind2-nlp-capstone/master/data/small_vocab_en\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 9085267 (8.7M) [text/plain]\n",
            "Saving to: ‘small_vocab_en?raw=true’\n",
            "\n",
            "small_vocab_en?raw= 100%[===================>]   8.66M  18.0MB/s    in 0.5s    \n",
            "\n",
            "2020-05-17 09:24:12 (18.0 MB/s) - ‘small_vocab_en?raw=true’ saved [9085267/9085267]\n",
            "\n",
            "--2020-05-17 09:24:14--  https://github.com/udacity/aind2-nlp-capstone/blob/master/data/small_vocab_fr?raw=true\n",
            "Resolving github.com (github.com)... 140.82.113.4\n",
            "Connecting to github.com (github.com)|140.82.113.4|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://github.com/udacity/aind2-nlp-capstone/raw/master/data/small_vocab_fr [following]\n",
            "--2020-05-17 09:24:15--  https://github.com/udacity/aind2-nlp-capstone/raw/master/data/small_vocab_fr\n",
            "Reusing existing connection to github.com:443.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/udacity/aind2-nlp-capstone/master/data/small_vocab_fr [following]\n",
            "--2020-05-17 09:24:15--  https://raw.githubusercontent.com/udacity/aind2-nlp-capstone/master/data/small_vocab_fr\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 10135742 (9.7M) [text/plain]\n",
            "Saving to: ‘small_vocab_fr?raw=true’\n",
            "\n",
            "small_vocab_fr?raw= 100%[===================>]   9.67M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2020-05-17 09:24:17 (75.2 MB/s) - ‘small_vocab_fr?raw=true’ saved [10135742/10135742]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BPnfwp3yq8uQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "\n",
        "\n",
        "def load_data(path):\n",
        "    \"\"\"\n",
        "    Load dataset\n",
        "    \"\"\"\n",
        "    input_file = os.path.join(path)\n",
        "    with open(input_file, \"r\", encoding='utf-8') as f:\n",
        "        data = f.read()\n",
        "\n",
        "    return data.split('\\n')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C84WoFkW-oov",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%load_ext autoreload\n",
        "%autoreload 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nNchawwkAnMG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "ad4cfc5f-06b8-413c-81a7-702c12ea0435"
      },
      "source": [
        "import collections\n",
        "import numpy as np\n",
        "\n",
        "import tensorflow as tf\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.models import Model\n",
        "from keras.layers import GRU, Input, Dense, TimeDistributed, Activation, RepeatVector, Bidirectional\n",
        "from keras.layers.embeddings import Embedding\n",
        "from keras.optimizers import Adam\n",
        "from keras.losses import sparse_categorical_crossentropy"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rOopvoZMsOtu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 586
        },
        "outputId": "90bb7b58-0133-488e-865f-17189debc3b1"
      },
      "source": [
        "from tensorflow.python.client import device_lib\n",
        "print(device_lib.list_local_devices())"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[name: \"/device:CPU:0\"\n",
            "device_type: \"CPU\"\n",
            "memory_limit: 268435456\n",
            "locality {\n",
            "}\n",
            "incarnation: 1206630308820650437\n",
            ", name: \"/device:XLA_CPU:0\"\n",
            "device_type: \"XLA_CPU\"\n",
            "memory_limit: 17179869184\n",
            "locality {\n",
            "}\n",
            "incarnation: 13303772889712347525\n",
            "physical_device_desc: \"device: XLA_CPU device\"\n",
            ", name: \"/device:XLA_GPU:0\"\n",
            "device_type: \"XLA_GPU\"\n",
            "memory_limit: 17179869184\n",
            "locality {\n",
            "}\n",
            "incarnation: 11161384669326527919\n",
            "physical_device_desc: \"device: XLA_GPU device\"\n",
            ", name: \"/device:GPU:0\"\n",
            "device_type: \"GPU\"\n",
            "memory_limit: 11150726272\n",
            "locality {\n",
            "  bus_id: 1\n",
            "  links {\n",
            "  }\n",
            "}\n",
            "incarnation: 16190803103595339390\n",
            "physical_device_desc: \"device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7\"\n",
            "]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vg9id0UpsTO1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        },
        "outputId": "dff5629e-2061-463a-e35f-720568efe412"
      },
      "source": [
        "# Add-on : Check framework versions being used \n",
        "import tensorflow as tf\n",
        "import keras as K\n",
        "import sys\n",
        "print(\"Python\", sys.version)\n",
        "print(\"Tensorflow version\", tf.__version__)\n",
        "print(\"Keras version\", K.__version__)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Python 3.6.9 (default, Apr 18 2020, 01:56:04) \n",
            "[GCC 8.4.0]\n",
            "Tensorflow version 2.2.0\n",
            "Keras version 2.3.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j7QyP2xGsXTD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "09892d16-168b-443e-d733-b18564224775"
      },
      "source": [
        "# Load English data\n",
        "english_sentences = load_data('small_vocab_en')\n",
        "# Load French data\n",
        "french_sentences = load_data('small_vocab_fr')\n",
        "\n",
        "print('Dataset Loaded')"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Dataset Loaded\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WsUDUPAysgIQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        },
        "outputId": "057e73e5-5b20-4253-b022-8b2640aa8803"
      },
      "source": [
        "for sample_i in range(2):\n",
        "    print('small_vocab_en Line {}:  {}'.format(sample_i + 1, english_sentences[sample_i]))\n",
        "    print('small_vocab_fr Line {}:  {}'.format(sample_i + 1, french_sentences[sample_i]))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "small_vocab_en Line 1:  new jersey is sometimes quiet during autumn , and it is snowy in april .\n",
            "small_vocab_fr Line 1:  new jersey est parfois calme pendant l' automne , et il est neigeux en avril .\n",
            "small_vocab_en Line 2:  the united states is usually chilly during july , and it is usually freezing in november .\n",
            "small_vocab_fr Line 2:  les états-unis est généralement froid en juillet , et il gèle habituellement en novembre .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d14_e9WasmkU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 182
        },
        "outputId": "b51e90b6-cf2b-4c2e-d70c-1c9a57263bd8"
      },
      "source": [
        "english_words_counter = collections.Counter([word for sentence in english_sentences for word in sentence.split()])\n",
        "french_words_counter = collections.Counter([word for sentence in french_sentences for word in sentence.split()])\n",
        "\n",
        "print('{} English words.'.format(len([word for sentence in english_sentences for word in sentence.split()])))\n",
        "print('{} unique English words.'.format(len(english_words_counter)))\n",
        "print('10 Most common words in the English dataset:')\n",
        "print('\"' + '\" \"'.join(list(zip(*english_words_counter.most_common(10)))[0]) + '\"')\n",
        "print()\n",
        "print('{} French words.'.format(len([word for sentence in french_sentences for word in sentence.split()])))\n",
        "print('{} unique French words.'.format(len(french_words_counter)))\n",
        "print('10 Most common words in the French dataset:')\n",
        "print('\"' + '\" \"'.join(list(zip(*french_words_counter.most_common(10)))[0]) + '\"')"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1823250 English words.\n",
            "227 unique English words.\n",
            "10 Most common words in the English dataset:\n",
            "\"is\" \",\" \".\" \"in\" \"it\" \"during\" \"the\" \"but\" \"and\" \"sometimes\"\n",
            "\n",
            "1961295 French words.\n",
            "355 unique French words.\n",
            "10 Most common words in the French dataset:\n",
            "\"est\" \".\" \",\" \"en\" \"il\" \"les\" \"mais\" \"et\" \"la\" \"parfois\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "basGT0rpsvfV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 239
        },
        "outputId": "2a509ff0-a025-4661-f8c5-dd0ca7d0ae55"
      },
      "source": [
        "def tokenize(x):\n",
        "    \"\"\"\n",
        "    Tokenize x\n",
        "    :param x: List of sentences/strings to be tokenized\n",
        "    :return: Tuple of (tokenized x data, tokenizer used to tokenize x)\n",
        "    \"\"\"\n",
        "    # TODO: Implement\n",
        "    \n",
        "    x_tk = Tokenizer()\n",
        "    x_tk.fit_on_texts(x)\n",
        "    return x_tk.texts_to_sequences(x), x_tk\n",
        "\n",
        "\n",
        "# Tokenize Example output\n",
        "text_sentences = [\n",
        "    'The quick brown fox jumps over the lazy dog .',\n",
        "    'By Jove , my quick study of lexicography won a prize .',\n",
        "    'This is a short sentence .']\n",
        "text_tokenized, text_tokenizer = tokenize(text_sentences)\n",
        "print(text_tokenizer.word_index)\n",
        "print()\n",
        "for sample_i, (sent, token_sent) in enumerate(zip(text_sentences, text_tokenized)):\n",
        "    print('Sequence {} in x'.format(sample_i + 1))\n",
        "    print('  Input:  {}'.format(sent))\n",
        "    print('  Output: {}'.format(token_sent))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'the': 1, 'quick': 2, 'a': 3, 'brown': 4, 'fox': 5, 'jumps': 6, 'over': 7, 'lazy': 8, 'dog': 9, 'by': 10, 'jove': 11, 'my': 12, 'study': 13, 'of': 14, 'lexicography': 15, 'won': 16, 'prize': 17, 'this': 18, 'is': 19, 'short': 20, 'sentence': 21}\n",
            "\n",
            "Sequence 1 in x\n",
            "  Input:  The quick brown fox jumps over the lazy dog .\n",
            "  Output: [1, 2, 4, 5, 6, 7, 1, 8, 9]\n",
            "Sequence 2 in x\n",
            "  Input:  By Jove , my quick study of lexicography won a prize .\n",
            "  Output: [10, 11, 12, 2, 13, 14, 15, 16, 3, 17]\n",
            "Sequence 3 in x\n",
            "  Input:  This is a short sentence .\n",
            "  Output: [18, 19, 3, 20, 21]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5wwHzfeVuA09",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 182
        },
        "outputId": "e498df5d-f2e2-4740-fd09-8479506f40c7"
      },
      "source": [
        "def pad(x, length=None):\n",
        "    \"\"\"\n",
        "    Pad x\n",
        "    :param x: List of sequences.\n",
        "    :param length: Length to pad the sequence to.  If None, use length of longest sequence in x.\n",
        "    :return: Padded numpy array of sequences\n",
        "    \"\"\"\n",
        "    # TODO: Implement\n",
        "    if length is None:\n",
        "        # Find the length of the longest sequence/sentence\n",
        "        length = max([len(seq) for seq in x])\n",
        "    \n",
        "    return pad_sequences(sequences=x, maxlen=length, padding='post')\n",
        "\n",
        "\n",
        "# Pad Tokenized output\n",
        "test_pad = pad(text_tokenized)\n",
        "for sample_i, (token_sent, pad_sent) in enumerate(zip(text_tokenized, test_pad)):\n",
        "    print('Sequence {} in x'.format(sample_i + 1))\n",
        "    print('  Input:  {}'.format(np.array(token_sent)))\n",
        "    print('  Output: {}'.format(pad_sent))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sequence 1 in x\n",
            "  Input:  [1 2 4 5 6 7 1 8 9]\n",
            "  Output: [1 2 4 5 6 7 1 8 9 0]\n",
            "Sequence 2 in x\n",
            "  Input:  [10 11 12  2 13 14 15 16  3 17]\n",
            "  Output: [10 11 12  2 13 14 15 16  3 17]\n",
            "Sequence 3 in x\n",
            "  Input:  [18 19  3 20 21]\n",
            "  Output: [18 19  3 20 21  0  0  0  0  0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "30UwVrSIuEVn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        },
        "outputId": "8e323ba8-aa6f-452a-ecff-572c97fa4bf6"
      },
      "source": [
        "def preprocess(x, y):\n",
        "    \"\"\"\n",
        "    Preprocess x and y\n",
        "    :param x: Feature List of sentences\n",
        "    :param y: Label List of sentences\n",
        "    :return: Tuple of (Preprocessed x, Preprocessed y, x tokenizer, y tokenizer)\n",
        "    \"\"\"\n",
        "    preprocess_x, x_tk = tokenize(x)\n",
        "    preprocess_y, y_tk = tokenize(y)\n",
        "\n",
        "    preprocess_x = pad(preprocess_x)\n",
        "    preprocess_y = pad(preprocess_y)\n",
        "\n",
        "    # Keras's sparse_categorical_crossentropy function requires the labels to be in 3 dimensions\n",
        "    preprocess_y = preprocess_y.reshape(*preprocess_y.shape, 1)\n",
        "\n",
        "    return preprocess_x, preprocess_y, x_tk, y_tk\n",
        "\n",
        "preproc_english_sentences, preproc_french_sentences, english_tokenizer, french_tokenizer =\\\n",
        "    preprocess(english_sentences, french_sentences)\n",
        "    \n",
        "max_english_sequence_length = preproc_english_sentences.shape[1]\n",
        "max_french_sequence_length = preproc_french_sentences.shape[1]\n",
        "english_vocab_size = len(english_tokenizer.word_index)\n",
        "french_vocab_size = len(french_tokenizer.word_index)\n",
        "\n",
        "print('Data Preprocessed')\n",
        "print(\"Max English sentence length:\", max_english_sequence_length)\n",
        "print(\"Max French sentence length:\", max_french_sequence_length)\n",
        "print(\"English vocabulary size:\", english_vocab_size)\n",
        "print(\"French vocabulary size:\", french_vocab_size)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Data Preprocessed\n",
            "Max English sentence length: 15\n",
            "Max French sentence length: 21\n",
            "English vocabulary size: 199\n",
            "French vocabulary size: 344\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eduY-QhLzHsH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "0897ec03-3360-4d23-dedd-6fbfa2bad95b"
      },
      "source": [
        "def logits_to_text(logits, tokenizer):\n",
        "    \"\"\"\n",
        "    Turn logits from a neural network into text using the tokenizer\n",
        "    :param logits: Logits from a neural network\n",
        "    :param tokenizer: Keras Tokenizer fit on the labels\n",
        "    :return: String that represents the text of the logits\n",
        "    \"\"\"\n",
        "    index_to_words = {id: word for word, id in tokenizer.word_index.items()}\n",
        "    index_to_words[0] = '<PAD>'\n",
        "\n",
        "    return ' '.join([index_to_words[prediction] for prediction in np.argmax(logits, 1)])\n",
        "\n",
        "print('`logits_to_text` function loaded.')\n",
        "\n",
        "# Add-on : Collect logs for Tensorboard\n",
        "from keras.callbacks import TensorBoard\n",
        "from time import time\n",
        "tensorboard = TensorBoard(log_dir=\"logs/{}\".format(time()), histogram_freq=1, write_graph=True)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "`logits_to_text` function loaded.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sDs5MuAyucaP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "902764fa-4dda-426e-9571-3611fe381682"
      },
      "source": [
        "def nmt_model(input_shape, output_sequence_length, english_vocab_size, french_vocab_size):\n",
        "    \"\"\"\n",
        "    Build and train a model that incorporates embedding, encoder-decoder, and bidirectional RNN on x and y\n",
        "    :param input_shape: Tuple of input shape\n",
        "    :param output_sequence_length: Length of output sequence\n",
        "    :param english_vocab_size: Number of unique English words in the dataset\n",
        "    :param french_vocab_size: Number of unique French words in the dataset\n",
        "    :return: Keras model built, but not trained\n",
        "    \"\"\"\n",
        "    \n",
        "    # TODO: Train the final model\n",
        "    #print(\"Debug input_shape =\" , input_shape, \"  input_shape[1:] =\", input_shape[1:], \"  Input length=\", input_shape[1:][0])\n",
        "    #print(\"Debug output_sequence_length =\" , output_sequence_length) \n",
        "    #print(\"Debug english_vocab_size =\" , english_vocab_size)\n",
        "    #print(\"Debug french_vocab_size =\" , french_vocab_size)\n",
        "    \n",
        "    \n",
        "    ## Model_final is a seq2seq model (encoder-decoder) using embedding and bidirectional LSTM\n",
        "    \n",
        "    # Hyperparameters\n",
        "    embedding_size = 256\n",
        "    rnn_cells = 300\n",
        "    dropout = 0.2\n",
        "    learning_rate = 1e-3\n",
        "    \n",
        "    from keras.layers import LSTM, concatenate\n",
        "    \n",
        "    # Input and embedding\n",
        "    encoder_input_seq = Input(shape=input_shape[1:]) \n",
        "    embedded_input_seq = Embedding(input_dim = english_vocab_size,\n",
        "                                   output_dim = embedding_size,\n",
        "                                   input_length=input_shape[1:][0])(encoder_input_seq)\n",
        "    \n",
        "    ## Note Bidirectional LSTM is used on the encoder side (see https://arxiv.org/pdf/1609.08144.pdf )\n",
        "    ## Alternate version : Encoder RNN Bidirectional layer (Using the Keras Bidirectional layer wrappers)\n",
        "    #encoder_output, forward_state_h, forward_state_c, backward_state_h, backward_state_c = Bidirectional(LSTM(units=rnn_cells,\n",
        "    #                                                                                                          dropout=dropout,\n",
        "    #                                                                                                          return_sequences=False,\n",
        "    #                                                                                                          return_state=True))(embedded_input_seq)\n",
        "    \n",
        "    # Encoder Forward RNN layer\n",
        "    encoder_forward_output, forward_state_h, forward_state_c = LSTM(units=rnn_cells,\n",
        "                                                                    dropout=dropout,\n",
        "                                                                    return_sequences=False,\n",
        "                                                                    return_state=True,\n",
        "                                                                    go_backwards=False)(embedded_input_seq)\n",
        "    \n",
        "    # Encoder backward RNN layer\n",
        "    encoder_backward_output, backward_state_h, backward_state_c = LSTM(units=rnn_cells,\n",
        "                                                                       dropout=dropout,\n",
        "                                                                       return_sequences=False,\n",
        "                                                                       return_state=True,\n",
        "                                                                       go_backwards=True)(embedded_input_seq)\n",
        "    \n",
        "    # Encoder output and states : Merge the LSTM Forward and Backward ouputs (using 'concatenate' method) \n",
        "    state_h = concatenate([forward_state_h, backward_state_h]) \n",
        "    state_c = concatenate([forward_state_c, backward_state_c])    \n",
        "    encoder_output = concatenate([encoder_forward_output, encoder_backward_output])        \n",
        "         \n",
        "    # Decoder Input   \n",
        "    decoder_input_seq = RepeatVector(output_sequence_length)(encoder_output)\n",
        "    \n",
        "    # Decoder RNN layer\n",
        "    # Note : we need twice more LSTM cells as we have concatenated backward and forward LSTM layers in encoder\n",
        "    decoder_output = LSTM(units=rnn_cells*2,\n",
        "                                  dropout=dropout,\n",
        "                                  return_sequences=True,\n",
        "                                  return_state=False,\n",
        "                                  go_backwards=False)(decoder_input_seq, initial_state=[state_h, state_c])\n",
        "    \n",
        "    # Decoder output\n",
        "    logits = TimeDistributed(Dense(units=french_vocab_size))(decoder_output) \n",
        "    \n",
        "    # Model\n",
        "    model = Model(encoder_input_seq, Activation('softmax')(logits))\n",
        "    model.compile(loss=sparse_categorical_crossentropy,\n",
        "                  optimizer=Adam(lr=learning_rate),\n",
        "                  metrics=['accuracy'])\n",
        "    \n",
        "    return model    \n",
        "       \n",
        "\n",
        "# Train and Print prediction(s)\n",
        "\n",
        "# Pad the input to work with the Embedding layer\n",
        "tmp_x = pad(preproc_english_sentences, max_french_sequence_length)\n",
        "#print(\"Debug tmp_x shape=\", tmp_x.shape )\n",
        "\n",
        "\n",
        "# Train the neural network \n",
        "translation_model = nmt_model(input_shape = tmp_x.shape,\n",
        "                              output_sequence_length = max_french_sequence_length,\n",
        "                              english_vocab_size = english_vocab_size+1,\n",
        "                              french_vocab_size = french_vocab_size+1)\n",
        "\n",
        "print(translation_model.summary(line_length=125))\n",
        "\n",
        "\n",
        "translation_model.fit(tmp_x, preproc_french_sentences, batch_size=1024, epochs=15, validation_split=0.2)\n",
        "\n",
        "# Print prediction(s)\n",
        "print(logits_to_text(nmt_model.predict(tmp_x[:1])[0:10], french_tokenizer))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "_____________________________________________________________________________________________________________________________\n",
            "Layer (type)                             Output Shape               Param #        Connected to                              \n",
            "=============================================================================================================================\n",
            "input_1 (InputLayer)                     (None, 21)                 0                                                        \n",
            "_____________________________________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)                  (None, 21, 256)            51200          input_1[0][0]                             \n",
            "_____________________________________________________________________________________________________________________________\n",
            "lstm_1 (LSTM)                            [(None, 300), (None, 300), 668400         embedding_1[0][0]                         \n",
            "_____________________________________________________________________________________________________________________________\n",
            "lstm_2 (LSTM)                            [(None, 300), (None, 300), 668400         embedding_1[0][0]                         \n",
            "_____________________________________________________________________________________________________________________________\n",
            "concatenate_3 (Concatenate)              (None, 600)                0              lstm_1[0][0]                              \n",
            "                                                                                   lstm_2[0][0]                              \n",
            "_____________________________________________________________________________________________________________________________\n",
            "repeat_vector_1 (RepeatVector)           (None, 21, 600)            0              concatenate_3[0][0]                       \n",
            "_____________________________________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)              (None, 600)                0              lstm_1[0][1]                              \n",
            "                                                                                   lstm_2[0][1]                              \n",
            "_____________________________________________________________________________________________________________________________\n",
            "concatenate_2 (Concatenate)              (None, 600)                0              lstm_1[0][2]                              \n",
            "                                                                                   lstm_2[0][2]                              \n",
            "_____________________________________________________________________________________________________________________________\n",
            "lstm_3 (LSTM)                            (None, 21, 600)            2882400        repeat_vector_1[0][0]                     \n",
            "                                                                                   concatenate_1[0][0]                       \n",
            "                                                                                   concatenate_2[0][0]                       \n",
            "_____________________________________________________________________________________________________________________________\n",
            "time_distributed_1 (TimeDistributed)     (None, 21, 345)            207345         lstm_3[0][0]                              \n",
            "_____________________________________________________________________________________________________________________________\n",
            "activation_1 (Activation)                (None, 21, 345)            0              time_distributed_1[0][0]                  \n",
            "=============================================================================================================================\n",
            "Total params: 4,477,745\n",
            "Trainable params: 4,477,745\n",
            "Non-trainable params: 0\n",
            "_____________________________________________________________________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train on 110288 samples, validate on 27573 samples\n",
            "Epoch 1/15\n",
            "110288/110288 [==============================] - 80s 723us/step - loss: 2.4890 - accuracy: 0.4875 - val_loss: 1.8058 - val_accuracy: 0.5365\n",
            "Epoch 2/15\n",
            "110288/110288 [==============================] - 78s 710us/step - loss: 1.4530 - accuracy: 0.6129 - val_loss: 1.2571 - val_accuracy: 0.6610\n",
            "Epoch 3/15\n",
            "110288/110288 [==============================] - 79s 712us/step - loss: 1.1502 - accuracy: 0.6825 - val_loss: 0.9862 - val_accuracy: 0.7194\n",
            "Epoch 4/15\n",
            "110288/110288 [==============================] - 78s 708us/step - loss: 0.9292 - accuracy: 0.7306 - val_loss: 0.8264 - val_accuracy: 0.7565\n",
            "Epoch 5/15\n",
            "110288/110288 [==============================] - 78s 710us/step - loss: 0.7613 - accuracy: 0.7732 - val_loss: 0.6587 - val_accuracy: 0.8016\n",
            "Epoch 6/15\n",
            "110288/110288 [==============================] - 78s 711us/step - loss: 0.6302 - accuracy: 0.8087 - val_loss: 0.5404 - val_accuracy: 0.8355\n",
            "Epoch 7/15\n",
            "110288/110288 [==============================] - 78s 710us/step - loss: 0.5070 - accuracy: 0.8458 - val_loss: 0.4139 - val_accuracy: 0.8782\n",
            "Epoch 8/15\n",
            "110288/110288 [==============================] - 78s 712us/step - loss: 0.3967 - accuracy: 0.8810 - val_loss: 0.3175 - val_accuracy: 0.9084\n",
            "Epoch 9/15\n",
            "110288/110288 [==============================] - 79s 714us/step - loss: 0.3141 - accuracy: 0.9066 - val_loss: 0.2448 - val_accuracy: 0.9310\n",
            "Epoch 10/15\n",
            "110288/110288 [==============================] - 79s 713us/step - loss: 0.2531 - accuracy: 0.9247 - val_loss: 0.2095 - val_accuracy: 0.9394\n",
            "Epoch 11/15\n",
            "110288/110288 [==============================] - 79s 714us/step - loss: 0.2106 - accuracy: 0.9368 - val_loss: 0.1786 - val_accuracy: 0.9465\n",
            "Epoch 12/15\n",
            "110288/110288 [==============================] - 79s 713us/step - loss: 0.1830 - accuracy: 0.9442 - val_loss: 0.1523 - val_accuracy: 0.9533\n",
            "Epoch 13/15\n",
            "110288/110288 [==============================] - 79s 715us/step - loss: 0.1605 - accuracy: 0.9505 - val_loss: 0.1350 - val_accuracy: 0.9590\n",
            "Epoch 14/15\n",
            "110288/110288 [==============================] - 79s 713us/step - loss: 0.1402 - accuracy: 0.9566 - val_loss: 0.1423 - val_accuracy: 0.9585\n",
            "Epoch 15/15\n",
            "110288/110288 [==============================] - 78s 712us/step - loss: 0.1322 - accuracy: 0.9589 - val_loss: 0.1126 - val_accuracy: 0.9655\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-14-68ee82d8c989>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;31m# Print prediction(s)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 102\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits_to_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnmt_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtmp_x\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfrench_tokenizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m: 'function' object has no attribute 'predict'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "06PVtrMWzkXY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 937
        },
        "outputId": "4cda6173-a900-4a83-9fe4-ae3b9ec7af77"
      },
      "source": [
        "def final_predictions(x, y, x_tk, y_tk):\n",
        "    \"\"\"\n",
        "    Gets predictions using the final model\n",
        "    :param x: Preprocessed English data\n",
        "    :param y: Preprocessed French data\n",
        "    :param x_tk: English tokenizer\n",
        "    :param y_tk: French tokenizer\n",
        "    \"\"\"\n",
        "    # TODO: Train neural network using model_final\n",
        "    model = nmt_model(input_shape = x.shape,\n",
        "                        output_sequence_length = y.shape[1],\n",
        "                        english_vocab_size = len(x_tk.word_index)+1,\n",
        "                        french_vocab_size = len(y_tk.word_index)+1)\n",
        "\n",
        "    model.fit(x, y, batch_size=1024, epochs=20, validation_split=0.2)\n",
        "    \n",
        "    ## DON'T EDIT ANYTHING BELOW THIS LINE\n",
        "    y_id_to_word = {value: key for key, value in y_tk.word_index.items()}\n",
        "    y_id_to_word[0] = '<PAD>'\n",
        "\n",
        "    sentence = 'he saw a old yellow truck'\n",
        "    sentence = [x_tk.word_index[word] for word in sentence.split()]\n",
        "    sentence = pad_sequences([sentence], maxlen=x.shape[-1], padding='post')\n",
        "    sentences = np.array([sentence[0], x[0]])\n",
        "    predictions = model.predict(sentences, len(sentences))\n",
        "\n",
        "    print('Sample 1:')\n",
        "    print(' '.join([y_id_to_word[np.argmax(x)] for x in predictions[0]]))\n",
        "    print('Il a vu un vieux camion jaune')\n",
        "    print('Sample 2:')\n",
        "    print(' '.join([y_id_to_word[np.argmax(x)] for x in predictions[1]]))\n",
        "    print(' '.join([y_id_to_word[np.max(x)] for x in y[0]]))\n",
        "\n",
        "final_predictions(preproc_english_sentences, preproc_french_sentences, english_tokenizer, french_tokenizer)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train on 110288 samples, validate on 27573 samples\n",
            "Epoch 1/20\n",
            "110288/110288 [==============================] - 71s 643us/step - loss: 2.6654 - accuracy: 0.4698 - val_loss: 1.7557 - val_accuracy: 0.5616\n",
            "Epoch 2/20\n",
            "110288/110288 [==============================] - 70s 633us/step - loss: 1.4816 - accuracy: 0.6097 - val_loss: 1.2382 - val_accuracy: 0.6601\n",
            "Epoch 3/20\n",
            "110288/110288 [==============================] - 70s 635us/step - loss: 1.1345 - accuracy: 0.6840 - val_loss: 0.9978 - val_accuracy: 0.7166\n",
            "Epoch 4/20\n",
            "110288/110288 [==============================] - 70s 635us/step - loss: 0.9449 - accuracy: 0.7262 - val_loss: 0.8483 - val_accuracy: 0.7499\n",
            "Epoch 5/20\n",
            "110288/110288 [==============================] - 70s 635us/step - loss: 0.7938 - accuracy: 0.7636 - val_loss: 0.6899 - val_accuracy: 0.7928\n",
            "Epoch 6/20\n",
            "110288/110288 [==============================] - 70s 634us/step - loss: 0.6535 - accuracy: 0.8025 - val_loss: 0.5652 - val_accuracy: 0.8290\n",
            "Epoch 7/20\n",
            "110288/110288 [==============================] - 70s 635us/step - loss: 0.5451 - accuracy: 0.8346 - val_loss: 0.4727 - val_accuracy: 0.8601\n",
            "Epoch 8/20\n",
            "110288/110288 [==============================] - 70s 637us/step - loss: 0.4446 - accuracy: 0.8657 - val_loss: 0.3726 - val_accuracy: 0.8909\n",
            "Epoch 9/20\n",
            "110288/110288 [==============================] - 70s 636us/step - loss: 0.3506 - accuracy: 0.8951 - val_loss: 0.2848 - val_accuracy: 0.9184\n",
            "Epoch 10/20\n",
            "110288/110288 [==============================] - 70s 633us/step - loss: 0.2747 - accuracy: 0.9193 - val_loss: 0.2293 - val_accuracy: 0.9338\n",
            "Epoch 11/20\n",
            "110288/110288 [==============================] - 70s 631us/step - loss: 0.2275 - accuracy: 0.9329 - val_loss: 0.1949 - val_accuracy: 0.9435\n",
            "Epoch 12/20\n",
            "110288/110288 [==============================] - 70s 632us/step - loss: 0.1907 - accuracy: 0.9430 - val_loss: 0.1602 - val_accuracy: 0.9521\n",
            "Epoch 13/20\n",
            "110288/110288 [==============================] - 70s 631us/step - loss: 0.1637 - accuracy: 0.9505 - val_loss: 0.1405 - val_accuracy: 0.9575\n",
            "Epoch 14/20\n",
            "110288/110288 [==============================] - 70s 632us/step - loss: 0.1416 - accuracy: 0.9569 - val_loss: 0.1256 - val_accuracy: 0.9614\n",
            "Epoch 15/20\n",
            "110288/110288 [==============================] - 70s 632us/step - loss: 0.1289 - accuracy: 0.9606 - val_loss: 0.1183 - val_accuracy: 0.9637\n",
            "Epoch 16/20\n",
            "110288/110288 [==============================] - 70s 632us/step - loss: 0.1150 - accuracy: 0.9647 - val_loss: 0.1086 - val_accuracy: 0.9665\n",
            "Epoch 17/20\n",
            "110288/110288 [==============================] - 70s 631us/step - loss: 0.1069 - accuracy: 0.9670 - val_loss: 0.0983 - val_accuracy: 0.9694\n",
            "Epoch 18/20\n",
            "110288/110288 [==============================] - 70s 630us/step - loss: 0.0968 - accuracy: 0.9700 - val_loss: 0.0901 - val_accuracy: 0.9725\n",
            "Epoch 19/20\n",
            "110288/110288 [==============================] - 70s 631us/step - loss: 0.0893 - accuracy: 0.9724 - val_loss: 0.0957 - val_accuracy: 0.9710\n",
            "Epoch 20/20\n",
            "110288/110288 [==============================] - 70s 631us/step - loss: 0.0849 - accuracy: 0.9737 - val_loss: 0.0859 - val_accuracy: 0.9733\n",
            "Sample 1:\n",
            "il a vu un vieux camion jaune <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD>\n",
            "Il a vu un vieux camion jaune\n",
            "Sample 2:\n",
            "new jersey est parfois calme pendant l' automne et il est neigeux en avril <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD>\n",
            "new jersey est parfois calme pendant l' automne et il est neigeux en avril <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fJO580wa5jU_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 172
        },
        "outputId": "d3cb753c-b274-462a-945a-30b04d152422"
      },
      "source": [
        "print(logits_to_text(nmt_model.predict(tmp_x[5:10])[0], french_tokenizer))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-16-6427e8a5d1ef>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits_to_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnmt_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtmp_x\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfrench_tokenizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m: 'function' object has no attribute 'predict'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LZG7UcO27ar6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}