machine_translation_encoder_decoder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import unicode_literals, print_function, division\n",
    "from io import open\n",
    "import unicodedata\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "import numpy as np\n",
    "import pickle\n",
    "import time\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "import pdb\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.switch_backend('agg')\n",
    "%matplotlib inline\n",
    "\n",
    "from model_architectures import Encoder_RNN, Decoder_RNN\n",
    "from data_prep import prepareData, tensorsFromPair\n",
    "from inference import generate_translation\n",
    "from misc import timeSince, load_cpickle_gc\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!export CUDA_LAUNCH_BLOCKING=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "SOS_token = 0\n",
    "EOS_token = 1\n",
    "MAX_LENGTH = 30\n",
    "teacher_forcing_ratio = 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data prep - train and val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_lang, target_lang, train_pairs = prepareData(\n",
    "    input_file = 'iwslt-vi-en-processed/train.vi', \n",
    "    target_file = 'iwslt-vi-en-processed/train.en', \n",
    "    input_lang = 'vi', \n",
    "    target_lang = 'en')\n",
    "\n",
    "pickle.dump(train_pairs, open(\"train_vi_en_pairs\", \"wb\"))\n",
    "pickle.dump(input_lang, open(\"train_vi_lang\", \"wb\"))\n",
    "pickle.dump(target_lang, open(\"train_en_lang\", \"wb\"))\n",
    "\n",
    "_, _, val_pairs = prepareData(\n",
    "    input_file = 'iwslt-vi-en-processed/dev.vi', \n",
    "    target_file = 'iwslt-vi-en-processed/dev.en',\n",
    "    input_lang = 'vi', \n",
    "    target_lang = 'en')\n",
    "\n",
    "pickle.dump(val_pairs, open(\"val_vi_en_pairs\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_idx_pairs = []\n",
    "for x in train_pairs:\n",
    "    indexed = list(tensorsFromPair(x, input_lang, target_lang))\n",
    "    train_idx_pairs.append(indexed)\n",
    "\n",
    "pickle.dump(train_idx_pairs, open(\"train_vi_en_idx_pairs\", \"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Laboratory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "class LanguagePairDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, sent_pairs): \n",
    "        # this is a list of sentences \n",
    "        self.sent_pairs_list = sent_pairs\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.sent_pairs_list)\n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        \"\"\"\n",
    "        Triggered when you call dataset[i]\n",
    "        \"\"\"\n",
    "        sent1 = self.sent_pairs_list[key][0][:MAX_LENGTH]\n",
    "        sent2 = self.sent_pairs_list[key][1][:MAX_LENGTH]\n",
    "        return [sent1, sent2, len(sent1), len(sent2)]\n",
    "\n",
    "def language_pair_dataset_collate_function(batch):\n",
    "    \"\"\"\n",
    "    Customized function for DataLoader that dynamically pads the batch so that all \n",
    "    data have the same length\n",
    "    \"\"\"\n",
    "    sent1_list = []\n",
    "    sent1_length_list = []\n",
    "    sent2_list = []\n",
    "    sent2_length_list = []\n",
    "    # padding\n",
    "    for datum in batch:\n",
    "        padded_vec_1 = np.pad(np.array(datum[0]).T.squeeze(), pad_width=((0,MAX_LENGTH-len(datum[0]))), \n",
    "                                mode=\"constant\", constant_values=0)\n",
    "        padded_vec_2 = np.pad(np.array(datum[1]).T.squeeze(), pad_width=((0,MAX_LENGTH-len(datum[1]))), \n",
    "                                mode=\"constant\", constant_values=0)\n",
    "        sent1_list.append(padded_vec_1)\n",
    "        sent2_list.append(padded_vec_2)\n",
    "        sent1_length_list.append(len(datum[0]))\n",
    "        sent2_length_list.append(len(datum[1]))\n",
    "    print(np.array(sent1_list).shape)\n",
    "    return [torch.from_numpy(np.array(sent1_list)), torch.LongTensor(sent1_length_list), \n",
    "            torch.from_numpy(np.array(sent2_list)), torch.LongTensor(sent2_length_list)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_idx_pairs = load_cpickle_gc(\"train_vi_en_idx_pairs\")\n",
    "train_dataset = LanguagePairDataset(train_idx_pairs)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n",
    "                                           batch_size=BATCH_SIZE, \n",
    "                                           collate_fn=language_pair_dataset_collate_function,\n",
    "                                           #shuffle=True\n",
    "                                          )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, (sent1_list, sent1_lengths, sent2_list, sent2_lengths) in enumerate(train_loader):\n",
    "    print(i)\n",
    "    print(sent1_list.size())\n",
    "    print(sent2_list.size())\n",
    "    \n",
    "    print(sent1_lengths.size())\n",
    "    print(sent2_lengths.size())\n",
    "    break\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder_Batch_RNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        super(Encoder_Batch_RNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size, batch_first=True)\n",
    "        \n",
    "    def init_hidden(self, batch_size):\n",
    "        return torch.zeros(1, batch_size, self.hidden_size, device=device)\n",
    "\n",
    "    def forward(self, sents, sent_lengths):\n",
    "        '''\n",
    "            sents is a tensor with the shape (batch_size, padded_length )\n",
    "        '''\n",
    "        batch_size = sents.size()[0]\n",
    "        sent_lengths = list(sent_lengths)\n",
    "        \n",
    "        descending_lengths = [x for x, _ in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)]\n",
    "        descending_indices = [x for _, x in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)]\n",
    "        descending_lengths = np.array(descending_lengths)\n",
    "        \n",
    "        descending_sents = torch.index_select(sents, 0, torch.tensor(descending_indices).to(device))\n",
    "        \n",
    "        # get embedding\n",
    "        embed = self.embedding(descending_sents)\n",
    "        # pack padded sequence\n",
    "        embed = torch.nn.utils.rnn.pack_padded_sequence(embed, descending_lengths, batch_first=True)\n",
    "        \n",
    "        # fprop though RNN\n",
    "        self.hidden = self.init_hidden(batch_size)\n",
    "        rnn_out, self.hidden = self.gru(embed, self.hidden)\n",
    "        \n",
    "        # change the order back\n",
    "        change_it_back = [x for _, x in sorted(zip(descending_indices, range(len(descending_indices))))]\n",
    "        self.hidden = torch.index_select(self.hidden, 1, torch.LongTensor(change_it_back).to(device)) \n",
    "        \n",
    "        # **TODO**: What is rnn_out?\n",
    "        return rnn_out, self.hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder_Batch_RNN(nn.Module):\n",
    "    def __init__(self, output_size, hidden_size):\n",
    "        super(Decoder_Batch_RNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "\n",
    "        self.embedding = nn.Embedding(output_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size, batch_first=True)\n",
    "        self.out = nn.Linear(hidden_size, output_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=1)\n",
    "        \n",
    "    def init_hidden(self, batch_size):\n",
    "        return torch.zeros(1, batch_size, self.hidden_size, device=device)\n",
    "\n",
    "    def forward(self, sents, sent_lengths, hidden):\n",
    "        \n",
    "        batch_size = sents.size()[0]\n",
    "        sent_lengths = list(sent_lengths)\n",
    "        \n",
    "        descending_lengths = [x for x, _ in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)]\n",
    "        descending_indices = [x for _, x in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)]\n",
    "        descending_lengths = np.array(descending_lengths)\n",
    "        \n",
    "        descending_sents = torch.index_select(sents, 0, torch.tensor(descending_indices).to(device))\n",
    "        \n",
    "        # get embedding\n",
    "        embed = self.embedding(descending_sents)\n",
    "        # pack padded sequence\n",
    "        embed = torch.nn.utils.rnn.pack_padded_sequence(embed, descending_lengths, batch_first=True)\n",
    "        \n",
    "        # fprop though RNN\n",
    "        self.hidden = hidden\n",
    "        rnn_out, self.hidden = self.gru(embed, self.hidden)\n",
    "        \n",
    "        change_it_back = [x for _, x in sorted(zip(descending_indices, range(len(descending_indices))))]\n",
    "        self.hidden = torch.index_select(self.hidden, 1, torch.LongTensor(change_it_back).to(device))\n",
    "        rnn_out, _ = torch.nn.utils.rnn.pad_packed_sequence(rnn_out, batch_first=True)\n",
    "        # rnn_out is batch_size x 28 x 256\n",
    "                \n",
    "        final_hidden = self.hidden\n",
    "        final_hidden = final_hidden.view(final_hidden.size(1), final_hidden.size(0), -1)\n",
    "        first_hidden = hidden\n",
    "        first_hidden = first_hidden.view(first_hidden.size(1), first_hidden.size(0), -1)\n",
    "        \n",
    "        rnn_out = torch.cat((first_hidden, rnn_out, final_hidden), 1)\n",
    "        \n",
    "        \n",
    "#         rnn_out = rnn_out.view(-1, rnn_out.size(2))\n",
    "        \n",
    "        output = self.softmax(self.out(rnn_out))\n",
    "        # now output is the size 28 by 31257 (vocab size)\n",
    "        return output, hidden"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=MAX_LENGTH):\n",
    "    encoder_hidden = encoder.initHidden()\n",
    "    encoder_optimizer.zero_grad()\n",
    "    decoder_optimizer.zero_grad()\n",
    "\n",
    "    # example of input_tensor: [2, 43, 23, 9, 19, 4]. Indexed on our vocabulary. \n",
    "    input_length = input_tensor.size(0)\n",
    "    target_length = target_tensor.size(0)\n",
    "\n",
    "    encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
    "    \n",
    "     # iterate GRU over words --> final hidden state is representation of source sentence. \n",
    "    loss = 0\n",
    "    for ei in range(input_length):\n",
    "        encoder_output, encoder_hidden = encoder(\n",
    "            input_tensor[ei], encoder_hidden)\n",
    "        encoder_outputs[ei] = encoder_output[0, 0]\n",
    "\n",
    "    decoder_input = torch.tensor([[SOS_token]], device=device)\n",
    "    decoder_hidden = encoder_hidden\n",
    "    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n",
    "\n",
    "    if use_teacher_forcing:\n",
    "        # Teacher forcing: Feed the target as the next input\n",
    "        for di in range(target_length):\n",
    "            decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden)\n",
    "            loss += criterion(decoder_output, target_tensor[di])\n",
    "            decoder_input = target_tensor[di]  # Teacher forcing\n",
    "\n",
    "    else:\n",
    "        # Without teacher forcing: use its own predictions as the next input\n",
    "        for di in range(target_length):\n",
    "            decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden)\n",
    "            topv, topi = decoder_output.topk(1)\n",
    "            decoder_input = topi.squeeze().detach()  # detach from history as input\n",
    "\n",
    "            loss += criterion(decoder_output, target_tensor[di])\n",
    "            if decoder_input.item() == EOS_token:\n",
    "                break\n",
    "\n",
    "    loss.backward()\n",
    "    encoder_optimizer.step()\n",
    "    decoder_optimizer.step()\n",
    "\n",
    "    return loss.item() / target_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = torch.randn(3, 4)\n",
    "a[0][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The whole training process looks like this:\n",
    "\n",
    "-  Start a timer\n",
    "-  Initialize optimizers and criterion\n",
    "-  Create set of training pairs\n",
    "-  Start empty losses array for plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainIters(encoder, decoder, n_iters, n_epochs, lang1, lang2, print_every=1000, plot_every=100, learning_rate=0.001):\n",
    "    \"\"\"\n",
    "    lang1 is the Lang object for language 1 \n",
    "    Lang2 is the Lang object for language 2\n",
    "    \"\"\"\n",
    "    pairs = load_cpickle_gc(\"train_\"+lang1.name+\"_\"+lang2.name+\"_pairs\")\n",
    "    validation_pairs = load_cpickle_gc(\"val_\"+lang1.name+\"_\"+lang2.name+\"_pairs\")\n",
    "    start = time.time()\n",
    "    plot_losses = []\n",
    "    print_loss_total = 0  # Reset every print_every\n",
    "    plot_loss_total = 0  # Reset every plot_every\n",
    "    encoder_optimizer = torch.optim.SGD(encoder.parameters(), lr=learning_rate)\n",
    "    decoder_optimizer = torch.optim.SGD(decoder.parameters(), lr=learning_rate)\n",
    "    \n",
    "    criterion = nn.NLLLoss()\n",
    "    \n",
    "    for epoch in range(n_epochs):\n",
    "        for step, (sent1s, sent1_lengths, sent2s, sent2_lengths) in enumerate(train_loader):\n",
    "            encoder.train()\n",
    "            sent1_batch, sent2_batch = sent1s.to(device), sent2s.to(device) \n",
    "            sent1_length_batch, sent2_length_batch = sent1_lengths.to(device), sent2_lengths.to(device)\n",
    "            \n",
    "            encoder_optimizer.zero_grad()\n",
    "            outputs, encoder_hidden = encoder(sent1_batch, sent1_length_batch)\n",
    "            \n",
    "            encoder_hidden_batch = encoder_hidden\n",
    "            decoder_hidden = encoder_hidden_batch\n",
    "            \n",
    "            decoder_input = torch.tensor([[SOS_token]], device=device)\n",
    "            use_teacher_forcing = True\n",
    "            \n",
    "            loss = 0\n",
    "            outputs, decoder_hidden = decoder(sent2_batch, sent2_length_batch, decoder_hidden)\n",
    "            count = 0\n",
    "            for i in range(len(sent2_batch)):\n",
    "                l = sent2_length_batch[i]\n",
    "                for j in range(l):\n",
    "                    o = outputs[i][j].view(1, -1)\n",
    "                    s = torch.tensor([sent2_batch[i][j]])\n",
    "                    loss += criterion(o, s)\n",
    "                    count += 1\n",
    "                    \n",
    "            \n",
    "            print_loss_total += loss\n",
    "            plot_loss_total += loss  \n",
    "            \n",
    "            if  (step+1) % print_every == 0:\n",
    "                print_loss_avg = print_loss_total / (count*print_every)\n",
    "                print_loss_total = 0\n",
    "                print('TRAIN SCORE %s (%d %d%%) %.4f' % (timeSince(start, step / n_epochs),\n",
    "                                             step, step / n_epochs * 100, print_loss_avg))\n",
    "#                 val_loss = test_model(encoder, decoder, search, validation_pairs, lang1, max_length=MAX_LENGTH)\n",
    "                # returns bleu score\n",
    "#                 print(\"VALIDATION BLEU SCORE: \"+str(val_loss))\n",
    "\n",
    "            if step % plot_every == 0:\n",
    "                plot_loss_avg = plot_loss_total / plot_every\n",
    "                plot_losses.append(plot_loss_avg)\n",
    "                plot_loss_total = 0\n",
    "\n",
    "    showPlot(plot_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "TRAIN SCORE 0m 6s (- -1m 56s) (9 225%) 3.1460\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "TRAIN SCORE 0m 13s (- -1m 49s) (19 475%) 3.5343\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n",
      "(32, 30)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-31-ee123ea862f0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m }\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mtrainIters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-30-a91e4ef5f880>\u001b[0m in \u001b[0;36mtrainIters\u001b[0;34m(encoder, decoder, n_iters, n_epochs, lang1, lang2, print_every, plot_every, learning_rate)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m             \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder_hidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdecoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent2_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msent2_length_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder_hidden\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m             \u001b[0mcount\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent2_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-87d4b3bc6e4d>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, sents, sent_lengths, hidden)\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;31m#         rnn_out = rnn_out.view(-1, rnn_out.size(2))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnn_out\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m         \u001b[0;31m# now output is the size 28 by 31257 (vocab size)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1026\u001b[0m     \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1027\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1028\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1029\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1030\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "hidden_size = 256\n",
    "input_lang = load_cpickle_gc(\"train_vi_lang\")\n",
    "target_lang = load_cpickle_gc(\"train_en_lang\")\n",
    "encoder1 = Encoder_Batch_RNN(input_lang.n_words, hidden_size).to(device)\n",
    "decoder1 = Decoder_Batch_RNN(target_lang.n_words, hidden_size).to(device)\n",
    "\n",
    "num_iters = 10000\n",
    "\n",
    "args = {\n",
    "    'n_iters': 10000,\n",
    "    'n_epochs': 4,\n",
    "    'learning_rate': 0.001,\n",
    "    'encoder': encoder1,\n",
    "    'decoder': decoder1,\n",
    "    'lang1': input_lang, \n",
    "    'lang2': target_lang,\n",
    "    'print_every': 10\n",
    "}\n",
    "\n",
    "trainIters(**args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(encoder1.state_dict(), \"encoder1_40000\")\n",
    "# torch.save(decoder1.state_dict(), \"decoder1_40000\")\n",
    "\n",
    "pairs = load_cpickle_gc(\"train_\"+input_lang.name+\"_\"+target_lang.name+\"_pairs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> cam on ong a toi ay hom nay .\n",
      "= thank you for coming here .\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Expected hidden size (1, 1, 256), got (1, 10, 256)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-36-bf7004a75ad2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mencoder1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mdecoder1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mevaluateRandomly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpairs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoder1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstrategy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'greedy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-25-e112f6aa4c1e>\u001b[0m in \u001b[0;36mevaluateRandomly\u001b[0;34m(pairs, encoder, decoder, n, strategy, k, max_length)\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'>'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpair\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'='\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpair\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0moutput_words\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgenerate_translation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpair\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msearch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstrategy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_length\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_length\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m         \u001b[0moutput_sentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_words\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'<'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_sentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-34-0c239af11d77>\u001b[0m in \u001b[0;36mgenerate_translation\u001b[0;34m(encoder, decoder, sentence, max_length, search, k)\u001b[0m\n\u001b[1;32m     77\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m         \u001b[0;31m# encode the source sentence\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 79\u001b[0;31m         \u001b[0mencoder_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoder_hidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mencoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_tensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0minput_length\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     80\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m         \u001b[0;31m# start decoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-10800d8e396f>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, sents, sent_lengths)\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0;31m# fprop though RNN\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m         \u001b[0mrnn_out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgru\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membed\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0;31m# change the order back\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m    176\u001b[0m             \u001b[0mflat_weight\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_forward_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    179\u001b[0m         func = self._backend.RNN(\n\u001b[1;32m    180\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mcheck_forward_args\u001b[0;34m(self, input, hidden, batch_sizes)\u001b[0m\n\u001b[1;32m    149\u001b[0m                               'Expected hidden[1] size {}, got {}')\n\u001b[1;32m    150\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m             \u001b[0mcheck_hidden_size\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexpected_hidden_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    152\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mcheck_hidden_size\u001b[0;34m(hx, expected_hidden_size, msg)\u001b[0m\n\u001b[1;32m    141\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcheck_hidden_size\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexpected_hidden_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Expected hidden size {}, got {}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    142\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mexpected_hidden_size\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 143\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexpected_hidden_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    145\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'LSTM'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Expected hidden size (1, 1, 256), got (1, 10, 256)"
     ]
    }
   ],
   "source": [
    "# hidden_size = 256\n",
    "# encoder1 = EncoderRNN(input_lang.n_words, hidden_size).to(device)\n",
    "# encoder1.load_state_dict(torch.load(\"encoder1_40000\", map_location='cpu'))\n",
    "# decoder1 = DecoderRNN(target_lang.n_words, hidden_size).to(device)\n",
    "# decoder1.load_state_dict(torch.load(\"decoder1_40000\", map_location='cpu'))\n",
    "encoder1.eval()\n",
    "decoder1.eval()\n",
    "evaluateRandomly(pairs, encoder1, decoder1, n = 1, strategy='greedy', k = 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluateRandomly(pairs, encoder, decoder, n=10, strategy=\"greedy\", k = None, max_length = MAX_LENGTH):\n",
    "    \"\"\"\n",
    "    Randomly select a sentence from the input dataset and try to produce its translation.\n",
    "    \"\"\"    \n",
    "    for i in range(n):\n",
    "        pair = random.choice(pairs)\n",
    "        print('>', pair[0])\n",
    "        print('=', pair[1])\n",
    "        output_words = generate_translation(encoder, decoder, pair[0], search=strategy, k = k, max_length = max_length)\n",
    "        output_sentence = ' '.join(output_words)\n",
    "        print('<', output_sentence)\n",
    "        print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def indexesFromSentence(lang, sentence):\n",
    "    words = sentence.split(' ')\n",
    "    indices = []\n",
    "    for word in words:\n",
    "        if lang.word2index.get(word) is not None:\n",
    "            indices.append(lang.word2index[word])\n",
    "        else:\n",
    "            indices.append(1) # UNK_INDEX\n",
    "    return indices\n",
    "\n",
    "def tensorFromSentence(lang, sentence):\n",
    "    indexes = indexesFromSentence(lang, sentence)\n",
    "    indexes.append(EOS_token)\n",
    "    return torch.tensor(indexes, dtype=torch.long, device=device).view(-1, 1)\n",
    "\n",
    "def greedy_search(decoder, decoder_input, hidden, max_length):\n",
    "    translation = []\n",
    "    for i in range(max_length):\n",
    "        next_word_softmax, hidden = decoder(decoder_input, hidden)\n",
    "        best_idx = torch.max(next_word_softmax, 1)[1].squeeze().item()\n",
    "\n",
    "        # convert idx to word\n",
    "        best_word = target_lang.index2word[best_idx]\n",
    "        translation.append(best_word)\n",
    "        decoder_input = torch.tensor([[best_idx]], device=device)\n",
    "        \n",
    "        if best_word == 'EOS':\n",
    "            break\n",
    "    return translation\n",
    "\n",
    "\n",
    "def beam_search(decoder, decoder_input, hidden, max_length, k):\n",
    "    \n",
    "    candidates = [(decoder_input, 0, hidden)]\n",
    "    potential_candidates = []\n",
    "    completed_translations = []\n",
    "\n",
    "    # put a cap on the length of generated sentences\n",
    "    for m in range(max_length):\n",
    "        for c in candidates:\n",
    "            # unpack the tuple\n",
    "            c_sequence = c[0]\n",
    "            c_score = c[1]\n",
    "            c_hidden = c[2]\n",
    "            # EOS token\n",
    "            if c_sequence[-1] == 1:\n",
    "                completed_translations.append((c_sequence, c_score))\n",
    "                k = k - 1\n",
    "            else:\n",
    "                next_word_probs, hidden = decoder(c_sequence[-1], c_hidden)\n",
    "                # in the worst-case, one sequence will have the highest k probabilities\n",
    "                # so to save computation, only grab the k highest_probability from each candidate sequence\n",
    "                top_probs, top_idx = torch.topk(next_word_probs, k)\n",
    "                for i in range(len(top_probs[0])):\n",
    "                    word = torch.from_numpy(np.array([top_idx[0][i]]).reshape(1, 1)).to(device)\n",
    "                    new_score = c_score + top_probs[0][i]\n",
    "                    potential_candidates.append((torch.cat((c_sequence, word)).to(device), new_score, hidden))\n",
    "\n",
    "        candidates = sorted(potential_candidates, key= lambda x: x[1], reverse=True)[0:k] \n",
    "        potential_candidates = []\n",
    "\n",
    "    completed = completed_translations + candidates\n",
    "    completed = sorted(completed, key= lambda x: x[1], reverse=True)[0] \n",
    "    final_translation = []\n",
    "    for x in completed[0]:\n",
    "        final_translation.append(target_lang.index2word[x.squeeze().item()])\n",
    "    return final_translation\n",
    "\n",
    "def generate_translation(encoder, decoder, sentence, max_length, search=\"greedy\", k = None):\n",
    "    \"\"\" \n",
    "    @param max_length: the max # of words that the decoder can return\n",
    "    @returns decoded_words: a list of words in target language\n",
    "    \"\"\"    \n",
    "    with torch.no_grad():\n",
    "        input_tensor = tensorFromSentence(input_lang, sentence)\n",
    "        input_length = input_tensor.size()[0]\n",
    "        \n",
    "        # encode the source sentence\n",
    "        encoder_output, encoder_hidden = encoder(input_tensor, [input_length])\n",
    "\n",
    "        # start decoding\n",
    "        decoder_input = torch.tensor([[SOS_token]], device=device)  # SOS\n",
    "        decoder_hidden = encoder_hidden\n",
    "        decoded_words = []\n",
    "        \n",
    "        if search == 'greedy':\n",
    "            decoded_words = greedy_search(decoder, decoder_input, decoder_hidden, max_length)\n",
    "        elif search == 'beam':\n",
    "            if k == None:\n",
    "                k = 2\n",
    "            decoded_words = beam_search(decoder, decoder_input, decoder_hidden, max_length, k)  \n",
    "\n",
    "        return decoded_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(encoder, decoder, sentence, search=\"greedy\", max_length=MAX_LENGTH):\n",
    "    \"\"\"\n",
    "    Function that generate translation.\n",
    "    First, feed the source sentence into the encoder and obtain the hidden states from encoder.\n",
    "    Secondly, feed the hidden states into the decoder and unfold the outputs from the decoder.\n",
    "    Lastly, for each outputs from the decoder, collect the corresponding words in the target language's vocabulary.\n",
    "    And collect the attention for each output words.\n",
    "    @param encoder: the encoder network\n",
    "    @param decoder: the decoder network\n",
    "    @param sentence: string, a sentence in source language to be translated\n",
    "    @param max_length: the max # of words that the decoder can return\n",
    "    @output decoded_words: a list of words in target language\n",
    "    @output decoder_attentions: a list of vector, each of which sums up to 1.0\n",
    "    \"\"\"    \n",
    "    # process input sentence\n",
    "    with torch.no_grad():\n",
    "        input_tensor = tensorFromSentence(input_lang, sentence)\n",
    "        input_length = input_tensor.size()[0]\n",
    "        # encode the source lanugage\n",
    "        encoder_hidden = encoder.initHidden()\n",
    "\n",
    "        encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
    "\n",
    "        for ei in range(input_length):\n",
    "            encoder_output, encoder_hidden = encoder(input_tensor[ei],\n",
    "                                                     encoder_hidden)\n",
    "            encoder_outputs[ei] += encoder_output[0, 0]\n",
    "        decoder_input = torch.tensor([[SOS_token]], device=device)  # SOS\n",
    "        # decode the context vector\n",
    "        decoder_hidden = encoder_hidden # decoder starts from the last encoding sentence\n",
    "        # output of this function\n",
    "        decoder_attentions = torch.zeros(max_length, max_length)\n",
    "        \n",
    "        if search == 'greedy':\n",
    "            decoded_words = greedy_search(decoder, decoder_input, decoder_hidden, max_length)\n",
    "        elif search == 'beam':\n",
    "            decoded_words = beam_search(decoder, decoder_input, decoder_hidden, max_length)  \n",
    "        return decoded_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sacrebleu\n",
    "def calculate_bleu(predictions, labels):\n",
    "\t\"\"\"\n",
    "\tOnly pass a list of strings \n",
    "\t\"\"\"\n",
    "\t# tthis is ony with n_gram = 4\n",
    "\n",
    "\tbleu = sacrebleu.raw_corpus_bleu(predictions, [labels], .01).score\n",
    "\treturn bleu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_e = EncoderRNN(5551, 256)\n",
    "model_e.load_state_dict(torch.load(\"encoder1_40000\", map_location='cpu'))\n",
    "model_e.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_d = DecoderRNN(19344, 256)\n",
    "model_d.load_state_dict(torch.load(\"decoder1_40000\", map_location='cpu'))\n",
    "model_d.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LENGTH = 100\n",
    "def test_model(encoder, decoder, search, test_pairs, lang1, max_length=MAX_LENGTH):\n",
    "    # for test, you only need the lang1 words to be tokenized,\n",
    "    # lang2 words is the true labels\n",
    "    encoder_inputs = [pair[0] for pair in test_pairs]\n",
    "    true_labels = [pair[1] for pair in test_pairs]\n",
    "    translated_predictions = []\n",
    "    for i in range(len(encoder_inputs)): \n",
    "        if i% 100== 0:\n",
    "            print(i)\n",
    "        e_input = encoder_inputs[i]\n",
    "        decoded_words = generate_translation(encoder, decoder, e_input, max_length=MAX_LENGTH)\n",
    "        translated_predictions.append(\" \".join(decoded_words))\n",
    "    return calculate_bleu(translated_predictions, true_labels)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TODO: \n",
    "    \n",
    "Yikes, teh decoder isn't' preforming very well. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pairs = pickle.load(open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_indices_pairs_test\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_lang = pickle.load(open(\"preprocessed_data_no_elmo/iwslt-vi-eng/preprocessed_no_elmo_vilang\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_model(model_e, model_d, \"greedy\", test_pairs, input_lang )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}