CTG+VAE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 1/20 [00:42<13:23, 42.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(10.6126, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "40644.60172941534\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 2/20 [01:24<12:35, 41.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(9.4371, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "12545.012760173635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 3/20 [02:05<11:50, 41.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(9.0147, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "8223.415756655515\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 4/20 [02:45<10:58, 41.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(8.6417, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "5662.8448644029195\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 5/20 [03:29<10:29, 41.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(8.3797, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "4357.550535796576\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 6/20 [04:12<09:53, 42.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(8.2274, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "3741.9375511069543\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 7/20 [04:52<09:00, 41.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(8.0681, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "3191.1182357241064\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 8/20 [05:32<08:13, 41.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.9492, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "2833.2240697442367\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 9/20 [06:13<07:30, 40.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.8584, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "2587.425033673666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 10/20 [06:53<06:48, 40.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.7860, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "2406.7261599557783\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 11/20 [07:33<06:05, 40.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.7132, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "2237.5998711865595\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 12/20 [08:13<05:22, 40.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.6146, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "2027.5897322684073\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 13/20 [08:54<04:43, 40.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.6103, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "2018.7981247652624\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 14/20 [09:35<04:04, 40.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.5357, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "1873.814207384793\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 15/20 [10:16<03:24, 40.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.5048, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "1816.6835540346658\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 16/20 [10:57<02:43, 40.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.4524, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "1724.0503890296986\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 17/20 [11:38<02:02, 40.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.4053, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "1644.6203633895284\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 18/20 [12:18<01:21, 40.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.3619, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "1574.8827316251268\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 19/20 [13:03<00:41, 41.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.3122, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "1498.4302910866338\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [13:43<00:00, 41.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.3045, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "1486.9537516699113\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "import math\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "from torch import optim\n",
    "from torch.utils.data import DataLoader\n",
    "import tqdm\n",
    "from Datasets import Datasets\n",
    "import math\n",
    "\n",
    "\n",
    "dataset = Datasets(\"C:\\Attention\\data\\\\train.txt\")\n",
    "\n",
    "dataset.bulid_vocab(dataset.en_data,dataset.ch_data)\n",
    "\n",
    "dataloader = DataLoader(dataset, batch_size=16, num_workers=0,collate_fn=dataset.collate_fn)\n",
    "\n",
    "\n",
    "maxlen = 128\n",
    "d_model = 512\n",
    "units = 512\n",
    "dropout_rate = 0.2\n",
    "numofblock = 4\n",
    "numofhead = 4\n",
    "# encoder_vocab = len(dataset.ch_vocab)\n",
    "vocab_size = len(dataset.en_vocab)\n",
    "epochs = 20\n",
    "latent_dim = 512\n",
    "DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "\n",
    "def get_padding_mask(seq_q,seq_k):\n",
    "    # print(seq_k.shape)\n",
    "    # print(seq_q.shape)\n",
    "    batch_size, len_q = seq_q.size()\n",
    "    batch_size, len_k = seq_k.size()\n",
    "    padding_mask = seq_k.data.eq(1).unsqueeze(1)\n",
    "    return padding_mask.expand(batch_size,len_q,len_k)\n",
    "\n",
    "\n",
    "class TokenEmbedding(nn.Module):\n",
    "    def __init__(self,vocab_size,emb_size):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size,emb_size)\n",
    "\n",
    "    def forward(self,x):\n",
    "        # print(x.shape)\n",
    "\n",
    "        return self.embedding(x).to(DEVICE) # shape = (batch_size,input_seq_len,emb_dim)\n",
    "\n",
    "\n",
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self, num_units, num_heads, dropout_rate, mask=False):\n",
    "        super().__init__()\n",
    "        self.num_units = num_units\n",
    "        self.num_head = num_heads\n",
    "        self.dropout_rate = dropout_rate\n",
    "        self.mask = mask\n",
    "        self.linearQ = nn.Linear(self.num_units,self.num_units)\n",
    "        self.linearK = nn.Linear(self.num_units,self.num_units)\n",
    "        self.linearV = nn.Linear(self.num_units,self.num_units)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.dropout = nn.Dropout(self.dropout_rate)\n",
    "        self.LayerNormalization = nn.LayerNorm(d_model)\n",
    "        self.Q = nn.Sequential(self.linearQ,self.relu)\n",
    "        self.K = nn.Sequential(self.linearK,self.relu)\n",
    "        self.V = nn.Sequential(self.linearV,self.relu)\n",
    "\n",
    "\n",
    "    def forward(self, queries, keys, values, self_padding_mask, enc_dec_padding_mask):\n",
    "        '''\n",
    "        :param queries: shape:(batch_size,input_seq_len,d_model)\n",
    "        :param keys: shape:(batch_size,input_seq_len,d_model)\n",
    "        :param values: shape:(batch_size,input_seq_len,d_model)\n",
    "        :return: None\n",
    "        '''\n",
    "        q, k, v = self.Q(queries), self.K(keys), self.V(values)\n",
    "\n",
    "        q_split, k_split, v_split = torch.chunk(q,self.num_head,dim=-1), torch.chunk(k,self.num_head,dim=-1), torch.chunk(v,self.num_head,dim=-1)\n",
    "        q_, k_, v_ = torch.stack(q_split,dim=1), torch.stack(k_split,dim=1), torch.stack(v_split,dim=1)\n",
    "        # shape : (batch_size, num_head, input_seq_len, depth = d_model/num_head)\n",
    "        a = torch.matmul(q_,k_.permute(0,1,3,2)) # a = q * k^T(后两个维度)\n",
    "        a = a / (k_.size()[-1] ** 0.5) # shape:(batch_size,num_head,seq_len,seq_len)\n",
    "        batch_size_shape = a.shape[0]\n",
    "        seq_len_shape = a.shape[2]\n",
    "        if self.mask:\n",
    "            self_padding_mask = self_padding_mask.unsqueeze(1).repeat(1, self.num_head, 1, 1)\n",
    "            masked = torch.ones((batch_size_shape,1,seq_len_shape,seq_len_shape))\n",
    "            masked = Variable((1 - torch.tril(masked, diagonal=0)) * (-2 ** 32 + 1)).to(DEVICE)\n",
    "\n",
    "            assert masked.shape[-1] == self_padding_mask.shape[-1]\n",
    "            a = a + masked\n",
    "            a.masked_fill_(self_padding_mask,-1e9)\n",
    "        else:\n",
    "            enc_dec_padding_mask = enc_dec_padding_mask.unsqueeze(1).repeat(1, self.num_head, 1, 1)\n",
    "            a.masked_fill_(enc_dec_padding_mask,-1e9)\n",
    "\n",
    "        a = F.softmax(a,dim=-1)\n",
    "\n",
    "        a = torch.matmul(a,v_)\n",
    "        a = torch.reshape(a.permute(0, 2, 1, 3), shape=(q.shape[0],q.shape[1],q.shape[2]))\n",
    "        a = self.dropout(a)\n",
    "        a += queries\n",
    "        a = self.LayerNormalization(a).to(DEVICE)\n",
    "        return a\n",
    "\n",
    "\n",
    "class FC(nn.Module):\n",
    "    def __init__(self,input_channels,units=(2048,512)):\n",
    "        super().__init__()\n",
    "        self.input_channels = input_channels\n",
    "        self.units = units\n",
    "        self.layer1 = nn.Linear(self.input_channels,units[0])\n",
    "        self.layer2 = nn.Linear(self.units[0],self.units[1])\n",
    "        self.relu = nn.ReLU()\n",
    "        self.LayerNormalization = nn.LayerNorm(d_model)\n",
    "\n",
    "\n",
    "    def forward(self,x):\n",
    "        outputs = self.layer1(x)\n",
    "        outputs = self.relu(outputs)\n",
    "        outputs = self.layer2(outputs)\n",
    "        outputs += x\n",
    "        outputs = self.LayerNormalization(outputs)\n",
    "        return outputs.to(DEVICE)\n",
    "\n",
    "\n",
    "class PositionalEncoding(nn.Module):\n",
    "    def __init__(self, d_model=d_model, dropout=0.1, max_len=5000):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
    "        self.register_buffer('pe', pe)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        x: [seq_len, batch_size, d_model]\n",
    "        \"\"\"\n",
    "        x = x + self.pe[:x.size(0), :]\n",
    "        return self.dropout(x)\n",
    "\n",
    "\n",
    "class EncoderLayer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "        self.mask_self_attention = MultiHeadAttention(units,numofhead,dropout_rate,True)\n",
    "        self.fc = FC(d_model)\n",
    "\n",
    "\n",
    "    def forward(self,inputs,padding_mask):\n",
    "        outputs = self.mask_self_attention(inputs,inputs,inputs,padding_mask,None)\n",
    "        outputs = self.fc(outputs)\n",
    "        return outputs.to(DEVICE)\n",
    "\n",
    "\n",
    "\n",
    "class DecoderLayer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "        self.self_attention = MultiHeadAttention(units,numofhead,dropout_rate,mask=False)\n",
    "        self.fc = FC(d_model)\n",
    "\n",
    "\n",
    "\n",
    "    def forward(self,enc_outputs):\n",
    "        outputs = self.fc(enc_outputs)\n",
    "        return outputs.to(DEVICE)\n",
    "\n",
    "\n",
    "\n",
    "class Decoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.layers = nn.ModuleList([DecoderLayer() for _ in range(numofblock)])\n",
    "\n",
    "\n",
    "    def forward(self,x):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self,vocab_size):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.pe = PositionalEncoding()\n",
    "        self.embedding = TokenEmbedding(vocab_size,units)\n",
    "        self.layers = nn.ModuleList([EncoderLayer() for _ in range(numofblock)])\n",
    "\n",
    "\n",
    "    def forward(self,inputs):\n",
    "        outputs = self.embedding(inputs)\n",
    "        outputs = self.pe(outputs.transpose(0, 1)).transpose(0, 1)\n",
    "\n",
    "        padding_mask = get_padding_mask(inputs,inputs)\n",
    "        for layer in self.layers:\n",
    "            outputs = layer(outputs,padding_mask)\n",
    "        return outputs\n",
    "\n",
    "\n",
    "\n",
    "class CTG(nn.Module):\n",
    "    def __init__(self,vocab_size):\n",
    "        super(CTG, self).__init__()\n",
    "        self.Encoder = Encoder(vocab_size)\n",
    "        self.Decoder = Decoder()\n",
    "        self.linear = nn.Linear(d_model,vocab_size)\n",
    "        self.mean = nn.Linear(d_model,latent_dim)\n",
    "        self.log_var = nn.Linear(d_model,latent_dim)\n",
    "\n",
    "    def reparameterize(self,z_mean,z_log_var):\n",
    "        std = torch.exp(0.5 * z_log_var)\n",
    "        eps = torch.randn_like(std)\n",
    "        return eps * std + z_mean\n",
    "\n",
    "\n",
    "    def forward(self,x):\n",
    "        enc_outputs= self.Encoder(x)\n",
    "\n",
    "        z_mean = self.mean(enc_outputs)\n",
    "        z_log_var = self.log_var(enc_outputs)\n",
    "\n",
    "        z = self.reparameterize(z_mean,z_log_var)\n",
    "        enc_outputs = self.Decoder(z)\n",
    "        logits = self.linear(enc_outputs)\n",
    "\n",
    "        logits = logits.view(-1, logits.size(-1))\n",
    "        return logits,z_mean,z_log_var\n",
    "\n",
    "\n",
    "model = CTG(vocab_size).to(DEVICE)\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=1)\n",
    "optimizer = optim.SGD(model.parameters(), lr=1e-3, momentum=0.99)\n",
    "\n",
    "\n",
    "for epoch in tqdm.tqdm(range(epochs)):\n",
    "    total = []\n",
    "    for _,dec_inputs,dec_outputs in dataloader:\n",
    "\n",
    "        dec_inputs,dec_outputs= dec_inputs.to(DEVICE),dec_outputs.to(DEVICE)\n",
    "\n",
    "        outputs,z_mean,z_log_var = model(dec_inputs)\n",
    "\n",
    "        normal_loss = criterion(outputs,dec_outputs.contiguous().view(-1))\n",
    "\n",
    "        reconstruction_loss = F.cross_entropy(outputs,dec_outputs.contiguous().view(-1))\n",
    "        kl_loss = torch.mean(0.5 * torch.sum(torch.exp(z_log_var) + z_mean ** 2 - 1. - z_log_var, 1))\n",
    "\n",
    "        loss = normal_loss + reconstruction_loss + kl_loss\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        total.append(loss)\n",
    "        optimizer.step()\n",
    "    cur_loss = sum(total)/len(total)\n",
    "    print(cur_loss)\n",
    "    print(math.exp(cur_loss))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\n",
    "def greedy_decoder(model, start_symbol):\n",
    "    \"\"\"贪心编码\n",
    "    For simplicity, a Greedy Decoder is Beam search when K=1. This is necessary for inference as we don't know the\n",
    "    target sequence input. Therefore we try to generate the target input word by word, then feed it into the transformer.\n",
    "    Starting Reference: http://nlp.seas.harvard.edu/2018/04/03/attention.html#greedy-decoding\n",
    "    :param model: Transformer Model\n",
    "    :param enc_input: The encoder input\n",
    "    :param start_symbol: The start symbol. In this example it is 'S' which corresponds to index 4\n",
    "    :return: The target input\n",
    "    \"\"\"\n",
    "    inputs = torch.zeros(1, 0).long()\n",
    "    terminal = False\n",
    "    next_symbol = start_symbol\n",
    "    while not terminal:\n",
    "        # 预测阶段：inputs序列会一点点变长（每次添加一个新预测出来的单词）\n",
    "        inputs = torch.cat([inputs.to(DEVICE), torch.tensor([[next_symbol]], dtype=inputs.dtype).to(DEVICE)],\n",
    "                              -1)\n",
    "        # print(\"inputs:\")\n",
    "        # print(inputs)\n",
    "        dec_outputs = model.Encoder(inputs)\n",
    "        dec_outputs = model.Decoder(dec_outputs)\n",
    "        dec_outputs = model.linear(dec_outputs)\n",
    "\n",
    "        prob = dec_outputs.squeeze(0).max(dim=-1, keepdim=False)[1]\n",
    "        # print(\"prob:\")\n",
    "        # print(dataset.idx2enwords(prob))\n",
    "        # 增量更新（我们希望重复单词预测结果是一样的）\n",
    "        # 我们在预测是会选择性忽略重复的预测的词，只摘取最新预测的单词拼接到输入序列中\n",
    "        next_word = prob.data[-1]  # 拿出当前预测的单词(数字)。我们用x'_t对应的输出z_t去预测下一个单词的概率，不用z_1,z_2..z_{t-1}\n",
    "        # print(next_word)\n",
    "\n",
    "        next_symbol = next_word\n",
    "        # print(dataset.idx2en(next_word))\n",
    "        if next_symbol == dataset.en_vocab[\"<eos>\"]:\n",
    "            terminal = True\n",
    "        # print(next_word)\n",
    "\n",
    "    # greedy_dec_predict = torch.cat(\n",
    "    #     [inputs.to(device), torch.tensor([[next_symbol]], dtype=enc_input.dtype).to(device)],\n",
    "    #     -1)\n",
    "    greedy_dec_predict = inputs[:, 1:]\n",
    "    return greedy_dec_predict\n",
    "\n",
    "for i in range(20):\n",
    "    greedy_dec_predict = greedy_decoder(model, start_symbol=dataset.en_vocab[\"<bos>\"])\n",
    "    # print(input[i], '->', greedy_dec_predict.squeeze())\n",
    "    print(\" \".join([dataset.idx2en(n.item()) for n in greedy_dec_predict.squeeze()]))\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bos> i'm very greene after him with the leg.\n",
      "<bos> she doesn't have to stay with me here?\n",
      "<bos> you're good yesterday.\n",
      "<bos> can was that?\n",
      "<bos> the list in tennis decisions\n",
      "<bos> they work more and the dress.\n",
      "<bos> he's roll in this light.\n",
      "<bos> see your way for sleep is a skirt.\n",
      "<bos> the book was often all first visit two to the secret.\n",
      "<bos> we'd just long novel.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def get_sequence():\n",
    "    s = dataset.words2idx(\"<bos>\".split(),\"en\")\n",
    "    s = s.unsqueeze(0).to(DEVICE)\n",
    "\n",
    "    # print(s.shape)\n",
    "    flag = True\n",
    "    data = torch.tensor([]).long().to(DEVICE).unsqueeze(0)\n",
    "    count = 0\n",
    "    while flag:\n",
    "        # print(s,data)\n",
    "        s = torch.cat((s,data),dim=-1)\n",
    "        dec_outputs,z_mean,z_log_var = model(s.to(DEVICE))\n",
    "        prob = F.softmax(dec_outputs, dim=-1)\n",
    "        # print(prob)\n",
    "        prob = torch.multinomial(prob, num_samples=1)\n",
    "        # print(prob)\n",
    "        data = prob[-1].unsqueeze(0)\n",
    "        # print(prob)\n",
    "        # data = prob\n",
    "#         print(data)\n",
    "        count += 1\n",
    "        if data == 3:\n",
    "            flag = False\n",
    "        if count == 20:\n",
    "            flag = False\n",
    "    # print()\n",
    "\n",
    "    # print(prob)\n",
    "#     print(s)\n",
    "    # for i in prob:\n",
    "    print(dataset.idx2enwords(s[-1]))\n",
    "for i in range(10):\n",
    "    get_sequence()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bos> he was a good books of the morning.\n",
      "<bos> he was a good time.\n",
      "<bos> he was a good books of the party.\n",
      "<bos> he was a good time.\n",
      "<bos> he was a good time.\n",
      "<bos> he was a good time of the house.\n",
      "<bos> he was a good time.\n",
      "<bos> he was a good time for the house.\n",
      "<bos> he was a good more and in the morning.\n",
      "<bos> he was a good more years than the house.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def get_sequence():\n",
    "    s = dataset.words2idx(\"<bos> he\".split(),\"en\")\n",
    "    s = s.unsqueeze(0).to(DEVICE)\n",
    "\n",
    "    # print(s.shape)\n",
    "    flag = True\n",
    "    data = torch.tensor([]).long().to(DEVICE).unsqueeze(0)\n",
    "    inputs = torch.zeros(1, 0).long()\n",
    "    count = 0\n",
    "    while flag:\n",
    "\n",
    "        s = torch.cat((s,data),dim=-1)\n",
    "        dec_outputs,_,_ = model(s.to(DEVICE))\n",
    "\n",
    "        prob = dec_outputs.squeeze(0).max(dim=-1, keepdim=False)[1]\n",
    "\n",
    "        data = prob[-1].unsqueeze(0).unsqueeze(0)\n",
    "\n",
    "\n",
    "#         print(data)\n",
    "        count += 1\n",
    "        if data == 3:\n",
    "            flag = False\n",
    "        if count == 20:\n",
    "            flag = False\n",
    "    # print()\n",
    "\n",
    "    # print(prob)\n",
    "    # for i in prob:\n",
    "    print(dataset.idx2enwords(s[-1]))\n",
    "for i in range(10):\n",
    "    get_sequence()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}