Inference_RNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import math\n",
    "\n",
    " \n",
    "import os\n",
    "import random\n",
    "import sklearn\n",
    "\n",
    "\n",
    "\n",
    "words = {}\n",
    "\n",
    "def load_input_sentences(filepath):\n",
    "    sentence_file = open(filepath,'r')\n",
    "    sentence = sentence_file.readline()\n",
    "    sentences = []\n",
    "    solutions = []\n",
    "    while sentence != '':\n",
    "        substrings = sentence.split(' ')\n",
    "        solution = substrings[-1].strip()\n",
    "        sentence = ''\n",
    "        for i in range(len(substrings)-1):\n",
    "            sentence = sentence + substrings[i].strip()+' '    \n",
    "        solutions.append(int(solution))\n",
    "        sentences.append(sentence.strip())\n",
    "        sentence = sentence_file.readline()\n",
    "        \n",
    "    return sentences, solutions\n",
    "\n",
    "\n",
    "\n",
    "def create_vocabulary():\n",
    "    num = 0\n",
    "    vocab_size = 0\n",
    "    for sentence in sentences:\n",
    "        substrings = sentence.split(' ')\n",
    "        for string in substrings:\n",
    "            if words.get(string) == None:\n",
    "                #print num\n",
    "                words[string] = num\n",
    "                num += 1\n",
    "                vocab_size = num\n",
    "    \n",
    "    return vocab_size\n",
    "\n",
    "def create_input_LSTM(sentences):\n",
    "    sentence_embeddings = []\n",
    "    for i in range(len(sentences)):\n",
    "        sentence = sentences[i]\n",
    "        substrings = sentence.split(' ')\n",
    "        for j in range(len(substrings)):\n",
    "            #print(vocab_size)\n",
    "            if j == len(sentence_embeddings):\n",
    "                sentence_embeddings.append([])\n",
    "            embedding = np.zeros([vocab_size])\n",
    "            #print(embedding.shape)\n",
    "            embedding[words.get(substrings[j])] = 1\n",
    "            sentence_embeddings[j].append(embedding)\n",
    "    for i in range(len(sentence_embeddings)):\n",
    "        sentence_embeddings[i] = np.array(sentence_embeddings[i])\n",
    "    return np.array(sentence_embeddings)\n",
    "    \n",
    "def one_hot(input, dimension):\n",
    "    one_hot_vectors = []\n",
    "    map = {}\n",
    "    last_index = 0\n",
    "    for index in input:\n",
    "        one_hot_vector = np.zeros([dimension])\n",
    "        #print(index)\n",
    "        if (map.get(index) == None):\n",
    "            map[index] = last_index\n",
    "            last_index += 1\n",
    "        one_hot_vector[map.get(index)] = 1\n",
    "        one_hot_vectors.append(one_hot_vector)\n",
    "    return np.array(one_hot_vectors)\n",
    "\n",
    "def fc(input, num_output, name = 'fc'):\n",
    "    with tf.variable_scope(name):\n",
    "        num_input = input.get_shape()[1]\n",
    "        W = tf.get_variable('w', [num_input, num_output], tf.float32, tf.random_normal_initializer(0.0, 0.02))\n",
    "        b = tf.get_variable('b', [num_output], initializer = tf.constant_initializer(0.0))\n",
    "        return tf.matmul(input, W) + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\n",
      "[[ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " ..., \n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  1. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  1.]] 6 multiplied by 9\n",
      "(5,) 6 multiplied by 9\n"
     ]
    }
   ],
   "source": [
    "sentences, solutions = load_input_sentences('sentences_nlp.txt')\n",
    "vocab_size = create_vocabulary()\n",
    "print(vocab_size)\n",
    "sentence_embeddings = create_input_LSTM(sentences)\n",
    "solutions_one_hot = one_hot(solutions,100)\n",
    "print (sentence_embeddings[3],sentences[3])\n",
    "print (sentence_embeddings.shape, sentences[3])\n",
    "#test_sentences, test_solutions = load_input_sentences('sentences_nlp_test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'module' object is not callable",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-19-0c78b9238aea>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;31m# Forward passes\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[0mcell\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn_cell\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mBasicLSTMCell\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m \u001b[0mstates_series\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcurrent_state\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcell\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs_series\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minit_state\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     26\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m \u001b[0mlogits_series\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mW2\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mb2\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mstate\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mstates_series\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;31m#Broadcasted addition\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: 'module' object is not callable"
     ]
    }
   ],
   "source": [
    "num_epochs = 100\n",
    "total_series_length = 50000\n",
    "truncated_backprop_length = 15\n",
    "state_size = vocab_size\n",
    "num_classes = 100\n",
    "echo_step = 3\n",
    "batch_size = 5\n",
    "num_batches = total_series_length//batch_size//truncated_backprop_length\n",
    "\n",
    "\n",
    "batchX_placeholder = tf.placeholder(tf.float32, [batch_size, truncated_backprop_length, vocab_size])\n",
    "batchY_placeholder = tf.placeholder(tf.int32, [batch_size, truncated_backprop_length, vocab_size])\n",
    "\n",
    "init_state = tf.placeholder(tf.float32, [batch_size, state_size, vocab_size])\n",
    "\n",
    "W2 = tf.Variable(np.random.rand(state_size, num_classes),dtype=tf.float32)\n",
    "b2 = tf.Variable(np.zeros((1,num_classes)), dtype=tf.float32)\n",
    "\n",
    "# Unpack columns\n",
    "inputs_series = tf.unstack(batchX_placeholder, axis=1)\n",
    "labels_series = tf.unstack(batchY_placeholder, axis=1)\n",
    "\n",
    "# Forward passes\n",
    "cell = tf.nn.rnn_cell.BasicLSTMCell(state_size)\n",
    "states_series, current_state = tf.contrib.rnn(cell, inputs_series, init_state)\n",
    "\n",
    "logits_series = [tf.matmul(state, W2) + b2 for state in states_series] #Broadcasted addition\n",
    "predictions_series = [tf.nn.softmax(logits) for logits in logits_series]\n",
    "\n",
    "losses = [tf.nn.sparse_softmax_cross_entropy_with_logits(logits, labels) for logits, labels in zip(logits_series,labels_series)]\n",
    "total_loss = tf.reduce_mean(losses)\n",
    "\n",
    "train_step = tf.train.AdagradOptimizer(0.3).minimize(total_loss)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.initialize_all_variables())\n",
    "    loss_list = []\n",
    "\n",
    "    for epoch_idx in range(num_epochs):\n",
    "        x = sentence_embeddings\n",
    "        y = solutions_one_hot\n",
    "        _current_state = np.zeros((batch_size, state_size, vocab_size))\n",
    "\n",
    "        print(\"New data, epoch\", epoch_idx)\n",
    "\n",
    "        for batch_idx in range(num_batches):\n",
    "            start_idx = batch_idx * truncated_backprop_length\n",
    "            end_idx = start_idx + truncated_backprop_length\n",
    "\n",
    "            batchX = x[:,start_idx:end_idx]\n",
    "            batchY = y[:,start_idx:end_idx]\n",
    "\n",
    "            _total_loss, _train_step, _current_state, _predictions_series = sess.run(\n",
    "                [total_loss, train_step, current_state, predictions_series],\n",
    "                feed_dict={\n",
    "                    batchX_placeholder:batchX,\n",
    "                    batchY_placeholder:batchY,\n",
    "                    init_state:_current_state\n",
    "                })\n",
    "\n",
    "            loss_list.append(_total_loss)\n",
    "\n",
    "            if batch_idx%100 == 0:\n",
    "                print(\"Step\",batch_idx, \"Loss\", _total_loss)\n",
    "                plot(loss_list, _predictions_series, batchX, batchY)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DeepLearning",
   "language": "python",
   "name": "deeplearning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}