check if there is GPU.ipynb

{
    "cells": [
        {
            "cell_type": "code", 
            "execution_count": 2, 
            "metadata": {}, 
            "source": "#from theano.sandbox import cuda\n#cuda.use('gpu1')", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 3, 
            "metadata": {}, 
            "source": "%matplotlib inline\nimport utils; reload(utils)\nfrom utils import *\nfrom __future__ import division, print_function", 
            "outputs": [
                {
                    "evalue": "No module named 'utils'", 
                    "traceback": [
                        "\u001b[1;31m\u001b[0m", 
                        "\u001b[1;31mImportError\u001b[0mTraceback (most recent call last)", 
                        "\u001b[1;32m<ipython-input-3-ee90c9585067>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mget_ipython\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmagic\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'matplotlib inline'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mutils\u001b[0m\u001b[1;33m;\u001b[0m \u001b[0mreload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mutils\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mutils\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[1;33m*\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0m__future__\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mdivision\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprint_function\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", 
                        "\u001b[1;31mImportError\u001b[0m: No module named 'utils'"
                    ], 
                    "ename": "ImportError", 
                    "output_type": "error"
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "## Setup", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "We're going to download the collected works of Nietzsche to use as our data for this class.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 4, 
            "metadata": {
                "hidden": true
            }, 
            "source": "path = get_file('nietzsche.txt', origin=\"https://s3.amazonaws.com/text-datasets/nietzsche.txt\")\ntext = open(path).read()\nprint('corpus length:', len(text))", 
            "outputs": [
                {
                    "evalue": "name 'get_file' is not defined", 
                    "traceback": [
                        "\u001b[1;31m\u001b[0m", 
                        "\u001b[1;31mNameError\u001b[0mTraceback (most recent call last)", 
                        "\u001b[1;32m<ipython-input-4-e4fa52ec9899>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mpath\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'nietzsche.txt'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morigin\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"https://s3.amazonaws.com/text-datasets/nietzsche.txt\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mtext\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'corpus length:'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", 
                        "\u001b[1;31mNameError\u001b[0m: name 'get_file' is not defined"
                    ], 
                    "ename": "NameError", 
                    "output_type": "error"
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 4, 
            "metadata": {
                "hidden": true
            }, 
            "source": "chars = sorted(list(set(text)))\nvocab_size = len(chars)+1\nprint('total chars:', vocab_size)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "total chars: 86\n", 
                    "name": "stdout"
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "Sometimes it's useful to have a zero value in the dataset, e.g. for padding", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 5, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "chars.insert(0, \"\\0\")", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 6, 
            "metadata": {
                "hidden": true
            }, 
            "source": "''.join(chars[1:-6])", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 6, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "'\\n !\"\\'(),-.0123456789:;=?ABCDEFGHIJKLMNOPQRSTUVWXYZ[]_abcdefghijklmnopqrstuvwxyz'"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "Map from chars to indices and back again", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 7, 
            "metadata": {
                "hidden": true
            }, 
            "source": "char_indices = dict((c, i) for i, c in enumerate(chars))\nindices_char = dict((i, c) for i, c in enumerate(chars))", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "*idx* will be the data we use from now own - it simply converts all the characters to their index (based on the mapping above)", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 8, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "idx = [char_indices[c] for c in text]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 9, 
            "metadata": {
                "hidden": true
            }, 
            "source": "idx[:10]", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 9, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "[40, 42, 29, 30, 25, 27, 29, 1, 1, 1]"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 10, 
            "metadata": {
                "hidden": true
            }, 
            "source": "''.join(indices_char[i] for i in idx[:70])", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 10, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "'PREFACE\\n\\n\\nSUPPOSING that Truth is a woman--what then? Is there not gro'"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "## 3 char model", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Create inputs", 
            "metadata": {
                "hidden": true, 
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "Create a list of every 4th character, starting at the 0th, 1st, 2nd, then 3rd characters", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 11, 
            "metadata": {
                "hidden": true
            }, 
            "source": "cs=3\nc1_dat = [idx[i] for i in xrange(0, len(idx)-1-cs, cs)]\nc2_dat = [idx[i+1] for i in xrange(0, len(idx)-1-cs, cs)]\nc3_dat = [idx[i+2] for i in xrange(0, len(idx)-1-cs, cs)]\nc4_dat = [idx[i+3] for i in xrange(0, len(idx)-1-cs, cs)]", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Our inputs", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 12, 
            "metadata": {
                "hidden": true
            }, 
            "source": "x1 = np.stack(c1_dat[:-2])\nx2 = np.stack(c2_dat[:-2])\nx3 = np.stack(c3_dat[:-2])", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Our output", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 13, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "y = np.stack(c4_dat[:-2])", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "The first 4 inputs and outputs", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 14, 
            "metadata": {
                "hidden": true, 
                "scrolled": true
            }, 
            "source": "x1[:4], x2[:4], x3[:4]", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 14, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "(array([40, 30, 29,  1]), array([42, 25,  1, 43]), array([29, 27,  1, 45]))"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 15, 
            "metadata": {
                "hidden": true
            }, 
            "source": "y[:4]", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 15, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "array([30, 29,  1, 40])"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 16, 
            "metadata": {
                "hidden": true
            }, 
            "source": "x1.shape, y.shape", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 16, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "((200297,), (200297,))"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "The number of latent factors to create (i.e. the size of the embedding matrix)", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 17, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "n_fac = 42", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Create inputs and embedding outputs for each of our 3 character inputs", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 18, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def embedding_input(name, n_in, n_out):\n    inp = Input(shape=(1,), dtype='int64', name=name)\n    emb = Embedding(n_in, n_out, input_length=1)(inp)\n    return inp, Flatten()(emb)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 19, 
            "metadata": {
                "hidden": true
            }, 
            "source": "c1_in, c1 = embedding_input('c1', vocab_size, n_fac)\nc2_in, c2 = embedding_input('c2', vocab_size, n_fac)\nc3_in, c3 = embedding_input('c3', vocab_size, n_fac)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Create and train model", 
            "metadata": {
                "hidden": true, 
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "Pick a size for our hidden state", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 20, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "n_hidden = 256", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "This is the 'green arrow' from our diagram - the layer operation from input to hidden.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 47, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "dense_in = Dense(n_hidden, activation='relu')", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Our first hidden activation is simply this function applied to the result of the embedding of the first character.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 48, 
            "metadata": {
                "hidden": true
            }, 
            "source": "c1_hidden = dense_in(c1)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "This is the 'orange arrow' from our diagram - the layer operation from hidden to hidden.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 49, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "dense_hidden = Dense(n_hidden, activation='tanh')", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Our second and third hidden activations sum up the previous hidden state (after applying dense_hidden) to the new input state.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 50, 
            "metadata": {
                "hidden": true
            }, 
            "source": "c2_dense = dense_in(c2)\nhidden_2 = dense_hidden(c1_hidden)\nc2_hidden = merge([c2_dense, hidden_2])", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 51, 
            "metadata": {
                "hidden": true
            }, 
            "source": "c3_dense = dense_in(c3)\nhidden_3 = dense_hidden(c2_hidden)\nc3_hidden = merge([c3_dense, hidden_3])", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "This is the 'blue arrow' from our diagram - the layer operation from hidden to output.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 52, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "dense_out = Dense(vocab_size, activation='softmax')", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "The third hidden state is the input to our output layer.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 53, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "c4_out = dense_out(c3_hidden)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 54, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model = Model([c1_in, c2_in, c3_in], c4_out)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 55, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam())", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 56, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "model.optimizer.lr=0.000001", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 57, 
            "metadata": {
                "hidden": true, 
                "scrolled": false
            }, 
            "source": "model.fit([x1, x2, x3], y, batch_size=64, nb_epoch=4)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/4\n200297/200297 [==============================] - 4s - loss: 4.2459     \nEpoch 2/4\n200297/200297 [==============================] - 5s - loss: 3.6853     \nEpoch 3/4\n200297/200297 [==============================] - 5s - loss: 3.3444     \nEpoch 4/4\n200297/200297 [==============================] - 5s - loss: 3.1719     \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 57, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7ff3778ec510>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 58, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "model.optimizer.lr=0.01", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 59, 
            "metadata": {
                "hidden": true, 
                "scrolled": false
            }, 
            "source": "model.fit([x1, x2, x3], y, batch_size=64, nb_epoch=4)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/4\n200297/200297 [==============================] - 5s - loss: 3.0829     \nEpoch 2/4\n200297/200297 [==============================] - 5s - loss: 3.0314     \nEpoch 3/4\n200297/200297 [==============================] - 4s - loss: 2.9973     \nEpoch 4/4\n200297/200297 [==============================] - 5s - loss: 2.9722     \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 59, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7ff3778ecdd0>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 42, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "model.optimizer.lr.set_value(0.000001)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 43, 
            "metadata": {
                "hidden": true, 
                "scrolled": false
            }, 
            "source": "model.fit([x1, x2, x3], y, batch_size=64, nb_epoch=4)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/4\n200297/200297 [==============================] - 5s - loss: 4.4125     \nEpoch 2/4\n200297/200297 [==============================] - 5s - loss: 4.2799     \nEpoch 3/4\n200297/200297 [==============================] - 5s - loss: 4.0000     \nEpoch 4/4\n200297/200297 [==============================] - 5s - loss: 3.5942     \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 43, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7ff3788d4d10>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 44, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "model.optimizer.lr.set_value(0.01)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 45, 
            "metadata": {
                "hidden": true, 
                "scrolled": false
            }, 
            "source": "model.fit([x1, x2, x3], y, batch_size=64, nb_epoch=4)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/4\n200297/200297 [==============================] - 5s - loss: 7.8651     \nEpoch 2/4\n200297/200297 [==============================] - 5s - loss: 5.1607     \nEpoch 3/4\n200297/200297 [==============================] - 5s - loss: 4.7043     \nEpoch 4/4\n200297/200297 [==============================] - 5s - loss: 4.7026     \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 45, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7ff37ba08a50>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Test model", 
            "metadata": {
                "hidden": true, 
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 159, 
            "metadata": {
                "hidden": true
            }, 
            "source": "def get_next(inp):\n    idxs = [char_indices[c] for c in inp]\n    arrs = [np.array(i)[np.newaxis] for i in idxs]\n    p = model.predict(arrs)\n    i = np.argmax(p)\n    return chars[i]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 160, 
            "metadata": {
                "hidden": true, 
                "scrolled": false
            }, 
            "source": "get_next('phi')", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 160, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "'l'"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 161, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_next(' th')", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 161, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "'e'"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 162, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_next(' an')", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 162, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "'d'"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "## Our first RNN!", 
            "metadata": {}
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Create inputs", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "This is the size of our unrolled RNN.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 73, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "cs=8", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "For each of 0 through 7, create a list of every 8th character with that starting point. These will be the 8 inputs to out model.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 74, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "c_in_dat = [[idx[i+n] for i in xrange(0, len(idx)-1-cs, cs)]\n            for n in range(cs)]", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Then create a list of the next character in each of these series. This will be the labels for our model.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 75, 
            "metadata": {
                "hidden": true
            }, 
            "source": "c_out_dat = [idx[i+cs] for i in xrange(0, len(idx)-1-cs, cs)]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 76, 
            "metadata": {
                "hidden": true
            }, 
            "source": "xs = [np.stack(c[:-2]) for c in c_in_dat]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 77, 
            "metadata": {
                "hidden": true
            }, 
            "source": "len(xs), xs[0].shape", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 77, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "(8, (75110,))"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 45, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "y = np.stack(c_out_dat[:-2])", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "So each column below is one series of 8 characters from the text.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 78, 
            "metadata": {
                "hidden": true
            }, 
            "source": "[xs[n][:cs] for n in range(cs)]", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 78, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "[array([40,  1, 33,  2, 72, 67, 73,  2]),\n array([42,  1, 38, 44,  2,  9, 61, 73]),\n array([29, 43, 31, 71, 54,  9, 58, 61]),\n array([30, 45,  2, 74,  2, 76, 67, 58]),\n array([25, 40, 73, 73, 76, 61, 24, 71]),\n array([27, 40, 61, 61, 68, 54,  2, 58]),\n array([29, 39, 54,  2, 66, 73, 33,  2]),\n array([ 1, 43, 73, 62, 54,  2, 72, 67])]"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "...and this is the next character after each sequence.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 57, 
            "metadata": {
                "hidden": true
            }, 
            "source": "y[:cs]", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 57, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "array([ 1, 33,  2, 72, 67, 73,  2, 68])"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 58, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "n_fac = 42", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Create and train model", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 33, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def embedding_input(name, n_in, n_out):\n    inp = Input(shape=(1,), dtype='int64', name=name+'_in')\n    emb = Embedding(n_in, n_out, input_length=1, name=name+'_emb')(inp)\n    return inp, Flatten()(emb)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 34, 
            "metadata": {
                "hidden": true
            }, 
            "source": "c_ins = [embedding_input('c'+str(n), vocab_size, n_fac) for n in range(cs)]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 35, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "n_hidden = 256", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 36, 
            "metadata": {
                "hidden": true
            }, 
            "source": "dense_in = Dense(n_hidden, activation='relu')\ndense_hidden = Dense(n_hidden, activation='relu', init='identity')\ndense_out = Dense(vocab_size, activation='softmax')", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "The first character of each sequence goes through dense_in(), to create our first hidden activations.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 37, 
            "metadata": {
                "hidden": true
            }, 
            "source": "hidden = dense_in(c_ins[0][1])", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Then for each successive layer we combine the output of dense_in() on the next character with the output of dense_hidden() on the current hidden state, to create the new hidden state.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 38, 
            "metadata": {
                "hidden": true
            }, 
            "source": "for i in range(1,cs):\n    c_dense = dense_in(c_ins[i][1])\n    hidden = dense_hidden(hidden)\n    hidden = merge([c_dense, hidden])", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Putting the final hidden state through dense_out() gives us our output.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 39, 
            "metadata": {
                "hidden": true
            }, 
            "source": "c_out = dense_out(hidden)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "So now we can create our model.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 179, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model = Model([c[0] for c in c_ins], c_out)\nmodel.compile(loss='sparse_categorical_crossentropy', optimizer=Adam())", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 180, 
            "metadata": {
                "hidden": true, 
                "scrolled": false
            }, 
            "source": "model.fit(xs, y, batch_size=64, nb_epoch=12)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/12\n75110/75110 [==============================] - 3s - loss: 2.5385     \nEpoch 2/12\n75110/75110 [==============================] - 3s - loss: 2.2645     \nEpoch 3/12\n75110/75110 [==============================] - 3s - loss: 2.1596     \nEpoch 4/12\n75110/75110 [==============================] - 3s - loss: 2.0888     \nEpoch 5/12\n75110/75110 [==============================] - 3s - loss: 2.0355     \nEpoch 6/12\n75110/75110 [==============================] - 3s - loss: 1.9897     \nEpoch 7/12\n75110/75110 [==============================] - 3s - loss: 1.9506     \nEpoch 8/12\n75110/75110 [==============================] - 3s - loss: 1.9149     \nEpoch 9/12\n75110/75110 [==============================] - 3s - loss: 1.8840     \nEpoch 10/12\n75110/75110 [==============================] - 3s - loss: 1.8546     \nEpoch 11/12\n75110/75110 [==============================] - 3s - loss: 1.8293     \nEpoch 12/12\n75110/75110 [==============================] - 3s - loss: 1.8050     \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 180, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7f25579a80d0>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Test model", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 181, 
            "metadata": {
                "hidden": true
            }, 
            "source": "def get_next(inp):\n    idxs = [np.array(char_indices[c])[np.newaxis] for c in inp]\n    p = model.predict(idxs)\n    return chars[np.argmax(p)]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 182, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_next('for thos')", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 182, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "'e'"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 432, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_next('part of ')", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 432, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "'t'"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 433, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_next('queens a')", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 433, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "'n'"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "## Our first RNN with keras!", 
            "metadata": {
                "collapsed": true, 
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 30, 
            "metadata": {
                "hidden": true
            }, 
            "source": "n_hidden, n_fac, cs, vocab_size = (256, 42, 8, 86)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "This is nearly exactly equivalent to the RNN we built ourselves in the previous section.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 31, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model=Sequential([\n        Embedding(vocab_size, n_fac, input_length=cs),\n        SimpleRNN(n_hidden, activation='relu', inner_init='identity'),\n        Dense(vocab_size, activation='softmax')\n    ])", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 32, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model.summary()", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "____________________________________________________________________________________________________\nLayer (type)                     Output Shape          Param #     Connected to                     \n====================================================================================================\nembedding_5 (Embedding)          (None, 8, 42)         3612        embedding_input_2[0][0]          \n____________________________________________________________________________________________________\nsimplernn_2 (SimpleRNN)          (None, 256)           76544       embedding_5[0][0]                \n____________________________________________________________________________________________________\ndense_2 (Dense)                  (None, 86)            22102       simplernn_2[0][0]                \n====================================================================================================\nTotal params: 102258\n____________________________________________________________________________________________________\n", 
                    "name": "stdout"
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 24, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam())", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 217, 
            "metadata": {
                "hidden": true, 
                "scrolled": false
            }, 
            "source": "model.fit(np.concatenate(xs,axis=1), y, batch_size=64, nb_epoch=8)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/8\n75110/75110 [==============================] - 3s - loss: 2.7939     \nEpoch 2/8\n75110/75110 [==============================] - 3s - loss: 2.2970     \nEpoch 3/8\n75110/75110 [==============================] - 3s - loss: 2.0814     \nEpoch 4/8\n75110/75110 [==============================] - 3s - loss: 1.9416     \nEpoch 5/8\n75110/75110 [==============================] - 3s - loss: 1.8406     \nEpoch 6/8\n75110/75110 [==============================] - 3s - loss: 1.7625     \nEpoch 7/8\n75110/75110 [==============================] - 3s - loss: 1.6960     \nEpoch 8/8\n75110/75110 [==============================] - 3s - loss: 1.6421     \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 217, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7fa18f2c0890>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 222, 
            "metadata": {
                "hidden": true
            }, 
            "source": "def get_next_keras(inp):\n    idxs = [char_indices[c] for c in inp]\n    arrs = np.array(idxs)[np.newaxis,:]\n    p = model.predict(arrs)[0]\n    return chars[np.argmax(p)]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 223, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_next_keras('this is ')", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 223, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "'t'"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 224, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_next_keras('part of ')", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 224, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "'t'"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 225, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_next_keras('queens a')", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 225, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "'n'"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "## Returning sequences", 
            "metadata": {}
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Create inputs", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "To use a sequence model, we can leave our input unchanged - but we have to change our output to a sequence (of course!)\n\nHere, c_out_dat is identical to c_in_dat, but moved across 1 character.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 64, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "#c_in_dat = [[idx[i+n] for i in xrange(0, len(idx)-1-cs, cs)]\n#            for n in range(cs)]\nc_out_dat = [[idx[i+n] for i in xrange(1, len(idx)-cs, cs)]\n            for n in range(cs)]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 65, 
            "metadata": {
                "hidden": true
            }, 
            "source": "ys = [np.stack(c[:-2]) for c in c_out_dat]", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Reading down each column shows one set of inputs and outputs.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 59, 
            "metadata": {
                "hidden": true
            }, 
            "source": "[xs[n][:cs] for n in range(cs)]", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 59, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "[array([40,  1, 33,  2, 72, 67, 73,  2]),\n array([42,  1, 38, 44,  2,  9, 61, 73]),\n array([29, 43, 31, 71, 54,  9, 58, 61]),\n array([30, 45,  2, 74,  2, 76, 67, 58]),\n array([25, 40, 73, 73, 76, 61, 24, 71]),\n array([27, 40, 61, 61, 68, 54,  2, 58]),\n array([29, 39, 54,  2, 66, 73, 33,  2]),\n array([ 1, 43, 73, 62, 54,  2, 72, 67])]"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 60, 
            "metadata": {
                "hidden": true
            }, 
            "source": "[ys[n][:cs] for n in range(cs)]", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 60, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "[array([42,  1, 38, 44,  2,  9, 61, 73]),\n array([29, 43, 31, 71, 54,  9, 58, 61]),\n array([30, 45,  2, 74,  2, 76, 67, 58]),\n array([25, 40, 73, 73, 76, 61, 24, 71]),\n array([27, 40, 61, 61, 68, 54,  2, 58]),\n array([29, 39, 54,  2, 66, 73, 33,  2]),\n array([ 1, 43, 73, 62, 54,  2, 72, 67]),\n array([ 1, 33,  2, 72, 67, 73,  2, 68])]"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Create and train model", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 47, 
            "metadata": {
                "hidden": true
            }, 
            "source": "dense_in = Dense(n_hidden, activation='relu')\ndense_hidden = Dense(n_hidden, activation='relu', init='identity')\ndense_out = Dense(vocab_size, activation='softmax', name='output')", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "We're going to pass a vector of all zeros as our starting point - here's our input layers for that:", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 48, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "inp1 = Input(shape=(n_fac,), name='zeros')\nhidden = dense_in(inp1)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 66, 
            "metadata": {
                "hidden": true
            }, 
            "source": "outs = []\n\nfor i in range(cs):\n    c_dense = dense_in(c_ins[i][1])\n    hidden = dense_hidden(hidden)\n    hidden = merge([c_dense, hidden], mode='sum')\n    # every layer now has an output\n    outs.append(dense_out(hidden))", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 67, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model = Model([inp1] + [c[0] for c in c_ins], outs)\nmodel.compile(loss='sparse_categorical_crossentropy', optimizer=Adam())", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 68, 
            "metadata": {
                "hidden": true
            }, 
            "source": "zeros = np.tile(np.zeros(n_fac), (len(xs[0]),1))\nzeros.shape", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 68, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "(75110, 42)"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 394, 
            "metadata": {
                "hidden": true, 
                "scrolled": true
            }, 
            "source": "model.fit([zeros]+xs, ys, batch_size=64, nb_epoch=12)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "INFO (theano.gof.compilelock): Refreshing lock /home/jhoward/.theano/compiledir_Linux-4.4--generic-x86_64-with-Ubuntu-16.04-xenial-x86_64-2.7.12-64/lock_dir/lock\n", 
                    "name": "stderr"
                }, 
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/12\n75110/75110 [==============================] - 7s - loss: 20.0841 - output_loss_1: 2.7123 - output_loss_2: 2.5681 - output_loss_3: 2.5143 - output_loss_4: 2.4739 - output_loss_5: 2.4675 - output_loss_6: 2.4442 - output_loss_7: 2.4627 - output_loss_8: 2.4410     \nEpoch 2/12\n75110/75110 [==============================] - 7s - loss: 17.8335 - output_loss_1: 2.5124 - output_loss_2: 2.3529 - output_loss_3: 2.2368 - output_loss_4: 2.1686 - output_loss_5: 2.1540 - output_loss_6: 2.1337 - output_loss_7: 2.1520 - output_loss_8: 2.1232     \nEpoch 3/12\n75110/75110 [==============================] - 7s - loss: 17.2340 - output_loss_1: 2.4967 - output_loss_2: 2.3306 - output_loss_3: 2.1766 - output_loss_4: 2.0814 - output_loss_5: 2.0529 - output_loss_6: 2.0291 - output_loss_7: 2.0475 - output_loss_8: 2.0192     \nEpoch 4/12\n75110/75110 [==============================] - 7s - loss: 16.8647 - output_loss_1: 2.4896 - output_loss_2: 2.3218 - output_loss_3: 2.1437 - output_loss_4: 2.0278 - output_loss_5: 1.9901 - output_loss_6: 1.9600 - output_loss_7: 1.9768 - output_loss_8: 1.9549     \nEpoch 5/12\n75110/75110 [==============================] - 7s - loss: 16.6200 - output_loss_1: 2.4858 - output_loss_2: 2.3158 - output_loss_3: 2.1287 - output_loss_4: 1.9941 - output_loss_5: 1.9481 - output_loss_6: 1.9151 - output_loss_7: 1.9276 - output_loss_8: 1.9047     \nEpoch 6/12\n75110/75110 [==============================] - 7s - loss: 16.4396 - output_loss_1: 2.4835 - output_loss_2: 2.3121 - output_loss_3: 2.1148 - output_loss_4: 1.9705 - output_loss_5: 1.9188 - output_loss_6: 1.8774 - output_loss_7: 1.8937 - output_loss_8: 1.8689     \nEpoch 7/12\n75110/75110 [==============================] - 7s - loss: 16.3016 - output_loss_1: 2.4825 - output_loss_2: 2.3090 - output_loss_3: 2.1054 - output_loss_4: 1.9523 - output_loss_5: 1.8957 - output_loss_6: 1.8514 - output_loss_7: 1.8639 - output_loss_8: 1.8414     \nEpoch 8/12\n75110/75110 [==============================] - 7s - loss: 16.1862 - output_loss_1: 2.4807 - output_loss_2: 2.3076 - output_loss_3: 2.0974 - output_loss_4: 1.9391 - output_loss_5: 1.8757 - output_loss_6: 1.8284 - output_loss_7: 1.8413 - output_loss_8: 1.8161     \nEpoch 9/12\n75110/75110 [==============================] - 7s - loss: 16.0887 - output_loss_1: 2.4802 - output_loss_2: 2.3055 - output_loss_3: 2.0913 - output_loss_4: 1.9275 - output_loss_5: 1.8603 - output_loss_6: 1.8101 - output_loss_7: 1.8200 - output_loss_8: 1.7938     \nEpoch 10/12\n75110/75110 [==============================] - 7s - loss: 16.0118 - output_loss_1: 2.4790 - output_loss_2: 2.3038 - output_loss_3: 2.0882 - output_loss_4: 1.9172 - output_loss_5: 1.8458 - output_loss_6: 1.7946 - output_loss_7: 1.8049 - output_loss_8: 1.7782     \nEpoch 11/12\n75110/75110 [==============================] - 7s - loss: 15.9393 - output_loss_1: 2.4784 - output_loss_2: 2.3027 - output_loss_3: 2.0827 - output_loss_4: 1.9095 - output_loss_5: 1.8341 - output_loss_6: 1.7803 - output_loss_7: 1.7885 - output_loss_8: 1.7631     \nEpoch 12/12\n75110/75110 [==============================] - 7s - loss: 15.8785 - output_loss_1: 2.4773 - output_loss_2: 2.3021 - output_loss_3: 2.0788 - output_loss_4: 1.9015 - output_loss_5: 1.8239 - output_loss_6: 1.7680 - output_loss_7: 1.7770 - output_loss_8: 1.7498     \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 394, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7fa168d005d0>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Test model", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 395, 
            "metadata": {
                "hidden": true
            }, 
            "source": "def get_nexts(inp):\n    idxs = [char_indices[c] for c in inp]\n    arrs = [np.array(i)[np.newaxis] for i in idxs]\n    p = model.predict([np.zeros(n_fac)[np.newaxis,:]] + arrs)\n    print(list(inp))\n    return [chars[np.argmax(o)] for o in p]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 396, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_nexts(' this is')", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "[' ', 't', 'h', 'i', 's', ' ', 'i', 's']\n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 396, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "['t', 'h', 'e', 't', ' ', 'c', 's', ' ']"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 397, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_nexts(' part of')", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "[' ', 'p', 'a', 'r', 't', ' ', 'o', 'f']\n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 397, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "['t', 'o', 'r', 't', ' ', 'o', 'f', ' ']"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Sequence model with keras", 
            "metadata": {
                "collapsed": true, 
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 50, 
            "metadata": {
                "hidden": true
            }, 
            "source": "n_hidden, n_fac, cs, vocab_size", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 50, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "(256, 42, 8, 86)"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "To convert our previous keras model into a sequence model, simply add the 'return_sequences=True' parameter, and add TimeDistributed() around our dense layer.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 67, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model=Sequential([\n        Embedding(vocab_size, n_fac, input_length=cs),\n        SimpleRNN(n_hidden, return_sequences=True, activation='relu', inner_init='identity'),\n        TimeDistributed(Dense(vocab_size, activation='softmax')),\n    ])", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 52, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model.summary()", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "____________________________________________________________________________________________________\nLayer (type)                     Output Shape          Param #     Connected to                     \n====================================================================================================\nembedding_6 (Embedding)          (None, 8, 42)         3612        embedding_input_3[0][0]          \n____________________________________________________________________________________________________\nsimplernn_3 (SimpleRNN)          (None, 8, 256)        76544       embedding_6[0][0]                \n____________________________________________________________________________________________________\ntimedistributed_1 (TimeDistribut (None, 8, 86)         22102       simplernn_3[0][0]                \n====================================================================================================\nTotal params: 102258\n____________________________________________________________________________________________________\n", 
                    "name": "stdout"
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 71, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam())", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 82, 
            "metadata": {
                "hidden": true
            }, 
            "source": "xs[0].shape", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 82, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "(75110,)"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 90, 
            "metadata": {
                "hidden": true
            }, 
            "source": "x_rnn=np.stack(np.squeeze(xs), axis=1)\ny_rnn=np.atleast_3d(np.stack(ys, axis=1))", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 91, 
            "metadata": {
                "hidden": true
            }, 
            "source": "x_rnn.shape, y_rnn.shape", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 91, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "((75110, 8), (75110, 8, 1))"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 92, 
            "metadata": {
                "hidden": true, 
                "scrolled": true
            }, 
            "source": "model.fit(x_rnn, y_rnn, batch_size=64, nb_epoch=8)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/8\n75110/75110 [==============================] - 4s - loss: 2.4284     \nEpoch 2/8\n75110/75110 [==============================] - 4s - loss: 2.0006     \nEpoch 3/8\n75110/75110 [==============================] - 4s - loss: 1.8863     \nEpoch 4/8\n75110/75110 [==============================] - 4s - loss: 1.8264     \nEpoch 5/8\n75110/75110 [==============================] - 4s - loss: 1.7882     \nEpoch 6/8\n75110/75110 [==============================] - 4s - loss: 1.7613     \nEpoch 7/8\n75110/75110 [==============================] - 4s - loss: 1.7417     \nEpoch 8/8\n75110/75110 [==============================] - 4s - loss: 1.7258     \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 92, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7f82761cc990>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 93, 
            "metadata": {
                "hidden": true
            }, 
            "source": "def get_nexts_keras(inp):\n    idxs = [char_indices[c] for c in inp]\n    arr = np.array(idxs)[np.newaxis,:]\n    p = model.predict(arr)[0]\n    print(list(inp))\n    return [chars[np.argmax(o)] for o in p]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 94, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_nexts_keras(' this is')", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "[' ', 't', 'h', 'i', 's', ' ', 'i', 's']\n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 94, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "['t', 'h', 'e', 's', ' ', 'i', 's', ' ']"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "### One-hot sequence model with keras", 
            "metadata": {
                "collapsed": true, 
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "This is the keras version of the theano model that we're about to create.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 95, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model=Sequential([\n        SimpleRNN(n_hidden, return_sequences=True, input_shape=(cs, vocab_size),\n                  activation='relu', inner_init='identity'),\n        TimeDistributed(Dense(vocab_size, activation='softmax')),\n    ])\nmodel.compile(loss='categorical_crossentropy', optimizer=Adam())", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 96, 
            "metadata": {
                "hidden": true
            }, 
            "source": "oh_ys = [to_categorical(o, vocab_size) for o in ys]\noh_y_rnn=np.stack(oh_ys, axis=1)\n\noh_xs = [to_categorical(o, vocab_size) for o in xs]\noh_x_rnn=np.stack(oh_xs, axis=1)\n\noh_x_rnn.shape, oh_y_rnn.shape", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 96, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "((75110, 8, 86), (75110, 8, 86))"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 97, 
            "metadata": {
                "hidden": true, 
                "scrolled": true
            }, 
            "source": "model.fit(oh_x_rnn, oh_y_rnn, batch_size=64, nb_epoch=8)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/8\n75110/75110 [==============================] - 4s - loss: 2.4383     \nEpoch 2/8\n75110/75110 [==============================] - 4s - loss: 2.0318     \nEpoch 3/8\n75110/75110 [==============================] - 4s - loss: 1.9195     \nEpoch 4/8\n75110/75110 [==============================] - 4s - loss: 1.8553     \nEpoch 5/8\n75110/75110 [==============================] - 4s - loss: 1.8133     \nEpoch 6/8\n75110/75110 [==============================] - 4s - loss: 1.7829     \nEpoch 7/8\n75110/75110 [==============================] - 4s - loss: 1.7593     \nEpoch 8/8\n75110/75110 [==============================] - 4s - loss: 1.7410     \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 97, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7f8210725c90>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 104, 
            "metadata": {
                "hidden": true
            }, 
            "source": "def get_nexts_oh(inp):\n    idxs = np.array([char_indices[c] for c in inp])\n    arr = to_categorical(idxs, vocab_size)\n    p = model.predict(arr[np.newaxis,:])[0]\n    print(list(inp))\n    return [chars[np.argmax(o)] for o in p]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 84, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_nexts_oh(' this is')", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "[' ', 't', 'h', 'i', 's', ' ', 'i', 's']\n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 84, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "['t', 'h', 'e', 's', ' ', 'i', 's', ' ']"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "## Stateful model with keras", 
            "metadata": {
                "collapsed": true, 
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 290, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "bs=64", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "A stateful model is easy to create (just add \"stateful=True\") but harder to train. We had to add batchnorm and use LSTM to get reasonable results.\n\nWhen using stateful in keras, you have to also add 'batch_input_shape' to the first layer, and fix the batch size there.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 338, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model=Sequential([\n        Embedding(vocab_size, n_fac, input_length=cs, batch_input_shape=(bs,8)),\n        BatchNormalization(),\n        LSTM(n_hidden, return_sequences=True, stateful=True),\n        TimeDistributed(Dense(vocab_size, activation='softmax')),\n    ])", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 339, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam())", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Since we're using a fixed batch shape, we have to ensure our inputs and outputs are a even multiple of the batch size.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 340, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "mx = len(x_rnn)//bs*bs", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 341, 
            "metadata": {
                "hidden": true, 
                "scrolled": false
            }, 
            "source": "model.fit(x_rnn[:mx], y_rnn[:mx], batch_size=bs, nb_epoch=4, shuffle=False)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "INFO (theano.gof.compilelock): Refreshing lock /home/jhoward/.theano/compiledir_Linux-4.4--generic-x86_64-with-Ubuntu-16.04-xenial-x86_64-2.7.12-64/lock_dir/lock\n", 
                    "name": "stderr"
                }, 
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/4\n75072/75072 [==============================] - 13s - loss: 2.2051    \nEpoch 2/4\n75072/75072 [==============================] - 13s - loss: 1.9621    \nEpoch 3/4\n75072/75072 [==============================] - 13s - loss: 1.8893    \nEpoch 4/4\n75072/75072 [==============================] - 13s - loss: 1.8453    \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 341, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7fa16f1d2690>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 342, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "model.optimizer.lr=1e-4", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 343, 
            "metadata": {
                "hidden": true, 
                "scrolled": false
            }, 
            "source": "model.fit(x_rnn[:mx], y_rnn[:mx], batch_size=bs, nb_epoch=4, shuffle=False)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/4\n75072/75072 [==============================] - 13s - loss: 1.8132    \nEpoch 2/4\n75072/75072 [==============================] - 13s - loss: 1.7877    \nEpoch 3/4\n75072/75072 [==============================] - 13s - loss: 1.7663    \nEpoch 4/4\n75072/75072 [==============================] - 13s - loss: 1.7475    \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 343, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7fa1773b8c10>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 344, 
            "metadata": {
                "hidden": true, 
                "scrolled": true
            }, 
            "source": "model.fit(x_rnn[:mx], y_rnn[:mx], batch_size=bs, nb_epoch=4, shuffle=False)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/4\n75072/75072 [==============================] - 13s - loss: 1.7308    \nEpoch 2/4\n75072/75072 [==============================] - 13s - loss: 1.7155    \nEpoch 3/4\n75072/75072 [==============================] - 13s - loss: 1.7014    \nEpoch 4/4\n75072/75072 [==============================] - 13s - loss: 1.6881    \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 344, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7fa1773b8d50>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "## Theano RNN", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 107, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "n_input = vocab_size\nn_output = vocab_size", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Using raw theano, we have to create our weight matrices and bias vectors ourselves - here are the functions we'll use to do so (using glorot initialization).\n\nThe return values are wrapped in `shared()`, which is how we tell theano that it can manage this data (copying it to and from the GPU as necessary).", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 108, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def init_wgts(rows, cols): \n    scale = math.sqrt(2/rows)\n    return shared(normal(scale=scale, size=(rows, cols)).astype(np.float32))\ndef init_bias(rows): \n    return shared(np.zeros(rows, dtype=np.float32))", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "We return the weights and biases together as a tuple. For the hidden weights, we'll use an identity initialization (as recommended by [Hinton](https://arxiv.org/abs/1504.00941).)", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 109, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def wgts_and_bias(n_in, n_out): \n    return init_wgts(n_in, n_out), init_bias(n_out)\ndef id_and_bias(n): \n    return shared(np.eye(n, dtype=np.float32)), init_bias(n)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Theano doesn't actually do any computations until we explicitly compile and evaluate the function (at which point it'll be turned into CUDA code and sent off to the GPU). So our job is to describe the computations that we'll want theano to do - the first step is to tell theano what inputs we'll be providing to our computation:", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 110, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "t_inp = T.matrix('inp')\nt_outp = T.matrix('outp')\nt_h0 = T.vector('h0')\nlr = T.scalar('lr')\n\nall_args = [t_h0, t_inp, t_outp, lr]", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Now we're ready to create our intial weight matrices.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 73, 
            "metadata": {
                "hidden": true
            }, 
            "source": "W_h = id_and_bias(n_hidden)\nW_x = wgts_and_bias(n_input, n_hidden)\nW_y = wgts_and_bias(n_hidden, n_output)\nw_all = list(chain.from_iterable([W_h, W_x, W_y]))", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Theano handles looping by using the [GPU scan](http://http.developer.nvidia.com/GPUGems3/gpugems3_ch39.html) operation. We have to tell theano what to do at each step through the scan - this is the function we'll use, which does a single forward pass for one character:", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 74, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def step(x, h, W_h, b_h, W_x, b_x, W_y, b_y):\n    # Calculate the hidden activations\n    h = nnet.relu(T.dot(x, W_x) + b_x + T.dot(h, W_h) + b_h)\n    # Calculate the output activations\n    y = nnet.softmax(T.dot(h, W_y) + b_y)\n    # Return both (the 'Flatten()' is to work around a theano bug)\n    return h, T.flatten(y, 1)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Now we can provide everything necessary for the scan operation, so we can setup that up - we have to pass in the function to call at each step, the sequence to step through, the initial values of the outputs, and any other arguments to pass to the step function.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 75, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "[v_h, v_y], _ = theano.scan(step, sequences=t_inp, \n                            outputs_info=[t_h0, None], non_sequences=w_all)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "We can now calculate our loss function, and *all* of our gradients, with just a couple of lines of code!", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 76, 
            "metadata": {
                "hidden": true
            }, 
            "source": "error = nnet.categorical_crossentropy(v_y, t_outp).sum()\ng_all = T.grad(error, w_all)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "We even have to show theano how to do SGD - so we set up this dictionary of updates to complete after every forward pass, which apply to standard SGD update rule to every weight.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 77, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def upd_dict(wgts, grads, lr): \n    return OrderedDict({w: w-g*lr for (w,g) in zip(wgts,grads)})\n\nupd = upd_dict(w_all, g_all, lr)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "We're finally ready to compile the function!", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 78, 
            "metadata": {
                "hidden": true
            }, 
            "source": "fn = theano.function(all_args, error, updates=upd, allow_input_downcast=True)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 123, 
            "metadata": {
                "hidden": true
            }, 
            "source": "X = oh_x_rnn\nY = oh_y_rnn\nX.shape, Y.shape", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 123, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "((75110, 8, 86), (75110, 8, 86))"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "To use it, we simply loop through our input data, calling the function compiled above, and printing our progress from time to time.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 86, 
            "metadata": {
                "hidden": true, 
                "scrolled": true
            }, 
            "source": "err=0.0; l_rate=0.01\nfor i in range(len(X)): \n    err+=fn(np.zeros(n_hidden), X[i], Y[i], l_rate)\n    if i % 1000 == 999: \n        print (\"Error:{:.3f}\".format(err/1000))\n        err=0.0", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Error:25.196\nError:21.489\nError:20.900\nError:19.913\nError:18.816\nError:19.202\nError:19.066\nError:18.473\nError:17.942\nError:18.251\nError:17.489\nError:17.570\nError:18.371\nError:17.331\nError:16.807\nError:17.681\nError:17.401\nError:17.136\nError:16.830\nError:16.651\nError:16.518\nError:16.430\nError:16.687\nError:16.161\nError:16.775\nError:16.566\nError:16.053\nError:16.296\nError:16.240\nError:16.454\nError:16.699\nError:16.396\nError:16.644\nError:16.328\nError:15.990\nError:16.644\nError:15.981\nError:16.359\nError:16.042\nError:16.326\nError:15.361\nError:15.690\nError:15.742\nError:16.048\nError:15.955\nError:15.866\nError:15.571\nError:16.069\nError:15.997\nError:16.030\nError:15.230\nError:15.612\nError:14.918\nError:14.821\nError:15.580\nError:15.380\nError:14.650\nError:15.499\nError:15.110\nError:14.972\nError:15.034\nError:15.427\nError:15.236\nError:15.037\nError:14.768\nError:14.781\nError:14.329\nError:14.726\nError:15.229\nError:14.809\nError:15.144\nError:14.755\nError:14.440\nError:14.431\nError:14.464\n", 
                    "name": "stdout"
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 87, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "f_y = theano.function([t_h0, t_inp], v_y, allow_input_downcast=True)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 336, 
            "metadata": {
                "hidden": true
            }, 
            "source": "pred = np.argmax(f_y(np.zeros(n_hidden), X[6]), axis=1)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 337, 
            "metadata": {
                "hidden": true
            }, 
            "source": "act = np.argmax(X[6], axis=1)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 338, 
            "metadata": {
                "hidden": true
            }, 
            "source": "[indices_char[o] for o in act]", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 338, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "['t', 'h', 'e', 'n', '?', ' ', 'I', 's']"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 339, 
            "metadata": {
                "hidden": true
            }, 
            "source": "[indices_char[o] for o in pred]", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 339, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "['h', 'e', ' ', ' ', ' ', 'T', 'n', ' ']"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "## Pure python RNN!", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Set up basic functions", 
            "metadata": {
                "hidden": true, 
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "Now we're going to try to repeat the above theano RNN, using just pure python (and numpy). Which means, we have to do everything ourselves, including defining the basic functions of a neural net! Below are all of the definitions, along with tests to check that they give the same answers as theano. The functions ending in `_d` are the derivatives of each function.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 33, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def sigmoid(x): return 1/(1+np.exp(-x))\ndef sigmoid_d(x): \n    output = sigmoid(x)\n    return output*(1-output)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 34, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def relu(x): return np.maximum(0., x)\ndef relu_d(x): return (x > 0.)*1.", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 35, 
            "metadata": {
                "hidden": true
            }, 
            "source": "relu(np.array([3.,-3.])), relu_d(np.array([3.,-3.]))", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 35, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "(array([ 3.,  0.]), array([ 1.,  0.]))"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 36, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def dist(a,b): return pow(a-b,2)\ndef dist_d(a,b): return 2*(a-b)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 37, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "import pdb", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 38, 
            "metadata": {
                "hidden": true
            }, 
            "source": "eps = 1e-7\ndef x_entropy(pred, actual): \n    return -np.sum(actual * np.log(np.clip(pred, eps, 1-eps)))\ndef x_entropy_d(pred, actual): return -actual/pred", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 39, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def softmax(x): return np.exp(x)/np.exp(x).sum()", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 40, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def softmax_d(x):\n    sm = softmax(x)\n    res = np.expand_dims(-sm,-1)*sm\n    res[np.diag_indices_from(res)] = sm*(1-sm)\n    return res", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 41, 
            "metadata": {
                "hidden": true
            }, 
            "source": "test_preds = np.array([0.2,0.7,0.1])\ntest_actuals = np.array([0.,1.,0.])\nnnet.categorical_crossentropy(test_preds, test_actuals).eval()", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 41, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "array(0.35667494393873245)"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 42, 
            "metadata": {
                "hidden": true
            }, 
            "source": "x_entropy(test_preds, test_actuals)", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 42, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "0.35667494393873245"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 43, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "test_inp = T.dvector()\ntest_out = nnet.categorical_crossentropy(test_inp, test_actuals)\ntest_grad = theano.function([test_inp], T.grad(test_out, test_inp))", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 44, 
            "metadata": {
                "hidden": true
            }, 
            "source": "test_grad(test_preds)", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 44, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "array([-0.    , -1.4286, -0.    ])"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 45, 
            "metadata": {
                "hidden": true
            }, 
            "source": "x_entropy_d(test_preds, test_actuals)", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 45, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "array([-0.    , -1.4286, -0.    ])"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 114, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "pre_pred = random(oh_x_rnn[0][0].shape)\npreds = softmax(pre_pred)\nactual = oh_x_rnn[0][0]", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 119, 
            "metadata": {
                "hidden": true
            }, 
            "source": "np.allclose(softmax_d(pre_pred).dot(x_entropy_d(preds,actual)), preds-actual)", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 119, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "True"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 46, 
            "metadata": {
                "hidden": true
            }, 
            "source": "softmax(test_preds)", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 46, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "array([ 0.2814,  0.464 ,  0.2546])"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 47, 
            "metadata": {
                "hidden": true
            }, 
            "source": "nnet.softmax(test_preds).eval()", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 47, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "array([[ 0.2814,  0.464 ,  0.2546]])"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 48, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "test_out = T.flatten(nnet.softmax(test_inp))", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 49, 
            "metadata": {
                "hidden": true
            }, 
            "source": "test_grad = theano.function([test_inp], theano.gradient.jacobian(test_out, test_inp))", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 50, 
            "metadata": {
                "hidden": true
            }, 
            "source": "test_grad(test_preds)", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 50, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "array([[ 0.2022, -0.1306, -0.0717],\n       [-0.1306,  0.2487, -0.1181],\n       [-0.0717, -0.1181,  0.1898]])"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 51, 
            "metadata": {
                "hidden": true, 
                "scrolled": true
            }, 
            "source": "softmax_d(test_preds)", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 51, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "array([[ 0.2022, -0.1306, -0.0717],\n       [-0.1306,  0.2487, -0.1181],\n       [-0.0717, -0.1181,  0.1898]])"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 76, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "act=relu\nact_d = relu_d", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 77, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "loss=x_entropy\nloss_d=x_entropy_d", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "We also have to define our own scan function. Since we're not worrying about running things in parallel, it's very simple to implement:", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 54, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def scan(fn, start, seq):\n    res = []\n    prev = start\n    for s in seq:\n        app = fn(prev, s)\n        res.append(app)\n        prev = app\n    return res", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "...for instance, `scan` on `+` is the cumulative sum.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 55, 
            "metadata": {
                "hidden": true
            }, 
            "source": "scan(lambda prev,curr: prev+curr, 0, range(5))", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 55, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "[0, 1, 3, 6, 10]"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Set up training", 
            "metadata": {
                "hidden": true, 
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "Let's now build the functions to do the forward and backward passes of our RNN. First, define our data and shape.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 65, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "inp = oh_x_rnn\noutp = oh_y_rnn\nn_input = vocab_size\nn_output = vocab_size", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 83, 
            "metadata": {
                "hidden": true
            }, 
            "source": "inp.shape, outp.shape", 
            "outputs": [
                {
                    "output_type": "execute_result", 
                    "execution_count": 83, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "((75110, 8, 86), (75110, 8, 86))"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "Here's the function to do a single forward pass of an RNN, for a single character.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 79, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def one_char(prev, item):\n    # Previous state\n    tot_loss, pre_hidden, pre_pred, hidden, ypred = prev\n    # Current inputs and output\n    x, y = item\n    pre_hidden = np.dot(x,w_x) + np.dot(hidden,w_h)\n    hidden = act(pre_hidden)\n    pre_pred = np.dot(hidden,w_y)\n    ypred = softmax(pre_pred)\n    return (\n        # Keep track of loss so we can report it\n        tot_loss+loss(ypred, y),\n        # Used in backprop\n        pre_hidden, pre_pred, \n        # Used in next iteration\n        hidden, \n        # To provide predictions\n        ypred)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "We use `scan` to apply the above to a whole sequence of characters.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 80, 
            "metadata": {
                "hidden": true
            }, 
            "source": "def get_chars(n): return zip(inp[n], outp[n])\ndef one_fwd(n): return scan(one_char, (0,0,0,np.zeros(n_hidden),0), get_chars(n))", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Now we can define the backward step. We use a loop to go through every element of the sequence. The derivatives are applying the chain rule to each step, and accumulating the gradients across the sequence.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 82, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "# \"Columnify\" a vector\ndef col(x): return x[:,newaxis]\n\ndef one_bkwd(args, n):\n    global w_x,w_y,w_h\n\n    i=inp[n]  # 8x86\n    o=outp[n] # 8x86\n    d_pre_hidden = np.zeros(n_hidden) # 256\n    for p in reversed(range(len(i))):\n        totloss, pre_hidden, pre_pred, hidden, ypred = args[p]\n        x=i[p] # 86\n        y=o[p] # 86\n        d_pre_pred = softmax_d(pre_pred).dot(loss_d(ypred,y))  # 86\n        d_pre_hidden = (np.dot(d_pre_hidden, w_h.T) \n                        + np.dot(d_pre_pred,w_y.T)) * act_d(pre_hidden) # 256\n\n        # d(loss)/d(w_y) = d(loss)/d(pre_pred) * d(pre_pred)/d(w_y)\n        w_y -= col(hidden) * d_pre_pred * alpha\n        # d(loss)/d(w_h) = d(loss)/d(pre_hidden[p-1]) * d(pre_hidden[p-1])/d(w_h)\n        if (p>0): w_h -= args[p-1][3].dot(d_pre_hidden) * alpha\n        w_x -= col(x)*d_pre_hidden * alpha\n    return d_pre_hidden", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Now we can set up our initial weight matrices. Note that we're not using bias at all in this example, in order to keep things simpler.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 126, 
            "metadata": {
                "hidden": true
            }, 
            "source": "scale=math.sqrt(2./n_input)\nw_x = normal(scale=scale, size=(n_input,n_hidden))\nw_y = normal(scale=scale, size=(n_hidden, n_output))\nw_h = np.eye(n_hidden, dtype=np.float32)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Our loop looks much like the theano loop in the previous section, except that we have to call the backwards step ourselves.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 127, 
            "metadata": {
                "hidden": true, 
                "scrolled": true
            }, 
            "source": "overallError=0\nalpha=0.0001\nfor n in range(10000):\n    res = one_fwd(n)\n    overallError+=res[-1][0]\n    deriv = one_bkwd(res, n)\n    if(n % 1000 == 999):\n        print (\"Error:{:.4f}; Gradient:{:.5f}\".format(\n                overallError/1000, np.linalg.norm(deriv)))\n        overallError=0", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Error:35.2380; Gradient:2.90002\nError:32.9176; Gradient:2.71170\nError:31.0649; Gradient:4.14135\nError:29.9798; Gradient:3.40467\nError:29.2453; Gradient:3.79049\nError:29.0070; Gradient:3.39826\nError:28.2358; Gradient:4.30422\nError:28.0086; Gradient:2.92011\nError:27.6885; Gradient:4.03503\nError:27.6905; Gradient:3.18526\n", 
                    "name": "stdout"
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "## Keras GRU", 
            "metadata": {
                "collapsed": true, 
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "Identical to the last keras rnn, but a GRU!", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 101, 
            "metadata": {
                "hidden": true
            }, 
            "source": "model=Sequential([\n        GRU(n_hidden, return_sequences=True, input_shape=(cs, vocab_size),\n                  activation='relu', inner_init='identity'),\n        TimeDistributed(Dense(vocab_size, activation='softmax')),\n    ])\nmodel.compile(loss='categorical_crossentropy', optimizer=Adam())", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 102, 
            "metadata": {
                "hidden": true, 
                "scrolled": true
            }, 
            "source": "model.fit(oh_x_rnn, oh_y_rnn, batch_size=64, nb_epoch=8)", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Epoch 1/8\n75110/75110 [==============================] - 9s - loss: 2.3991     \nEpoch 2/8\n75110/75110 [==============================] - 9s - loss: 1.9818     \nEpoch 3/8\n75110/75110 [==============================] - 9s - loss: 1.8704     \nEpoch 4/8\n75110/75110 [==============================] - 9s - loss: 1.8070     \nEpoch 5/8\n75110/75110 [==============================] - 9s - loss: 1.7653     \nEpoch 6/8\n75110/75110 [==============================] - 10s - loss: 1.7346    \nEpoch 7/8\n75110/75110 [==============================] - 9s - loss: 1.7108     \nEpoch 8/8\n75110/75110 [==============================] - 9s - loss: 1.6918     \n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 102, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "<keras.callbacks.History at 0x7f820e8bae50>"
                    }
                }
            ]
        }, 
        {
            "cell_type": "code", 
            "execution_count": 105, 
            "metadata": {
                "hidden": true
            }, 
            "source": "get_nexts_oh(' this is')", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "[' ', 't', 'h', 'i', 's', ' ', 'i', 's']\n", 
                    "name": "stdout"
                }, 
                {
                    "output_type": "execute_result", 
                    "execution_count": 105, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "['t', 'h', 'e', 's', ' ', 'i', 's', ' ']"
                    }
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "## Theano GRU", 
            "metadata": {}
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Separate weights", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "The theano GRU looks just like the simple theano RNN, except for the use of the reset and update gates. Each of these gates requires its own hidden and input weights, so we add those to our weight matrices.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 139, 
            "metadata": {
                "hidden": true
            }, 
            "source": "W_h = id_and_bias(n_hidden)\nW_x = init_wgts(n_input, n_hidden)\nW_y = wgts_and_bias(n_hidden, n_output)\nrW_h = init_wgts(n_hidden, n_hidden)\nrW_x = wgts_and_bias(n_input, n_hidden)\nuW_h = init_wgts(n_hidden, n_hidden)\nuW_x = wgts_and_bias(n_input, n_hidden)\nw_all = list(chain.from_iterable([W_h, W_y, uW_x, rW_x]))\nw_all.extend([W_x, uW_h, rW_h])", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Here's the definition of a gate - it's just a sigmoid applied to the addition of the dot products of the input vectors.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 140, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def gate(x, h, W_h, W_x, b_x):\n    return nnet.sigmoid(T.dot(x, W_x) + b_x + T.dot(h, W_h))", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Our step is nearly identical to before, except that we multiply our hidden state by our reset gate, and we update our hidden state based on the update gate.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 146, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def step(x, h, W_h, b_h, W_y, b_y, uW_x, ub_x, rW_x, rb_x, W_x, uW_h, rW_h):\n    reset = gate(x, h, rW_h, rW_x, rb_x)\n    update = gate(x, h, uW_h, uW_x, ub_x)\n    h_new = gate(x, h * reset, W_h, W_x, b_h)\n    h = update*h + (1-update)*h_new\n    y = nnet.softmax(T.dot(h, W_y) + b_y)\n    return h, T.flatten(y, 1)", 
            "outputs": []
        }, 
        {
            "cell_type": "markdown", 
            "source": "Everything from here on is identical to our simple RNN in theano.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 147, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "[v_h, v_y], _ = theano.scan(step, sequences=t_inp, \n                            outputs_info=[t_h0, None], non_sequences=w_all)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 148, 
            "metadata": {
                "hidden": true
            }, 
            "source": "error = nnet.categorical_crossentropy(v_y, t_outp).sum()\ng_all = T.grad(error, w_all)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 149, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "upd = upd_dict(w_all, g_all, lr)\nfn = theano.function(all_args, error, updates=upd, allow_input_downcast=True)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 150, 
            "metadata": {
                "hidden": true, 
                "scrolled": true
            }, 
            "source": "err=0.0; l_rate=0.1\nfor i in range(len(X)): \n    err+=fn(np.zeros(n_hidden), X[i], Y[i], l_rate)\n    if i % 1000 == 999: \n        l_rate *= 0.95\n        print (\"Error:{:.2f}\".format(err/1000))\n        err=0.0", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Error:21.89\nError:20.52\nError:20.55\nError:19.87\nError:19.01\nError:19.58\nError:19.45\nError:18.93\nError:18.51\nError:18.75\nError:18.16\nError:18.18\nError:18.90\nError:18.03\nError:17.50\nError:18.39\nError:18.11\nError:17.92\nError:17.50\nError:17.38\nError:17.17\nError:17.11\nError:17.49\nError:17.04\nError:17.40\nError:17.23\nError:16.83\nError:16.97\nError:17.02\nError:17.25\nError:17.46\nError:17.18\nError:17.41\nError:17.07\nError:16.78\nError:17.39\nError:16.68\nError:17.23\nError:16.75\nError:16.96\n", 
                    "name": "stdout"
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "### Combined weights", 
            "metadata": {
                "heading_collapsed": true
            }
        }, 
        {
            "cell_type": "markdown", 
            "source": "We can make the previous section simpler and faster by concatenating the hidden and input matrices and inputs together. We're not going to step through this cell by cell - you'll see it's identical to the previous section except for this concatenation.", 
            "metadata": {
                "hidden": true
            }
        }, 
        {
            "cell_type": "code", 
            "execution_count": 186, 
            "metadata": {
                "hidden": true
            }, 
            "source": "W = (shared(np.concatenate([np.eye(n_hidden), normal(size=(n_input, n_hidden))])\n            .astype(np.float32)), init_bias(n_hidden))\n\nrW = wgts_and_bias(n_input+n_hidden, n_hidden)\nuW = wgts_and_bias(n_input+n_hidden, n_hidden)\nW_y = wgts_and_bias(n_hidden, n_output)\nw_all = list(chain.from_iterable([W, W_y, uW, rW]))", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 187, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def gate(m, W, b): return nnet.sigmoid(T.dot(m, W) + b)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 188, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def step(x, h, W, b, W_y, b_y, uW, ub, rW, rb):\n    m = T.concatenate([h, x])\n    reset = gate(m, rW, rb)\n    update = gate(m, uW, ub)\n    m = T.concatenate([h*reset, x])\n    h_new = gate(m, W, b)\n    h = update*h + (1-update)*h_new\n    y = nnet.softmax(T.dot(h, W_y) + b_y)\n    return h, T.flatten(y, 1)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 189, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "[v_h, v_y], _ = theano.scan(step, sequences=t_inp, \n                            outputs_info=[t_h0, None], non_sequences=w_all)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 190, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "def upd_dict(wgts, grads, lr): \n    return OrderedDict({w: w-g*lr for (w,g) in zip(wgts,grads)})", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 191, 
            "metadata": {
                "hidden": true
            }, 
            "source": "error = nnet.categorical_crossentropy(v_y, t_outp).sum()\ng_all = T.grad(error, w_all)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 192, 
            "metadata": {
                "collapsed": true, 
                "hidden": true
            }, 
            "source": "upd = upd_dict(w_all, g_all, lr)\nfn = theano.function(all_args, error, updates=upd, allow_input_downcast=True)", 
            "outputs": []
        }, 
        {
            "cell_type": "code", 
            "execution_count": 193, 
            "metadata": {
                "hidden": true
            }, 
            "source": "err=0.0; l_rate=0.01\nfor i in range(len(X)): \n    err+=fn(np.zeros(n_hidden), X[i], Y[i], l_rate)\n    if i % 1000 == 999: \n        print (\"Error:{:.2f}\".format(err/1000))\n        err=0.0", 
            "outputs": [
                {
                    "output_type": "stream", 
                    "text": "Error:24.71\nError:22.16\nError:21.99\nError:21.26\nError:20.44\nError:20.97\nError:20.69\nError:20.15\nError:19.91\nError:20.26\nError:19.54\nError:19.64\nError:20.26\nError:19.49\nError:18.95\nError:19.94\nError:19.71\nError:19.56\nError:18.95\nError:18.78\nError:18.46\nError:18.50\nError:19.02\nError:18.45\nError:18.72\nError:18.50\nError:18.27\nError:18.31\nError:18.29\nError:18.46\nError:18.75\nError:18.33\nError:18.58\nError:18.24\nError:17.95\nError:18.53\nError:17.82\nError:18.36\nError:17.87\nError:18.01\nError:17.32\nError:17.70\nError:17.54\nError:17.87\nError:17.79\nError:17.84\nError:17.59\nError:17.78\nError:17.65\nError:17.75\nError:17.09\nError:17.31\nError:16.71\nError:16.77\nError:17.38\nError:17.22\nError:16.70\nError:17.28\nError:17.00\nError:16.85\nError:16.62\nError:17.06\nError:16.88\nError:16.71\nError:16.46\nError:16.49\nError:16.23\nError:16.44\nError:16.98\nError:16.37\nError:16.79\nError:16.32\nError:16.12\nError:16.13\nError:16.11\n", 
                    "name": "stdout"
                }
            ]
        }, 
        {
            "cell_type": "markdown", 
            "source": "### End", 
            "metadata": {}
        }
    ], 
    "nbformat": 4, 
    "nbformat_minor": 1, 
    "metadata": {
        "nav_menu": {}, 
        "kernelspec": {
            "language": "python", 
            "display_name": "Python 3.5 (Experimental) with Spark 2.1", 
            "name": "python3-spark21"
        }, 
        "toc": {
            "sideBar": true, 
            "threshold": 6, 
            "navigate_menu": true, 
            "toc_section_display": "block", 
            "number_sections": true, 
            "toc_window_display": false, 
            "toc_cell": true
        }, 
        "language_info": {
            "pygments_lexer": "ipython3", 
            "mimetype": "text/x-python", 
            "nbconvert_exporter": "python", 
            "name": "python", 
            "file_extension": ".py", 
            "version": "3.5.2", 
            "codemirror_mode": {
                "version": 3, 
                "name": "ipython"
            }
        }
    }
}