hierarchical-attention-network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.6.0'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "tf.set_random_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HierarchicalAttentionNetwork(object):\n",
    "    def __init__(self, embedding_matrix, num_class, hidden_dim = None):\n",
    "        super(HierarchicalAttentionNetwork, self).__init__()\n",
    "        self._embedding_matrix = embedding_matrix\n",
    "        self._num_class = num_class\n",
    "        self._hidden_dim = \\\n",
    "            hidden_dim if hidden_dim is not None else self._embedding_matrix.shape[1]\n",
    "    def _make_rnn_cell(self):\n",
    "        cell = tf.nn.rnn_cell.GRUCell(num_units=self._hidden_dim)\n",
    "        return cell\n",
    "    def _make_graph_batch(self, graph):\n",
    "        with graph.as_default():\n",
    "            words = tf.placeholder(tf.int32, [None, None, None], name='words')\n",
    "            words_length = tf.placeholder(tf.int32, [None, None], name='words_length')\n",
    "            sentences_length = tf.placeholder(tf.int32, [None], name='sentences_length')\n",
    "            labels = tf.placeholder(tf.int32, [None], name='labels')\n",
    "            init_state = tf.placeholder(tf.float32, [None, None, self._hidden_dim * 2], name='init_state')\n",
    "            \n",
    "            with tf.variable_scope('embeddings'):\n",
    "                embedding = \\\n",
    "                    tf.get_variable('parameter', \n",
    "                                    shape=self._embedding_matrix.shape, \n",
    "                                    initializer=tf.constant_initializer(embedding_matrix), \n",
    "                                    dtype=tf.float32, trainable=True)\n",
    "                embedded  = tf.nn.embedding_lookup(embedding, words, name='lookup')\n",
    "            with tf.variable_scope('words_lstm'):\n",
    "                cell_fw = self._make_rnn_cell()\n",
    "                cell_bw = self._make_rnn_cell()\n",
    "                def step(state, inp):\n",
    "                    data = inp[0]\n",
    "                    length = inp[1]\n",
    "                    fw_state = tf.split(tf.map_fn(lambda x: x[0][x[1], :], (state, length - 1), dtype=tf.float32), 2, axis=1)[0]\n",
    "                    bw_state = tf.split(state[:, 0, :], 2, axis=1)[1]\n",
    "                    (outputs_fw, outputs_bw), _ = \\\n",
    "                        tf.nn.bidirectional_dynamic_rnn(\n",
    "                            cell_fw, cell_bw, data, sequence_length=length, \n",
    "                            initial_state_fw=fw_state, initial_state_bw=bw_state, dtype=tf.float32\n",
    "                        )\n",
    "                    outputs = tf.concat([outputs_fw, outputs_bw], axis=2)\n",
    "                    return outputs\n",
    "                embedded_t = tf.transpose(embedded, perm=[1, 0, 2, 3])\n",
    "                words_length_t = tf.transpose(words_length)\n",
    "                outputs = tf.scan(step, (embedded_t, words_length_t), initializer=init_state)\n",
    "                outputs = tf.transpose(outputs, perm=[1, 0, 2, 3])\n",
    "#                 def fn(inp):\n",
    "#                     (outputs_fw, outputs_bw), _ = \\\n",
    "#                         tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, inp[0], sequence_length=inp[1], dtype=tf.float32)\n",
    "#                     return tf.concat([outputs_fw, outputs_bw], axis=2)\n",
    "#                 outputs = tf.map_fn(fn, (embedded, words_length), dtype=tf.float32)\n",
    "            with tf.variable_scope('words_attention'):\n",
    "                hidden = \\\n",
    "                    tf.layers.dense(outputs, units=self._hidden_dim * 2, \n",
    "                                    activation=tf.nn.tanh, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))\n",
    "                attention = \\\n",
    "                    tf.layers.dense(outputs, units=1, \n",
    "                                    activation=None, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))\n",
    "                attention = tf.transpose(tf.nn.softmax(tf.transpose(attention, perm=[0, 1, 3, 2])), perm=[0, 1, 3, 2])\n",
    "            outputs = tf.reduce_sum(outputs * attention, axis=2)\n",
    "            with tf.variable_scope('sentence_lstm'):\n",
    "                cell_fw = self._make_rnn_cell()\n",
    "                cell_bw = self._make_rnn_cell()\n",
    "                (outputs_fw, outputs_bw), _ = \\\n",
    "                    tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, outputs, sequence_length=sentences_length, dtype=tf.float32)\n",
    "            outputs = tf.concat([outputs_fw, outputs_bw], axis=2)\n",
    "            with tf.variable_scope('sentence_attention'):\n",
    "                hidden = \\\n",
    "                    tf.layers.dense(outputs, units=self._hidden_dim * 2, \n",
    "                                    activation=tf.nn.tanh, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))\n",
    "                attention = \\\n",
    "                    tf.layers.dense(hidden, units=1, \n",
    "                                    activation=None, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))\n",
    "                attention = tf.transpose(tf.nn.softmax(tf.transpose(attention, perm=[0, 2, 1])), perm=[0, 2, 1])\n",
    "            outputs = tf.reduce_sum(outputs * attention, axis=1)\n",
    "            logits = tf.layers.dense(outputs, units=self._num_class, activation=None)\n",
    "            loss = tf.log(tf.boolean_mask(tf.nn.softmax(logits), tf.one_hot(labels, self._num_class, on_value=True, off_value=False)))\n",
    "            loss = -tf.reduce_sum(loss)\n",
    "            training_op = tf.train.AdamOptimizer(learning_rate=0.01).minimize(loss)\n",
    "            return words, words_length, sentences_length, labels, init_state, logits, loss, training_op\n",
    "    def _make_graph(self, graph):\n",
    "        with graph.as_default():\n",
    "            words = tf.placeholder(tf.int32, [None, None], name='words')\n",
    "            length = tf.placeholder(tf.int32, [None], name='length')\n",
    "            labels = tf.placeholder(tf.int32, (), name='labels')\n",
    "            init_state = tf.placeholder(tf.float32, [None, self._hidden_dim * 2], name='initial_state')\n",
    "            \n",
    "            with tf.variable_scope('embeddings'):\n",
    "                embedding = \\\n",
    "                    tf.get_variable('parameter', \n",
    "                                    shape=self._embedding_matrix.shape, \n",
    "                                    initializer=tf.constant_initializer(embedding_matrix), \n",
    "                                    dtype=tf.float32, trainable=True)\n",
    "                embedded  = tf.nn.embedding_lookup(embedding, words, name='lookup')\n",
    "            with tf.variable_scope('words_lstm'):\n",
    "                cell_fw = self._make_rnn_cell()\n",
    "                cell_bw = self._make_rnn_cell()\n",
    "                def step(state, inp):\n",
    "                    data = tf.expand_dims(inp[0], axis=0)\n",
    "                    length = tf.expand_dims(inp[1], axis=0)\n",
    "                    fw_state = tf.split(state[inp[1] - 1, :], 2)[0]\n",
    "                    bw_state = tf.split(state[0, :], 2)[1]\n",
    "                    fw_state = tf.expand_dims(fw_state, axis=0)\n",
    "                    bw_state = tf.expand_dims(bw_state, axis=0)\n",
    "                    (outputs_fw, outputs_bw), _ = \\\n",
    "                        tf.nn.bidirectional_dynamic_rnn(\n",
    "                            cell_fw, cell_bw, data, sequence_length=length, \n",
    "                            initial_state_fw=fw_state, initial_state_bw=bw_state, dtype=tf.float32\n",
    "                        )\n",
    "                    outputs = tf.squeeze(tf.concat([outputs_fw, outputs_bw], axis=2), axis=[0])\n",
    "                    return outputs\n",
    "                outputs = tf.scan(step, (embedded, length), initializer=init_state)\n",
    "            with tf.variable_scope('words_attention'):\n",
    "                hidden = \\\n",
    "                    tf.layers.dense(outputs, units=self._hidden_dim * 2, \n",
    "                                    activation=tf.nn.tanh, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))\n",
    "                attention = \\\n",
    "                    tf.layers.dense(outputs, units=1, \n",
    "                                    activation=None, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))\n",
    "                attention = tf.transpose(tf.nn.softmax(tf.transpose(attention, perm=[0, 2, 1])), perm=[0, 2, 1])\n",
    "            sentence_embedding = tf.reduce_sum(outputs * attention, axis=1)\n",
    "            sentence_embedding = tf.expand_dims(sentence_embedding, axis=0)\n",
    "            \n",
    "            with tf.variable_scope('sentence_lstm'):\n",
    "                cell_fw = self._make_rnn_cell()\n",
    "                cell_bw = self._make_rnn_cell()\n",
    "                (outputs_fw, outputs_bw), _ = \\\n",
    "                    tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, sentence_embedding, dtype=tf.float32)\n",
    "            outputs = tf.squeeze(tf.concat([outputs_fw, outputs_bw], axis=2), axis=[0])\n",
    "            with tf.variable_scope('sentence_attention'):\n",
    "                hidden = \\\n",
    "                    tf.layers.dense(outputs, units=self._hidden_dim * 2, \n",
    "                                    activation=tf.nn.tanh, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))\n",
    "                attention = \\\n",
    "                    tf.layers.dense(hidden, units=1, \n",
    "                                    activation=None, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))\n",
    "                attention = tf.transpose(tf.nn.softmax(tf.transpose(attention)))                \n",
    "            outputs = tf.reduce_sum(outputs * attention, axis=0)\n",
    "            outputs = tf.expand_dims(outputs, axis=0)\n",
    "            logits = tf.layers.dense(outputs, units=self._num_class, activation=None)\n",
    "            loss = -tf.log(tf.nn.softmax(logits)[:, labels], name='loss')\n",
    "#             loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=tf.expand_dims(labels, axis=0))\n",
    "#             loss = tf.squeeze(loss, axis=[0])\n",
    "            training_op = tf.train.AdamOptimizer(learning_rate=0.01).minimize(loss)\n",
    "            return words, length, labels, init_state, logits, loss, training_op            \n",
    "    def evaluate_batch(self, train_words_seq, train_length_seq, train_labels_seq, eval_words_seq, eval_length_seq, eval_labels_seq, num_epochs=20, batch_size=16, model=None):\n",
    "        def prepare_data_batch(words_seq, length_seq, label_seq):\n",
    "            max_words_len = max([max([w.shape[0] for w in ws]) for ws in words_seq])\n",
    "            max_sents_len = max([ws.shape[0] for ws in words_seq])\n",
    "            def pad(array):\n",
    "                expand = np.zeros((max_sents_len, max_words_len), dtype=np.int32)\n",
    "                expand[:array.shape[0], :array.shape[1]] = array\n",
    "                return expand\n",
    "            words = np.stack([pad(ws) for ws in words_seq])    \n",
    "            words_length = tf.keras.preprocessing.sequence.pad_sequences(length_seq, padding='post', value=0.0)\n",
    "            sentence_length = np.array([w.shape[0] for w in words_seq])\n",
    "            labels = np.array(label_seq)\n",
    "            return words, words_length, sentence_length, labels        \n",
    "        \n",
    "        graph = tf.Graph()\n",
    "        words, words_length, sents_length, labels, init_state, logits, loss, training_op = \\\n",
    "            self._make_graph_batch(graph)\n",
    "        \n",
    "        with graph.as_default():\n",
    "            predictions = \\\n",
    "                tf.to_float(tf.nn.in_top_k(logits, labels, k=1))\n",
    "        \n",
    "        # pre calculate the padded training sequence\n",
    "        train_words_batch, train_words_length_batch, train_sents_length_batch, train_labels_batch = \\\n",
    "            prepare_data_batch(train_words_seq, train_length_seq, train_labels_seq)\n",
    "        init_state_batch = np.zeros((train_words_batch.shape[0], train_words_batch.shape[2], 2 * self._hidden_dim), dtype=np.float32)\n",
    "        train_feed_dict = \\\n",
    "            { \n",
    "                words : train_words_batch, \n",
    "                words_length : train_words_length_batch, \n",
    "                sents_length : train_sents_length_batch, \n",
    "                labels : train_labels_batch,\n",
    "                init_state : init_state_batch\n",
    "            } \n",
    "        # pre calculate the padded eval sequence\n",
    "        eval_words_batch, eval_words_length_batch, eval_sents_length_batch, eval_labels_batch = \\\n",
    "            prepare_data_batch(eval_words_seq, eval_length_seq, eval_labels_seq)\n",
    "        init_state_val = np.zeros((eval_words_batch.shape[0], eval_words_batch.shape[2], 2 * self._hidden_dim), dtype=np.float32)\n",
    "        eval_feed_dict = \\\n",
    "            { \n",
    "                words : eval_words_batch, \n",
    "                words_length : eval_words_length_batch, \n",
    "                sents_length : eval_sents_length_batch, \n",
    "                labels : eval_labels_batch,\n",
    "                init_state : init_state_val\n",
    "            }\n",
    "        \n",
    "        with tf.Session(graph=graph) as sess:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            for epoch in range(num_epochs):\n",
    "                st = None\n",
    "                en = None\n",
    "                while True:\n",
    "                    if st is None: st = 0\n",
    "                    else: st += batch_size\n",
    "                    if en is None: en = batch_size\n",
    "                    else: en += batch_size\n",
    "                        \n",
    "                    if not train_words_seq[st:en]: break\n",
    "                    train_words_batch, train_words_length_batch, train_sents_length_batch, train_labels_batch = \\\n",
    "                        prepare_data_batch(train_words_seq[st:en], train_length_seq[st:en], train_labels_seq[st:en])\n",
    "                    init_state_batch = np.zeros((train_words_batch.shape[0], train_words_batch.shape[2], 2 * self._hidden_dim), dtype=np.float32)\n",
    "                    feed_dict = \\\n",
    "                        { \n",
    "                            words : train_words_batch, \n",
    "                            words_length : train_words_length_batch, \n",
    "                            sents_length : train_sents_length_batch, \n",
    "                            labels : train_labels_batch,\n",
    "                            init_state : init_state_batch\n",
    "                        }\n",
    "                    sess.run(training_op, feed_dict=feed_dict)\n",
    "                loss_val, pred_val = sess.run([loss, predictions], feed_dict=train_feed_dict)\n",
    "                eval_pred_val = sess.run(predictions, feed_dict=eval_feed_dict)                \n",
    "                print('Epoch [%d/%d], Loss %.3f, accuracy %.3f, eval accuracy %.3f' % (epoch + 1, num_epochs, loss_val, np.mean(pred_val), np.mean(eval_pred_val)))\n",
    "            if model is not None:\n",
    "                with graph().as_default():\n",
    "                    tf.train.Saver().save(sess, model)\n",
    "    def evaluate(self, train_words_seq, train_length_seq, train_labels_seq, eval_words_seq, eval_length_seq, eval_labels_seq, num_epochs=20, model=None):\n",
    "        graph = tf.Graph()\n",
    "#         words, length, labels, logits, loss, training_op = self._make_graph(graph)\n",
    "        words, length, labels, init_state, logits, loss, training_op = self._make_graph(graph)\n",
    "        with graph.as_default():\n",
    "            prediction = \\\n",
    "                tf.to_float(tf.nn.in_top_k(logits, tf.expand_dims(labels, axis=0), k=1))\n",
    "        with tf.Session(graph=graph) as sess:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            for epoch in range(num_epochs):\n",
    "                for words_val, length_val, labels_val in zip(train_words_seq, train_length_seq, train_labels_seq):\n",
    "                    init_state_val = np.zeros((words_val.shape[-1], 2 * self._hidden_dim), dtype=np.float32)\n",
    "                    feed_dict = { words : words_val, length : length_val, labels : labels_val, init_state : init_state_val }\n",
    "                    sess.run(training_op, feed_dict=feed_dict)\n",
    "                losses = []\n",
    "                preds = []\n",
    "                for words_val, length_val, labels_val in zip(train_words_seq, train_length_seq, train_labels_seq):\n",
    "                    init_state_val = np.zeros((words_val.shape[-1], 2 * self._hidden_dim), dtype=np.float32)\n",
    "                    feed_dict = { words : words_val, length : length_val, labels : labels_val, init_state : init_state_val }\n",
    "                    loss_val, pred_val = sess.run([loss, prediction], feed_dict=feed_dict)\n",
    "                    losses.append(loss_val)\n",
    "                    preds.append(pred_val)\n",
    "                eval_preds = []\n",
    "                for words_val, length_val, labels_val in zip(eval_words_seq, eval_length_seq, eval_labels_seq):\n",
    "                    init_state_val = np.zeros((words_val.shape[-1], 2 * self._hidden_dim), dtype=np.float32)\n",
    "                    feed_dict = { words : words_val, length : length_val, labels : labels_val, init_state : init_state_val }\n",
    "                    pred_val = sess.run(prediction, feed_dict=feed_dict)\n",
    "                    eval_preds.append(pred_val)\n",
    "                print('Epoch [%d/%d], Loss %.3f, accuracy %.3f, eval accuracy %.3f' % (epoch + 1, num_epochs, np.sum(losses), np.mean(preds), np.mean(eval_preds)))\n",
    "            if model is not None:\n",
    "                with graph.as_default():\n",
    "                    tf.train.Saver().save(sess, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_data shape:  11314\n",
      "test_data shape:  7532\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "import spacy\n",
    "# from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "# from gensim.models import KeyedVectors\n",
    "\n",
    "train_data = fetch_20newsgroups(subset='train', remove=('headers', 'footers', 'quotes'))\n",
    "test_data  = fetch_20newsgroups(subset='test', remove=('headers', 'footers', 'quotes'))\n",
    "nlp = spacy.load('en')\n",
    "\n",
    "# glove2word2vec('data/glove.6B.100d.txt', 'data/glove.6B.100d.converted.txt')\n",
    "# embeddings = KeyedVectors.load_word2vec_format('data/glove.6B.100d.converted.txt')\n",
    "\n",
    "print('train_data shape: ', train_data.target.shape[0])\n",
    "print('test_data shape: ', test_data.target.shape[0])\n",
    "# print('embeddings shape: ', embeddings.syn0.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_to_ix = dict([(n, i) for i, n in enumerate(train_data.target_names)])\n",
    "\n",
    "train_data_size = 1000\n",
    "test_data_size = 100\n",
    "\n",
    "batch_size = train_data_size / 10\n",
    "\n",
    "def prepare_embeddings(data):\n",
    "    embeddings = dict()\n",
    "    for i, doc in enumerate(data):\n",
    "        for token in nlp(doc):\n",
    "            if token.is_punct: continue\n",
    "            w = token.lower_.strip()\n",
    "            if not w: continue\n",
    "            embeddings[w] = len(embeddings)\n",
    "#         if (i + 1) % batch_size == 0: print('processed %d documents' % (i + 1))\n",
    "    return embeddings\n",
    "\n",
    "def prepare_data(data, labels, embeddings):\n",
    "    sequences_seq = []\n",
    "    for i, doc in enumerate(data):\n",
    "        sequences = []\n",
    "        for sent in nlp(doc).sents:\n",
    "            sequence = []\n",
    "            for token in sent:\n",
    "                if token.is_punct: continue\n",
    "                w = token.lower_.strip()\n",
    "                if not w: continue\n",
    "                \n",
    "                if w in embeddings: sequence.append(embeddings[w] + 1)\n",
    "                else: sequence.append(0)\n",
    "            if sequence: sequences.append(sequence)\n",
    "        if sequences: sequences_seq.append(sequences)\n",
    "#         if (i + 1) % batch_size == 0: print('processed %d documents' % (i + 1))\n",
    "                \n",
    "    words_seq = []\n",
    "    length_seq = []\n",
    "    label_seq = []\n",
    "    for i, (sequences, label) in enumerate(zip(sequences_seq, labels)):\n",
    "        words = tf.keras.preprocessing.sequence.pad_sequences(sequences, padding='post', value=-1)\n",
    "        length = np.apply_along_axis(lambda ws : next(i for i, d in enumerate(ws) if d < 0), axis=1, arr=np.c_[words, np.ones((len(words), 1)) * -1])\n",
    "        words_seq.append(words + np.where(words < 0, 1, 0).astype(np.int32))\n",
    "        length_seq.append(length)\n",
    "        label_seq.append(label)\n",
    "#         if (i + 1) % batch_size == 0: print('padded %d documents' % (i + 1))\n",
    "    return words_seq, length_seq, label_seq\n",
    "\n",
    "# print('preparing embeddings')\n",
    "embeddings = prepare_embeddings(train_data.data[:train_data_size])# + test_data.data[:test_data_size])\n",
    "# print('converting training documents')\n",
    "train_words_seq, train_length_seq, train_label_seq = \\\n",
    "    prepare_data(train_data.data[:train_data_size], train_data.target[:train_data_size], embeddings)\n",
    "# print('converting test documents')\n",
    "test_words_seq, test_length_seq, test_label_seq = \\\n",
    "    prepare_data(test_data.data[:test_data_size], test_data.target[:test_data_size], embeddings)                                \n",
    "#     prepare_data(train_data.data[train_data_size:train_data_size+test_data_size], train_data.target[train_data_size:train_data_size+test_data_size], embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "online training (feeding one training example at a time)\n",
      "Epoch [1/40], Loss 2897.101, accuracy 0.070, eval accuracy 0.071\n",
      "Epoch [2/40], Loss 2818.146, accuracy 0.128, eval accuracy 0.040\n",
      "Epoch [3/40], Loss 2617.001, accuracy 0.156, eval accuracy 0.030\n",
      "Epoch [4/40], Loss 2173.917, accuracy 0.288, eval accuracy 0.040\n",
      "Epoch [5/40], Loss 1915.840, accuracy 0.351, eval accuracy 0.051\n",
      "Epoch [6/40], Loss 1545.377, accuracy 0.492, eval accuracy 0.040\n",
      "Epoch [7/40], Loss 1295.022, accuracy 0.554, eval accuracy 0.040\n",
      "Epoch [8/40], Loss 1179.530, accuracy 0.598, eval accuracy 0.121\n",
      "Epoch [9/40], Loss 751.712, accuracy 0.753, eval accuracy 0.020\n",
      "Epoch [10/40], Loss 729.572, accuracy 0.755, eval accuracy 0.051\n",
      "Epoch [11/40], Loss 920.071, accuracy 0.709, eval accuracy 0.030\n",
      "Epoch [12/40], Loss 695.593, accuracy 0.776, eval accuracy 0.000\n",
      "Epoch [13/40], Loss 382.206, accuracy 0.879, eval accuracy 0.030\n",
      "Epoch [14/40], Loss 515.913, accuracy 0.836, eval accuracy 0.030\n",
      "Epoch [15/40], Loss 485.860, accuracy 0.856, eval accuracy 0.020\n",
      "Epoch [16/40], Loss 458.447, accuracy 0.857, eval accuracy 0.040\n",
      "Epoch [17/40], Loss 586.943, accuracy 0.827, eval accuracy 0.020\n",
      "Epoch [18/40], Loss 516.249, accuracy 0.840, eval accuracy 0.040\n",
      "Epoch [19/40], Loss 402.776, accuracy 0.864, eval accuracy 0.020\n",
      "Epoch [20/40], Loss 263.329, accuracy 0.929, eval accuracy 0.040\n",
      "Epoch [21/40], Loss 327.029, accuracy 0.893, eval accuracy 0.010\n",
      "Epoch [22/40], Loss 388.008, accuracy 0.882, eval accuracy 0.061\n",
      "Epoch [23/40], Loss 316.884, accuracy 0.906, eval accuracy 0.061\n",
      "Epoch [24/40], Loss 393.318, accuracy 0.864, eval accuracy 0.040\n",
      "Epoch [25/40], Loss 224.964, accuracy 0.930, eval accuracy 0.020\n",
      "Epoch [26/40], Loss 291.909, accuracy 0.898, eval accuracy 0.061\n",
      "Epoch [27/40], Loss 197.819, accuracy 0.934, eval accuracy 0.030\n",
      "Epoch [28/40], Loss 397.169, accuracy 0.878, eval accuracy 0.040\n",
      "Epoch [29/40], Loss 298.272, accuracy 0.909, eval accuracy 0.040\n",
      "Epoch [30/40], Loss 239.823, accuracy 0.921, eval accuracy 0.040\n",
      "Epoch [31/40], Loss 226.239, accuracy 0.936, eval accuracy 0.091\n",
      "Epoch [32/40], Loss 234.233, accuracy 0.935, eval accuracy 0.061\n",
      "Epoch [33/40], Loss 374.498, accuracy 0.891, eval accuracy 0.051\n",
      "Epoch [34/40], Loss 388.733, accuracy 0.884, eval accuracy 0.111\n",
      "Epoch [35/40], Loss 244.678, accuracy 0.924, eval accuracy 0.040\n",
      "Epoch [36/40], Loss 380.534, accuracy 0.894, eval accuracy 0.030\n",
      "Epoch [37/40], Loss 258.382, accuracy 0.921, eval accuracy 0.030\n",
      "Epoch [38/40], Loss 202.243, accuracy 0.932, eval accuracy 0.030\n",
      "Epoch [39/40], Loss 261.395, accuracy 0.914, eval accuracy 0.010\n",
      "Epoch [40/40], Loss 223.165, accuracy 0.925, eval accuracy 0.020\n"
     ]
    }
   ],
   "source": [
    "print('online training (feeding one training example at a time)')\n",
    "embedding_matrix = np.random.randn(len(embeddings) + 2, 10)\n",
    "model = HierarchicalAttentionNetwork(embedding_matrix, len(tag_to_ix), hidden_dim=10)\n",
    "model.evaluate(train_words_seq, train_length_seq, train_label_seq, test_words_seq, test_length_seq, test_label_seq, 40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mini batch training (batch size = 100)\n",
      "Epoch [1/40], Loss 2910.292, accuracy 0.062, eval accuracy 0.051\n",
      "Epoch [2/40], Loss 2906.813, accuracy 0.068, eval accuracy 0.040\n",
      "Epoch [3/40], Loss 2901.758, accuracy 0.068, eval accuracy 0.081\n",
      "Epoch [4/40], Loss 2901.286, accuracy 0.068, eval accuracy 0.071\n",
      "Epoch [5/40], Loss 2898.722, accuracy 0.070, eval accuracy 0.081\n",
      "Epoch [6/40], Loss 2897.740, accuracy 0.067, eval accuracy 0.061\n",
      "Epoch [7/40], Loss 2889.184, accuracy 0.084, eval accuracy 0.051\n",
      "Epoch [8/40], Loss 2813.171, accuracy 0.111, eval accuracy 0.051\n",
      "Epoch [9/40], Loss 2598.731, accuracy 0.172, eval accuracy 0.040\n",
      "Epoch [10/40], Loss 2480.859, accuracy 0.199, eval accuracy 0.030\n",
      "Epoch [11/40], Loss 2452.607, accuracy 0.229, eval accuracy 0.040\n",
      "Epoch [12/40], Loss 2432.214, accuracy 0.225, eval accuracy 0.010\n",
      "Epoch [13/40], Loss 2357.286, accuracy 0.191, eval accuracy 0.020\n",
      "Epoch [14/40], Loss 2241.114, accuracy 0.293, eval accuracy 0.010\n",
      "Epoch [15/40], Loss 2037.580, accuracy 0.364, eval accuracy 0.061\n",
      "Epoch [16/40], Loss 1765.356, accuracy 0.480, eval accuracy 0.040\n",
      "Epoch [17/40], Loss 1663.632, accuracy 0.521, eval accuracy 0.061\n",
      "Epoch [18/40], Loss 1433.553, accuracy 0.619, eval accuracy 0.061\n",
      "Epoch [19/40], Loss 1250.479, accuracy 0.661, eval accuracy 0.010\n",
      "Epoch [20/40], Loss 1230.629, accuracy 0.659, eval accuracy 0.030\n",
      "Epoch [21/40], Loss 1211.852, accuracy 0.689, eval accuracy 0.051\n",
      "Epoch [22/40], Loss 1056.850, accuracy 0.720, eval accuracy 0.051\n",
      "Epoch [23/40], Loss 881.422, accuracy 0.768, eval accuracy 0.030\n",
      "Epoch [24/40], Loss 882.507, accuracy 0.764, eval accuracy 0.030\n",
      "Epoch [25/40], Loss 854.966, accuracy 0.774, eval accuracy 0.040\n",
      "Epoch [26/40], Loss 861.715, accuracy 0.777, eval accuracy 0.020\n",
      "Epoch [27/40], Loss 647.595, accuracy 0.836, eval accuracy 0.030\n",
      "Epoch [28/40], Loss 598.886, accuracy 0.853, eval accuracy 0.030\n",
      "Epoch [29/40], Loss 560.431, accuracy 0.857, eval accuracy 0.040\n",
      "Epoch [30/40], Loss 517.531, accuracy 0.866, eval accuracy 0.030\n",
      "Epoch [31/40], Loss 497.521, accuracy 0.874, eval accuracy 0.030\n",
      "Epoch [32/40], Loss 457.789, accuracy 0.889, eval accuracy 0.030\n",
      "Epoch [33/40], Loss 459.997, accuracy 0.887, eval accuracy 0.030\n",
      "Epoch [34/40], Loss 454.208, accuracy 0.883, eval accuracy 0.030\n",
      "Epoch [35/40], Loss 429.582, accuracy 0.893, eval accuracy 0.030\n",
      "Epoch [36/40], Loss 432.151, accuracy 0.888, eval accuracy 0.030\n",
      "Epoch [37/40], Loss 393.054, accuracy 0.908, eval accuracy 0.040\n",
      "Epoch [38/40], Loss 416.267, accuracy 0.896, eval accuracy 0.051\n",
      "Epoch [39/40], Loss 376.841, accuracy 0.908, eval accuracy 0.051\n",
      "Epoch [40/40], Loss 375.753, accuracy 0.909, eval accuracy 0.061\n"
     ]
    }
   ],
   "source": [
    "batch_size = 100\n",
    "print('mini batch training (batch size = %d)' % batch_size)\n",
    "embedding_matrix = np.random.randn(len(embeddings) + 2, 10)\n",
    "model = HierarchicalAttentionNetwork(embedding_matrix, len(tag_to_ix), hidden_dim=10)\n",
    "model.evaluate_batch(train_words_seq, train_length_seq, train_label_seq, test_words_seq, test_length_seq, test_label_seq, num_epochs=40, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}