src/05c-ng-sim-eeap.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Document Similarity - w/ AttentionM (sentence model) + AttentionMM (document similarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division, print_function\n",
    "from keras import backend as K\n",
    "from keras.layers import Input\n",
    "from keras.layers.core import Dense, Dropout, Lambda\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.layers.pooling import GlobalMaxPooling1D\n",
    "from keras.layers.wrappers import TimeDistributed, Bidirectional\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.models import Model\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "import custom_attn\n",
    "import logging\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import os\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_DIR = \"../data\"\n",
    "\n",
    "VOCAB_FILE = os.path.join(DATA_DIR, \"ng-vocab.tsv\")\n",
    "MIN_OCCURS = 5\n",
    "\n",
    "GLOVE_FILE = os.path.join(DATA_DIR, \"glove.840B.300d.txt\")\n",
    "\n",
    "DOCSIM_IDLABELS = os.path.join(DATA_DIR, \"docsim-idlabels.tsv\")\n",
    "DOCSIM_TEXTS = os.path.join(DATA_DIR, \"docsim-texts.tsv\")\n",
    "\n",
    "# covers about 95% of input data\n",
    "MAX_SENTS = 40 # maximum number of sentences per document\n",
    "MAX_WORDS = 60 # maximum number of words per sentence\n",
    "\n",
    "WORD_EMBED_SIZE = 300\n",
    "SENT_EMBED_SIZE = 100\n",
    "DOC_EMBED_SIZE = 50\n",
    "NUM_CLASSES = 2\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "NUM_EPOCHS = 5\n",
    "\n",
    "logging.basicConfig()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_size: 40730\n"
     ]
    }
   ],
   "source": [
    "word2id = {\"PAD\": 0, \"UNK\": 1}\n",
    "fvocab = open(VOCAB_FILE, \"rb\")\n",
    "for i, line in enumerate(fvocab):\n",
    "    word, count = line.strip().split(\"\\t\")\n",
    "    if int(count) <= MIN_OCCURS:\n",
    "        break\n",
    "    word2id[word] = i\n",
    "fvocab.close()\n",
    "id2word = {v:k for k, v in word2id.items()}\n",
    "vocab_size = len(word2id)\n",
    "print(\"vocab_size: {:d}\".format(vocab_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load GloVe Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40730, 300)\n"
     ]
    }
   ],
   "source": [
    "E = np.zeros((vocab_size, WORD_EMBED_SIZE))\n",
    "E[1] = np.random.random(WORD_EMBED_SIZE)\n",
    "fglove = open(GLOVE_FILE, \"rb\")\n",
    "for line in fglove:\n",
    "    cols = line.strip().split(\" \")\n",
    "    word = cols[0]\n",
    "    if not word2id.has_key(word):\n",
    "        continue\n",
    "    vec = np.array([float(x) for x in cols[1:]])\n",
    "    idx = word2id[word]\n",
    "    E[idx] = vec\n",
    "fglove.close()\n",
    "print(E.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute Document Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['like', 'a', 'mouse']\n",
      "['PAD', 'The', 'cat', 'fought', 'like', 'a', 'mouse']\n"
     ]
    }
   ],
   "source": [
    "def pad_or_truncate(xs, maxlen):\n",
    "    if len(xs) > maxlen:\n",
    "        xs = xs[len(xs) - maxlen:]\n",
    "    elif len(xs) < maxlen:\n",
    "        xs = [\"PAD\"] * (maxlen - len(xs)) + xs\n",
    "    return xs\n",
    "\n",
    "xs = [\"The\", \"cat\", \"fought\", \"like\", \"a\", \"mouse\"]\n",
    "print(pad_or_truncate(xs, 3))\n",
    "print(pad_or_truncate(xs, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1885 (40, 60)\n"
     ]
    }
   ],
   "source": [
    "docid2mat = {}\n",
    "ftext = open(DOCSIM_TEXTS, \"rb\")\n",
    "for line in ftext:\n",
    "    rec_id, text = line.strip().split(\"\\t\")\n",
    "    M = np.zeros((MAX_SENTS, MAX_WORDS))\n",
    "    sents = pad_or_truncate(nltk.sent_tokenize(text), MAX_SENTS)\n",
    "    for sid, sent in enumerate(sents):\n",
    "        words = pad_or_truncate(nltk.word_tokenize(sent), MAX_WORDS)\n",
    "        for wid, word in enumerate(words):\n",
    "            try:\n",
    "                word_id = word2id[word]\n",
    "            except KeyError:\n",
    "                word_id = word2id[\"UNK\"]\n",
    "            M[sid, wid] = word_id\n",
    "    docid2mat[int(rec_id)] = M\n",
    "ftext.close()\n",
    "print(len(docid2mat), docid2mat[list(docid2mat.keys())[0]].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract Label and DocID pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70253, 2) (70253, 2)\n"
     ]
    }
   ],
   "source": [
    "xdata, ydata = [], []\n",
    "fidl = open(DOCSIM_IDLABELS, \"rb\")\n",
    "for line in fidl:\n",
    "    label, docid_left, docid_right = line.strip().split(\"\\t\")\n",
    "    xdata.append((int(docid_left), int(docid_right)))\n",
    "    ydata.append(int(label))\n",
    "X = np.array(xdata)\n",
    "Y = to_categorical(np.array(ydata), num_classes=NUM_CLASSES)\n",
    "print(X.shape, Y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Partition into training, validation and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(44259, 2) (44259, 2) (4918, 2) (4918, 2) (21076, 2) (21076, 2)\n"
     ]
    }
   ],
   "source": [
    "Xtv, Xtest, Ytv, Ytest = train_test_split(X, Y, train_size=0.7)\n",
    "Xtrain, Xval, Ytrain, Yval = train_test_split(Xtv, Ytv, train_size=0.9)\n",
    "print(Xtrain.shape, Ytrain.shape, Xval.shape, Yval.shape, \n",
    "      Xtest.shape, Ytest.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Data Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 40, 60) (64, 40, 60) (64, 2)\n"
     ]
    }
   ],
   "source": [
    "def datagen(X, Y, docid2mat, batch_size=BATCH_SIZE):\n",
    "    while True:\n",
    "        num_recs = X.shape[0]\n",
    "        indices = np.random.permutation(np.arange(num_recs))\n",
    "        num_batches = num_recs // batch_size\n",
    "        for bid in range(num_batches):\n",
    "            batch_ids = indices[bid * batch_size : (bid + 1) * batch_size]\n",
    "            Xbatch_l = np.zeros((batch_size, MAX_SENTS, MAX_WORDS))\n",
    "            Xbatch_r = np.zeros((batch_size, MAX_SENTS, MAX_WORDS))\n",
    "            for idx, (docid_l, docid_r) in enumerate(X[batch_ids, :]):\n",
    "                Xbatch_l[idx] = docid2mat[docid_l]\n",
    "                Xbatch_r[idx] = docid2mat[docid_r]\n",
    "            Ybatch = Y[batch_ids, :]\n",
    "            yield [Xbatch_l, Xbatch_r], Ybatch\n",
    "\n",
    "train_gen = datagen(Xtrain, Ytrain, docid2mat)\n",
    "[Xbatch_left, Xbatch_right], Ybatch = train_gen.next()\n",
    "print(Xbatch_left.shape, Xbatch_right.shape, Ybatch.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Network\n",
    "\n",
    "### Sentence Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 60)                0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 60, 300)           12219000  \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 60, 200)           240600    \n",
      "_________________________________________________________________\n",
      "attention_m_1 (AttentionM)   (None, 200)               260       \n",
      "=================================================================\n",
      "Total params: 12,459,860\n",
      "Trainable params: 12,459,860\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "sent_in = Input(shape=(MAX_WORDS,), dtype=\"int32\")\n",
    "\n",
    "sent_emb = Embedding(input_dim=vocab_size,\n",
    "                       output_dim=WORD_EMBED_SIZE,\n",
    "                       weights=[E])(sent_in)\n",
    "\n",
    "sent_enc = Bidirectional(GRU(SENT_EMBED_SIZE,\n",
    "                               return_sequences=True))(sent_emb)\n",
    "\n",
    "sent_att = custom_attn.AttentionM()(sent_enc)\n",
    "\n",
    "sent_model = Model(inputs=sent_in, outputs=sent_att)\n",
    "sent_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Document Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sum_over_axis(X, axis):\n",
    "    return K.mean(X, axis=axis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_2 (InputLayer)             (None, 40, 60)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "input_3 (InputLayer)             (None, 40, 60)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistribu (None, 40, 200)       12459860    input_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistribu (None, 40, 200)       12459860    input_3[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional)  (None, 40, 100)       75300       time_distributed_1[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional)  (None, 40, 100)       75300       time_distributed_2[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "attention_mm_1 (AttentionMM)     (None, 200)           76000       bidirectional_2[0][0]            \n",
      "                                                                   bidirectional_3[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNorm (None, 200)           800         attention_mm_1[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 200)           0           batch_normalization_1[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 50)            10050       dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 50)            0           dense_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 2)             102         dropout_2[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 12,697,412\n",
      "Trainable params: 12,697,012\n",
      "Non-trainable params: 400\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# LHS document\n",
    "doc_in_l = Input(shape=(MAX_SENTS, MAX_WORDS), dtype=\"int32\")\n",
    "\n",
    "doc_emb_l = TimeDistributed(sent_model)(doc_in_l)\n",
    "\n",
    "doc_enc_l = Bidirectional(GRU(DOC_EMBED_SIZE,\n",
    "                              return_sequences=True))(doc_emb_l)\n",
    "\n",
    "# RHS document\n",
    "doc_in_r = Input(shape=(MAX_SENTS, MAX_WORDS), dtype=\"int32\")\n",
    "\n",
    "doc_emb_r = TimeDistributed(sent_model)(doc_in_r)\n",
    "\n",
    "doc_enc_r = Bidirectional(GRU(DOC_EMBED_SIZE,\n",
    "                              return_sequences=True))(doc_emb_r)\n",
    "\n",
    "# Attention\n",
    "doc_att = custom_attn.AttentionMM(\"concat\")([doc_enc_l, doc_enc_r])\n",
    "doc_att = BatchNormalization()(doc_att)\n",
    "\n",
    "# Prediction\n",
    "fc1_dropout = Dropout(0.2)(doc_att)\n",
    "fc1 = Dense(50, activation=\"relu\")(fc1_dropout)\n",
    "fc2_dropout = Dropout(0.2)(fc1)\n",
    "doc_pred = Dense(NUM_CLASSES, activation=\"softmax\")(fc2_dropout)\n",
    "\n",
    "model = Model(inputs=[doc_in_l, doc_in_r], outputs=doc_pred)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "691/691 [==============================] - 1031s - loss: 0.1804 - acc: 0.9072 - val_loss: 1.0721 - val_acc: 0.6394\n",
      "Epoch 2/5\n",
      "691/691 [==============================] - 1025s - loss: 0.0421 - acc: 0.9889 - val_loss: 0.2148 - val_acc: 0.9221\n",
      "Epoch 3/5\n",
      "691/691 [==============================] - 1026s - loss: 0.0368 - acc: 0.9908 - val_loss: 0.1051 - val_acc: 0.9663\n",
      "Epoch 4/5\n",
      "691/691 [==============================] - 1026s - loss: 0.0335 - acc: 0.9916 - val_loss: 0.0587 - val_acc: 0.9852\n",
      "Epoch 5/5\n",
      "691/691 [==============================] - 1025s - loss: 0.0290 - acc: 0.9925 - val_loss: 0.1056 - val_acc: 0.9700\n"
     ]
    }
   ],
   "source": [
    "train_gen = datagen(Xtrain, Ytrain, docid2mat, batch_size=BATCH_SIZE)\n",
    "val_gen = datagen(Xval, Yval, docid2mat, batch_size=BATCH_SIZE)\n",
    "\n",
    "num_train_steps = len(Xtrain) // BATCH_SIZE\n",
    "num_val_steps = len(Xval) // BATCH_SIZE\n",
    "\n",
    "history = model.fit_generator(train_gen, \n",
    "                              steps_per_epoch=num_train_steps,\n",
    "                              epochs=NUM_EPOCHS,\n",
    "                              validation_data=val_gen,\n",
    "                              validation_steps=num_val_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8VNW9///XhxAIARQIKEiAxMtXES+oEVGstdoLUgta\n5eKlXqu/tlovbU9Lv20ttfZb23PaWk/1WC9UqVakqJVaLK0VDlZBCRYvqFVUkIDKRbkJAQKf3x9r\nQiaTSTIJk9l7kvfz8diP2TN7zezPbML+zFp77bXM3REREYmbTlEHICIiko4SlIiIxJISlIiIxJIS\nlIiIxJISlIiIxJISlIiIxJISlIiIxJISlIiIxJISlEhMWKD/kyIJ+s8gksLMJpvZW2a22cxeNbOz\nk7ZdYWavJW07NvH6IDN7xMzWmtl6M/tN4vUpZnZ/0vvLzMzNrHPi+Twz+4mZPQNsBQ40s0uT9vG2\nmf1/KfGNM7MlZrYpEedoMxtvZotTyn3DzB5ruyMl0rY6Rx2ASAy9BXwCeB8YD9xvZgcDJwNTgLOA\nSuAgYKeZFQCPA08BXwJ2ARUt2N+XgDOAfwMGHAqcCbwNnAI8YWaL3P0FMxsBTAPOBf4BDAB6Au8A\nvzWzoe7+WtLn3tSaAyASB6pBiaRw9z+6+2p33+3uDwFvAiOALwM/d/dFHixz9xWJbQcA/+HuH7t7\ntbv/swW7vNfdl7p7jbvvdPe/uPtbiX38L/A3QsIEuByY6u5/T8S3yt1fd/ftwEPAhQBmNgwoIyRO\nkbykBCWSwswuSjShbTCzDcARQF9gEKF2lWoQsMLda1q5y5Up+z/DzBaa2YeJ/Y9J7L92X+liALgP\nON/MjFB7mpFIXCJ5SQlKJImZDQHuAq4GSty9F/AKoeltJaFZL9VKYHDtdaUUHwPFSc/7pymzZ0oB\nM+sKPAz8F7B/Yv+zE/uv3Ve6GHD3hcAOQm3rfOD36b+lSH5QghKprzshYawFMLNLCTUogLuBb5nZ\ncYkedwcnEtrzwHvAzWbW3cyKzGxU4j1LgFPMbLCZ7Qt8t5n9dwG6JvZfY2ZnAJ9N2n4PcKmZnW5m\nncxsoJkdlrR9GvAbYGcLmxlFYkcJSiSJu78K/AJYAHwAHAk8k9j2R+AnwB+AzcCfgD7uvgv4AnAw\n8C5QBUxMvOfvhGtDLwGLaeaakLtvBq4BZgAfEWpCs5K2Pw9cCvwK2Aj8LzAk6SN+T0io9yOS50wT\nFoq0H2bWDVgDHOvub0Ydj8jeUA1KpH35KrBIyUnaA90HJdJOmNlyQmeKsyIORSQr1MQnIiKxpCY+\nERGJpWab+MxsKmHYlTXufkSa7Qb8mnAz4VbgEnd/obnP7du3r5eVlbU4YBERyW+LFy9e5+79miuX\nyTWoewn3VUxrZPsZwCGJ5QTgfxKPTSorK6OysjKD3YuISHtiZisyKddsE5+7zwc+bKLIOGBaYtyw\nhUAvMxuQWZgiIiLpZaMX30DqjyVWlXjtvdSCZnYlcCXA4MGDs7BrERHJmHtYmltvanuXLlBYmJNw\nc9rN3N3vBO4EqKioUPfBOFu1Cj76qO4Pc/fujrWeq31A608Y2X6tPe2vJds7Qtlsuusu+PKXs/+5\naWQjQa0ijLBcqzTxmuSLnTvhpZfgmWfC8uyzUFUVdVTx0akTmIUlW+vJC9R/bGw9F69l83Nqv28b\n7m+3G1t2dWNzTTc21RSzaVd3qncV0q3zTrp33h6Wwu10L9hOt847sU6tjENl69ZHjCBXspGgZgFX\nm9l0QueIje7eoHlPYuSjj2Dhwrpk9NxzsHVr2DZoEJx8Mpx4IgwYkN2Tci7Xs/VZknW7d8PHH8Pm\nzbBpU+OPTW2rfdyypWX7Li6G7t2bXjIpk65cjlq9OpRMupk/CJwK9DWzKuCHQCGAu99BmApgDLCM\n0M380rYKVlrBHZYtC4moNiEtXRq2FRTA8OGhun7SSWEZNKjpz5MOyT38hsk0cTSVbLZsyazlqXNn\n2GefsPTsGR5LSqC8vO55usdu3WDbtpAE0y1btzZ8bdWqhmV27GjZMSosbH3Ca65Mt24d8/dSswnK\n3c9rZrsDV2UtItk71dWweHH9hLR2bdjWq1eoGU2aBKNGhap69+7Rxittxj2cqPe2lrJ5c1h2725+\nnwUFDRNG794weHDDZNPcY9eu0Z6Ud+5Mn8yaSnTpymzcCKtXN3y9pZeHWluzy+fan8biy3cffBCS\nUG1CWry47qffwQfDmDGhZjRqFAwdGpquJLbcw2+Mva2l1CaVXbua32enTnVJoTZB7LsvlJZmlkiS\n14uK2s8v/cLCcBz23Tf7n13746Elia6xbatWNdzelrW/CRPghGbvdM0OJah8sns3vPpq/drRsmVh\nW5cuUFEB115b11y3337Rxiu4w7p1sGIFLF9e9/jBB40nm5oMJo43S58wDjggfeJo6rG4uP0klXxh\nFo57cXHzZVujpqb1Nb7k5+lqf0ccoQQlEP4annuuLiEtWBD+YgD69Qu1oiuvDI/HHht+vkpOucOa\nNSHpJCeg5PXa/ie19tkn9D/Zd9+QIPbfP/Nmr9r17t2VVKRxnTu3Xe0vl5Sg4mTlyrqa0TPPwIsv\n1rXRDBsGEyfWNdcddJDOUDmweze8/37DpJP8WF1d/z19+sCQIXDoofC5z4X1srK6x169cv0tRPKT\nElRUampCAkpOSLX3HhUXhzr0d78bEtLIkeFKs2Tdrl2hCaOx2s+77zZsz+/bNySaI4+EM88M67UJ\naMiQUMsRkb2nBJUrGzaEJrraZJR871FpaagVjRoVEtLRR4c6uuy1mpqQ99Mln+XLQ6U19ZrP/vuH\nhHPccfDFLzZMQOr4KJIbOgu2BXd46636taPke4+OPhouv7yuuU73HrXajh0hyTSWgFatqt+TzSxc\n/ykrq+txn9z8NnhwuOdERKKnBJUN27eH7t21CenZZ8OVcwhXKWvPhCedFO496tEj2njzSHV1aGZL\nvfZTu75qVf37STp1goEDQ7I55ZT6tZ+ysvBboGvXCL6IiLSYElRrrFlTv6t3ZWX9e49Gj65rrjv8\ncN171IStW0MCaqwX3Hspg2YVFIQkU1YGp5/eMAGVlsb3pkMRaRklqObs3g2vvVa/uS753qPjjoNr\nrqm792j//aONN2a2bGm89rN8eV1Fs1ZhYWhmKyuDM85omIAOOECX50Q6Cv1XT/Xxx/D883UJacGC\n0MEBwr1HJ50U7j066aSQnDr4vUebNjXeBXv5cli/vn75rl3rOhuMG1c/+ZSVQf/+oZYkIqIEtXJl\n/ea6JUvqrqoffjiMH1/XXHfwwR3q3iP3kJsbq/0sX16Xu2t161aXdI4/vn4CGjIkVDDV4ikimehY\nCar23qPkhLQyMRlwcXHowDB5ckhGJ57YYe89WrkyDHC+cGGoISXr0aMu2Ywa1TAB9evXoXK4iLSh\n9p2gNmyom/co9d6jgQPDGfZb36q790hX15k3LwwGWV0NF18cpjZIvg7Up48SkIjkRvtJULX3HqXO\ne+Qe2pSGD4fLLqtrrhs8OOqIY8UdfvUr+Pa34ZBD4NFH4bDDoo5KRDqyjBKUmY0Gfg0UAHe7+80p\n24cAU4F+wIfAhe7etnOG1957lJyQaruE7bNPaKKbMCEkoxNO0L1HTfj449CkN316GDnh3nvDgKQi\nIlHKZEbdAuA24DNAFbDIzGa5+6tJxf4LmObu95nZacBPgS+1RcB7LFgAn/pUWD/ooDAqZ23taNgw\nXYnP0LJlcPbZYRaPn/4UvvMdNeGJSDxkUoMaASxz97cBzGw6MA5ITlCHA99IrM8F/pTNINNHNQIe\neSTUlPr3b/PdtUezZ8MFF4Rc/sQT8NnPRh2RiEidTKoZA4GVSc+rEq8lexH4YmL9bKCnmZWkfpCZ\nXWlmlWZWubZ2GvLWKi4OP/2VnFps92748Y/DSNzl5aGlVMlJROImW+1g3wI+aWb/Aj4JrAIaTDbt\n7ne6e4W7V/Tr1y9Lu5aW2LgRzjoLbrgBLrwwXL4rK4s6KhGRhjJp4lsFJA+3XZp4bQ93X02iBmVm\nPYBz3D3lFk6J2tKlodL5zjvw3/8NV12l600iEl+Z1KAWAYeYWbmZdQEmAbOSC5hZXzOr/azvEnr0\nSYz88Y+hM+PmzTB3Llx9tZKTiMRbswnK3WuAq4E5wGvADHdfamY3mtnYRLFTgX+b2RvA/sBP2ihe\naaGamnBv04QJ4V7kxYvh5JOjjkpEpHnmyZPp5FBFRYVXVlZGsu+OYu3aMA3VU0/B174WbsTt0iXq\nqESkozOzxe5e0Vy59jOShNSzeHG46faDD+B3v4NLLok6IhGRltHdrO3QvfeGe5Yh9NJTchKRfKQE\n1Y7s2BGa8i69NFxnWrw4TFklIpKPlKDaidWr4dRT4X/+J3SK+OtfoW/fqKMSEWk9XYNqB55+Osyr\nuGULzJgR1kVE8p1qUHnMPdxwe9ppYQD3555TchKR9kMJKk9t3RomFLzmGhgzBhYtCoO4i4i0F0pQ\neeidd0IvvfvvD4O+Pvoo7Ltv1FGJiGSXrkHlmb/9Dc47L4xI/vjjofYkItIeqQaVJ9zDhIKjR8PA\ngVBZqeQkIu2balB5YNOmcLPto4+GoYvuvhu6d486KhGRtqUEFXOvvx6myHjzTfjlL+G66zQKuYh0\nDEpQMfboo6GnXlERPPlkuBFXRKSj0DWoGNq1C773vTDY69Ch8MILSk4i0vGoBhUzH34I558Pc+bA\nFVeEG3G7do06KhGR3MuoBmVmo83s32a2zMwmp9k+2Mzmmtm/zOwlM1P/slZYsgQqKsKMt3feGRYl\nJxHpqJpNUGZWANwGnAEcDpxnZoenFPs+YabdYwhTwt+e7UDbu/vvh5NOCiOSz58fak8iIh1ZJjWo\nEcAyd3/b3XcA04FxKWUc2Cexvi+wOnshtm87d8K118KXvgQjRoQpMk44IeqoRESil8k1qIHAyqTn\nVUDqKXQK8Dcz+zrQHfh0VqJr595/HyZMCKORX389/OxnUFgYdVQiIvGQrV585wH3unspMAb4vZk1\n+Gwzu9LMKs2scu3atVnadX5asCBMJlhZCX/4Q7jHSclJRKROJglqFTAo6Xlp4rVklwMzANx9AVAE\nNJguz93vdPcKd6/o169f6yLOc+5wxx3wyU+G+5sWLgxj64mISH2ZJKhFwCFmVm5mXQidIGallHkX\nOB3AzIYSElTHriKlUV0Nl18OX/0qfPrTofZ01FFRRyUiEk/NJih3rwGuBuYArxF66y01sxvNbGyi\n2DeBK8zsReBB4BJ397YKOh+9+y584hPwu9/BDTeEkch79446KhGR+MroRl13nw3MTnnthqT1V4FR\n2Q2t/XjqKZg4MXQhf+wxGDu2+feIiHR0GkmiDbnDL34B3/kOHHYYPPIIHHpo1FGJSNR27txJVVUV\n1dXVUYfSpoqKiigtLaWwlT3AlKDayJYt4XrTjBlw7rkwdSr07Bl1VCISB1VVVfTs2ZOysjKsnU5P\n4O6sX7+eqqoqysvLW/UZGiy2Dbz5JowcCTNnws9/HpKUkpOI1KqurqakpKTdJicAM6OkpGSvaomq\nQWXZ44/DBReEe5rmzAm99UREUrXn5FRrb7+jalBZsns3TJkCX/gCHHxwGLJIyUlEpPWUoLJgw4bQ\nM+9HPwpTs//znzBkSNRRiYikt2HDBm6/veVjeo8ZM4YNGza0QUTpKUHtpZdfDlNk/O1vcPvtoTNE\nt25RRyUi0rjGElRNTU2T75s9eza9evVqq7Aa0DWovTB9euipt+++MG9emC5DRCTuJk+ezFtvvcXw\n4cMpLCykqKiI3r178/rrr/PGG29w1llnsXLlSqqrq7n22mu58sorASgrK6OyspItW7ZwxhlncPLJ\nJ/Pss88ycOBAHnvsMbpl+de5ElQr1NTA5MnhHqdRo+CPf4QBA6KOSkTy0nXXhdlKs2n4cLjllkY3\n33zzzbzyyissWbKEefPm8fnPf55XXnllT3fwqVOn0qdPH7Zt28bxxx/POeecQ0lJSb3PePPNN3nw\nwQe56667mDBhAg8//DAXXnhhVr+GElQLrVkTRoWYNw+uvjokqS5doo5KRKT1RowYUe9epVtvvZVH\nH30UgJUrV/Lmm282SFDl5eUMHz4cgOOOO47ly5dnPS4lqBZYtAi++EVYtw7uuw8uuijqiEQk7zVR\n08mV7t2771mfN28eTz75JAsWLKC4uJhTTz017b1MXbt23bNeUFDAtm3bsh6XOklk6J574OSToXNn\nePZZJScRyV89e/Zk8+bNabdt3LiR3r17U1xczOuvv87ChQtzHF0d1aCasX17mJL9t7+Fz3wGHnwQ\nUmq6IiJ5paSkhFGjRnHEEUfQrVs39t9//z3bRo8ezR133MHQoUM59NBDGTlyZGRxWlSzYlRUVHhl\nZWUk+85UVVUYR++550KniJtugoKCqKMSkXz32muvMXTo0KjDyIl039XMFrt7RXPvVQ2qEf/7vzBh\nAmzdGsbUO+ecqCMSEelYdA0qhTv8+tdw+ulhQsHnn1dyEhGJQkYJysxGm9m/zWyZmU1Os/1XZrYk\nsbxhZrkbCyOLtm6FCy8MtyWceWZITh2kFi4iEjvNNvGZWQFwG/AZoApYZGazErPoAuDu1yeV/zpw\nTBvE2qbefhvOPjsMXfSTn4RrTp1UvxQRiUwm16BGAMvc/W0AM5sOjANebaT8ecAPsxNebvz1r3D+\n+WH9iSfgc5+LNh4REcmsiW8gsDLpeVXitQbMbAhQDjzVyPYrzazSzCrXrl3b0lizbvfuUFsaMwYG\nD4bKSiUnEZG4yHYj1iRgprvvSrfR3e909wp3r+jXr1+Wd90yGzeGUSG+/30477xw8+2BB0YakohI\nLPXo0SOS/WaSoFYBg5KelyZeS2cS8ODeBtXWXn0VRowIs9/++tdw//1QXBx1VCIikiyTa1CLgEPM\nrJyQmCYB56cWMrPDgN7AgqxGmGUPPxwmFSwuhqeeglNOiToiEZHcmjx5MoMGDeKqq64CYMqUKXTu\n3Jm5c+fy0UcfsXPnTm666SbGjRsXaZzNJih3rzGzq4E5QAEw1d2XmtmNQKW7z0oUnQRM96iGpmjG\nrl3wve/Bz34GI0eGm28Hpr2SJiKSOxHMtsHEiRO57rrr9iSoGTNmMGfOHK655hr22Wcf1q1bx8iR\nIxk7dixmlt3gWiCjkSTcfTYwO+W1G1KeT8leWNm1fj1MmgRPPglf+Ur4h0saiFdEpEM55phjWLNm\nDatXr2bt2rX07t2b/v37c/311zN//nw6derEqlWr+OCDD+jfv39kcbb7oY5eeCF0hnj//TAi+WWX\nRR2RiEidqGbbGD9+PDNnzuT9999n4sSJPPDAA6xdu5bFixdTWFhIWVlZ2mk2cqld34o6bVqY8Xb3\nbnj6aSUnEZFaEydOZPr06cycOZPx48ezceNG9ttvPwoLC5k7dy4rVqyIOsT2maB27Aiz3V58MZx4\nYri/6fjjo45KRCQ+hg0bxubNmxk4cCADBgzgggsuoLKykiOPPJJp06Zx2GGHRR1i+2vie+89GD8e\nnnkGvvUt+OlPwySDIiJS38svv7xnvW/fvixYkL4T9pYtW3IVUj3t6tT9zDNh/qZNm2D6dJg4MeqI\nRESktdpFE5873HYbnHoq9OgRJhhUchIRyW95n6C2bYNLLw3XnEaPhkWL4Igjoo5KRKRpMb1lNKv2\n9jvmdYJasQJOPhnuuw+mTIHHHoNevaKOSkSkaUVFRaxfv75dJyl3Z/369RQVFbX6M/L2GtTzz4dR\nyGtq4M9/DhMMiojkg9LSUqqqqojDrA5tqaioiNLS0la/P28T1EEHhSGLfvUrOOSQqKMREclcYWEh\n5eXlUYcRe3mboEpKwmjkIiLSPuX1NSgREWm/lKBERCSWLKpeJGa2FsjGYE99gXVZ+JxcyKdYIb/i\nzadYQfG2pXyKFTpmvEPcvdlp1SNLUNliZpXuXhF1HJnIp1ghv+LNp1hB8balfIoVFG9T1MQnIiKx\npAQlIiKx1B4S1J1RB9AC+RQr5Fe8+RQrKN62lE+xguJtVN5fgxIRkfapPdSgRESkHVKCEhGRWMqL\nBGVmo83s32a2zMwmp9ne1cweSmx/zszKch9lvXiai/cSM1trZksSy5ejiDMRy1QzW2NmrzSy3czs\n1sR3ecnMjs11jCnxNBfvqWa2MenY3pDrGJNiGWRmc83sVTNbambXpikTi+ObYaxxOrZFZva8mb2Y\niPdHacrE5ryQYbyxOS8k4ikws3+ZWYNB5XJ2bN091gtQALwFHAh0AV4EDk8p8zXgjsT6JOChmMd7\nCfCbqI9tIpZTgGOBVxrZPgZ4AjBgJPBczOM9FXg86uOaiGUAcGxivSfwRpq/hVgc3wxjjdOxNaBH\nYr0QeA4YmVImTueFTOKNzXkhEc83gD+k+zfP1bHNhxrUCGCZu7/t7juA6cC4lDLjgPsS6zOB083M\nchhjskzijQ13nw982ESRccA0DxYCvcxsQG6iayiDeGPD3d9z9xcS65uB14CBKcVicXwzjDU2Esdr\nS+JpYWJJ7fEVm/NChvHGhpmVAp8H7m6kSE6ObT4kqIHAyqTnVTT8j7OnjLvXABuBkpxE11Am8QKc\nk2jSmWlmg3ITWqtk+n3i5MREU8oTZjYs6mAAEk0gxxB+OSeL3fFtIlaI0bFNNEEtAdYAf3f3Ro9t\nDM4LmcQL8Tkv3AJ8G9jdyPacHNt8SFDt0Z+BMnc/Cvg7db9EZO+9QBjn62jgv4E/RRwPZtYDeBi4\nzt03RR1PU5qJNVbH1t13uftwoBQYYWZHRBlPczKINxbnBTM7E1jj7ouj2H+yfEhQq4DkXxKlidfS\nljGzzsC+wPqcRNdQs/G6+3p33554ejdwXI5ia41Mjn9suPum2qYUd58NFJpZ36jiMbNCwgn/AXd/\nJE2R2Bzf5mKN27Gt5e4bgLnA6JRNcTov7NFYvDE6L4wCxprZcsIlitPM7P6UMjk5tvmQoBYBh5hZ\nuZl1IVyQm5VSZhZwcWL9XOApT1y9i0Cz8aZcYxhLaO+Pq1nARYneZiOBje7+XtRBNcbM+te2hZvZ\nCMLfeCQnpUQc9wCvufsvGykWi+ObSawxO7b9zKxXYr0b8Bng9ZRisTkvZBJvXM4L7v5ddy919zLC\n+espd78wpVhOjm3sZ9R19xozuxqYQ+ghN9Xdl5rZjUClu88i/Mf6vZktI1xAnxTzeK8xs7FATSLe\nS6KK18weJPTO6mtmVcAPCRdwcfc7gNmEnmbLgK3ApdFEGmQQ77nAV82sBtgGTIrwx8oo4EvAy4lr\nDwD/FxgMsTu+mcQap2M7ALjPzAoIiXKGuz8e1/MCmcUbm/NCOlEcWw11JCIisZQPTXwiItIBKUGJ\niEgsKUGJiEgsKUGJiEgsKUGJiEgsKUGJiEgsKUGJiEgsKUGJiEgsKUGJiEgsKUGJiEgsKUGJiEgs\nKUGJiEgsKUGJZJGZLTezT0cdh0h7oAQlIiKxpAQlIiKxpAQl0gbMrKuZ3WJmqxPLLWbWNbGtr5k9\nbmYbzOxDM3vazDoltn3HzFaZ2WYz+7eZnR7tNxGJTuxn1BXJU98DRgLDAQceA74P/AD4JlAF9EuU\nHQm4mR0KXA0c7+6rzayMMCuzSIekGpRI27gAuNHd17j7WuBHhCnVAXYSpgAf4u473f3pxNTpu4Cu\nwOFmVujuy939rUiiF4kBJSiRtnEAsCLp+YrEawD/CSwD/mZmb5vZZAB3XwZcB0wB1pjZdDM7AJEO\nSglKpG2sBoYkPR+ceA133+zu33T3A4GxwDdqrzW5+x/c/eTEex34WW7DFokPJSiRtvEg8H0z62dm\nfYEbgPsBzOxMMzvYzAzYSGja221mh5rZaYnOFNXANmB3RPGLRE4JSqRt3ARUAi8BLwMvJF4DOAR4\nEtgCLABud/e5hOtPNwPrgPeB/YDv5jZskfiwcG1WREQkXlSDEhGRWFKCEhGRWFKCEhGRWFKCEhGR\nWIpsqKO+fft6WVlZVLsXEZGILF68eJ2792uuXGQJqqysjMrKyqh2LyIiETGzFc2XUhOfiIjEVN4m\nqB074Ic/hM2bo45ERETaQt4mqKefhptugpNOgrffjjoaERHJtrydD+r002HOHJgwAY4/Hh56CD79\n6aijEhFp3s6dO6mqqqK6ujrqUNpUUVERpaWlFBYWtur9eZugICSkRYtg3Dj43OfgF7+Aa68Fs6gj\nExFpXFVVFT179qSsrAxrpycsd2f9+vVUVVVRXl7eqs9otonPzKaa2Roze6WR7WZmt5rZMjN7ycyO\nbVUkrXTQQbBgQUhS118Pl14K7fxHiYjkuerqakpKStptcgIwM0pKSvaqlpjJNah7gdFNbD+DMDrz\nIcCVwP+0OppW6tkTZs6EKVPgvvvgk5+EVatyHYWISObac3KqtbffsdkE5e7zgQ+bKDIOmObBQqCX\nmQ3Yq6haoVOn0Kvv0Ufh1VfDdamFC3MdhYiIZEs2evENBFYmPa9KvNaAmV1pZpVmVrl27dos7Lqh\ns84KTX7duoWa1O9+1ya7ERHJWxs2bOD2229v8fvGjBnDhg0b2iCi9HLazdzd73T3Cnev6Nev2VEu\nWu2II0LniVNOgcsuCx0ndu5ss92JiOSVxhJUTU1Nk++bPXs2vXr1aquwGshGgloFDEp6Xpp4LVJ9\n+sATT4SOE7feGnr5rVsXdVQiItGbPHkyb731FsOHD+f444/nE5/4BGPHjuXwww8H4KyzzuK4445j\n2LBh3HnnnXveV1ZWxrp161i+fDlDhw7liiuuYNiwYXz2s59l27ZtWY8zG93MZwFXm9l04ARgo7u/\nl4XP3WudO8MvfwnDh8OVV4brUo89BkcdFXVkIiIJ110HS5Zk9zOHD4dbbml0880338wrr7zCkiVL\nmDdvHp///Od55ZVX9nQHnzp1Kn369GHbtm0cf/zxnHPOOZSUlNT7jDfffJMHH3yQu+66iwkTJvDw\nww9z4YUXZvVrZNLN/EFgAXComVWZ2eVm9hUz+0qiyGzgbWAZcBfwtaxGmAUXXQTz54fhkU48ER5+\nOOqIRESHsZLCAAASzklEQVTiY8SIEfXuVbr11ls5+uijGTlyJCtXruTNN99s8J7y8nKGDx8OwHHH\nHcfy5cuzHlezNSh3P6+Z7Q5clbWI2siIEVBZCV/8Ipx7LvzgB6Fbeqe8HexJRNqFJmo6udK9e/c9\n6/PmzePJJ59kwYIFFBcXc+qpp6a9l6lr16571gsKCtqkia9DnZ4HDIB580LHiR//GM4+GzZtijoq\nEZHc6tmzJ5sbGWl748aN9O7dm+LiYl5//XUWRni/Tl4PddQaXbvC3XfDMceEpt+RI8N1qUMOiToy\nEZHcKCkpYdSoURxxxBF069aN/ffff8+20aNHc8cddzB06FAOPfRQRo4cGVmcFlrocq+iosKjnrBw\n7lwYPx527YLp00NPPxGRtvbaa68xdOjQqMPIiXTf1cwWu3tFc+/tUE18qT71qXC/1ODBMGYM/Nd/\nQUT5WkREUnToBAVQXg7PPBM6T/zHf8CXvgRtcK1PRERaqMMnKIAePWDGjDAB4gMPwCc+AVVVUUcl\nItKxKUElmMH3vhc6TLzxBlRUhJqViIhEQwkqxdix8NxzYQqPT30K7ror6ohERDomJag0hg6F55+H\n004LQyRddZUGmxURyTUlqEb07g1/+UvoOHH77WF6+TaaIUREJNZ69OgRyX6VoJpQUAA//zncf3+o\nUVVUZH9MRxERSU8JKgMXXAD//Cfs3g0nnQQPPRR1RCIirTd58mRuu+22Pc+nTJnCTTfdxOmnn86x\nxx7LkUceyWOPPRZhhEGHG+qotY47Lgw2e845MGkSvPhiGM+voCDqyEQkn0Uw2wYTJ07kuuuu46qr\nwjjfM2bMYM6cOVxzzTXss88+rFu3jpEjRzJ27FjMLLvBtYASVAvsvz889RR8/evw05/CSy+F+6b2\n3TfqyEREMnfMMcewZs0aVq9ezdq1a+nduzf9+/fn+uuvZ/78+XTq1IlVq1bxwQcf0L9//8jiVIJq\noS5d4Le/DYPNfv3rcMIJ4d6pQw+NOjIRyUdRzbYxfvx4Zs6cyfvvv8/EiRN54IEHWLt2LYsXL6aw\nsJCysrK002zkkq5BtdJXvgL/+AesXx/mmpo9O+qIREQyN3HiRKZPn87MmTMZP348GzduZL/99qOw\nsJC5c+eyYsWKqENUgtobp5wSrksdeCCceSb87GcabFZE8sOwYcPYvHkzAwcOZMCAAVxwwQVUVlZy\n5JFHMm3aNA477LCoQ1QT394aMiQMiXTZZTB5crjYec89UFwcdWQiIk17+eWX96z37duXBQsWpC23\nZcuWXIVUj2pQWVBcDA8+GDpOPPQQnHwyvPtu1FGJiOQ3JagsMQs1qMcfh7feCjf1zp8fdVQiIvlL\nCSrLxowJo0707g2nnw533BF1RCISR1HNZp5Le/sdlaDawKGHhhHRP/tZ+OpXQ4+/HTuijkpE4qKo\nqIj169e36yTl7qxfv56ioqJWf4Y6SbSRXr1g1iz4wQ/CtamlS2HmzHCzr4h0bKWlpVRVVbG2nY9A\nXVRURGlpaavfrwTVhgoK4P/9PzjqqNDLr6IC/vSnMGySiHRchYWFlJeXRx1G7KmJLwcmTQpd0Tt1\nCj38/vCHqCMSEYk/JagcOeYYWLQojDpxwQXw7W/Drl1RRyUiEl9KUDm0337w5JPwta/Bf/5nGH3i\no4+ijkpEJJ6UoHKssBBuuy0MOPuPf4TBZl97LeqoRETiRwkqIldeGabu2LgxJKk//znqiERE4kUJ\nKkInnxwGm/0//wfGjYOf/ESDzYqI1FKCitigQfD003D++fD978PEifDxx1FHJSISPSWoGOjWDX7/\n+9Bx4uGH4aSTYPnyqKMSEYlWRgnKzEab2b/NbJmZTU6z/RIzW2tmSxLLl7MfavtmBt/6FvzlL7Bi\nRbipd968qKMSEYlOswnKzAqA24AzgMOB88zs8DRFH3L34Ynl7izH2WGMHh3ul9pvP/j0p+E3v9F1\nKRHpmDKpQY0Alrn72+6+A5gOjGvbsDq2Qw6BhQvDyOhf/zpccQVs3x51VCIiuZVJghoIrEx6XpV4\nLdU5ZvaSmc00s0FZia4D22efMG7f978fZuj91KfgvfeijkpEJHey1Uniz0CZux8F/B24L10hM7vS\nzCrNrLK9j+KbDZ06wY9/DH/8I7z4Yrgu9fzzUUclIpIbmSSoVUByjag08doe7r7e3Wsboe4G0o7X\n7e53unuFu1f069evNfF2SOeeC88+C126wCmnwLRpUUckItL2MklQi4BDzKzczLoAk4BZyQXMbEDS\n07GABu/JsqOPDp0nTjoJLr4YvvlNqKmJOioRkbbTbIJy9xrgamAOIfHMcPelZnajmY1NFLvGzJaa\n2YvANcAlbRVwR9a3L8yZEzpO/PKXoRPFhx9GHZWISNuwqKYcrqio8MrKykj23R5MnRqmky8tDTP3\nDhsWdUQiIpkxs8XuXtFcOY0kkacuuyzcyLt1K4wcGXr8iYi0J0pQeezEE8Ngs0OHwtlnw403wu7d\nUUclIpIdSlB5buBAmD8fLroIfvjD0ONv8+aooxIR2XtKUO1AURHcey/86lfw2GOhp9/bb0cdlYjI\n3lGCaifM4Lrr4K9/hVWr4Pjjw4y9IiL5SgmqnfnMZ8L9UgMGwOc+B7fcosFmRSQ/KUG1QwcdBAsW\nwBe+ANdfD5deCtXVUUclItIySlDtVM+eYfLDKVPgvvvgk5+E1aujjkpEJHNKUO1Yp06hZ98jj8DS\npWGw2YULo45KRCQzSlAdwNlnh8TUrVuoSf3ud1FHJCLSPCWoDuKII0LniVNOCaNQXHst7NwZdVQi\nIo1TgupA+vSBJ54IHSduvTVML79+fdRRiYikpwTVwXTuHEZCv/deeOaZcL/USy9FHZWISENKUB3U\nxReHIZK2bw9j+j38cNQRiYjUpwTVgY0YEQabPeqoMIbfDTdosFkRiQ8lqA5uwIAwbcell8KPfxx6\n/G3aFHVUIiJKUAJ07Qr33BM6TvzlL6HJb9myqKMSkY6uc9QBSDyYhankhw2DCRNC54nLLw8z9g4c\nCAccEB4HDAgJTUSkrSlBST2nnRbul7roIvjNb0InilR9+9ZPWrWPyet9+4aRLEREWksJShooL4en\nnw6joH/4YRjDb9WqsNSu1z7+61/wwQcNR0wvLAy1rXTJK/mxZ89ovqOIxF/+Jqg33gh9pcvL4cAD\nw1K7XloKBQVRR5j3zKCkJCxHHtl4uZ074f33Gyav2vWlS+Hvf0/f+aJnz+ZrY/37h4QnIh1L/iao\n6mooLg6DzM2YAbt21W3r3BmGDKlLWKmPffqEs69kRWEhDBoUlqZs2dJ0bWz+/LCeOgSTGey3X/O1\nsZIS/bOKtCfmEc1mV1FR4ZWVldn5sJ07YeVKeOedMNd56uO6dfXL9+yZPnGVl0NZWRhVVSKxe3cY\nfqmxJFa7vnZtw/d27RoSVbrklZzYiotz/71EpI6ZLXb3imbLtYsE1ZzNm0OyaiyBpc7mN2BAw2bD\n2scDDtDV/xjYvj00K6ZLXsmPH3/c8L29ejXdpHjAAbD//qEiLiLZpwSVKfdwlT85YSWvV1XV7wHQ\npUuoZTXWfNirV2RfRepzD79NGktetY/vvVe/hRjCb5D+/ZuvjfXqpWZFkZZSgsqW7dvh3Xcbr319\n9FH98r16Nd58OGSIbiKKoV27QpNhU02Kq1aFHo2punVrvjZ2wAFQVJT77yUSV0pQubJhQ+PNh++8\nAzt21JU1C2etdAnswAPDT3b9HI+t6urGa2HJCS21xRhCv5zk5NWvH3TvDj16hMfapbHnxcVqWZbs\n2rUrNIHXLlu2NFxP99oFF8CoUXu370wTlFrZ91avXnDMMWFJtXt3aD9KbTZ8553Q73r16vrli4pC\nwkpX+yovh332yc13krSKiup+SzTGPfxmaao29uKLoSNI8m+XTHTrlnlCa8nzoiL9Loor99CI05IE\nkun2dD+kmlJUFP5eTjxx7xNUplSDilJ1NSxf3njzYeqNQyUljTcfDh6sm4XyzM6d9X/Bpp5E9uZ5\n6jW1pphlL+GlvtalS8dIfq2tjTS3fcuWls0wUFDQ8N8g3Xpz21PLFhdn99ZS1aDyQVERHHZYWFK5\nh+tb6WpfixfDI49ATU1d+U6dwo1IjTUf9uvXMc4UeaSwMFTAs92vxj3UztIlsJYmvTVrGm5vyW/a\ngoK2qfV1797yXpZxqo1065Y+IfTp0/qk0qNH+/tBoAQVV2bhr7VPH6hI80Nj167QwzBd7esvfwk9\nE5MVFzde+yovD3/h0i6Yhb44XbuGP59scodt2/a+lvfRR+HPN3n7tm0ti6VLl/QJrGtX2Lo1O7WR\ndImgT5/wW7C1NZVs10baMzXxtVcff9x082HqDUL77dcwcfXsGc52nTqFx9ol+Xlrt2Xrc3K5TdrU\n7t0hsextrW/79pAE9qZJqyM1T0ZBvfikce5hdI10zYdvvx1G5WjJRYyOJFtJL90Czb+WSZm9eS2u\nn7U3n5/p8d+b8nHdR1vENXRo6I66F7J6DcrMRgO/BgqAu9395pTtXYFpwHHAemCiuy9vadCSI2bh\nmlS/fnDCCQ2379wZ2l+2bg3JzD38vK1dT30ep21R778l25IXaN1rrX1ftj6r9ju1ZVx78/mZHPts\nvqcjuPNOuOKKnOyq2QRlZgXAbcBngCpgkZnNcvdXk4pdDnzk7geb2STgZ8DEtghYcqCwEMrLo45C\nJP/kIgnm4j1NlT/44JwdzkxqUCOAZe7+NoCZTQfGAckJahwwJbE+E/iNmZlH1X4oIhKF5KZG2WuZ\n3Js+EFiZ9Lwq8VraMu5eA2wESlI/yMyuNLNKM6tcm244ahERkYScDp7i7ne6e4W7V/Tr1y+XuxYR\nkTyTSRPfKiB5KrrSxGvpylSZWWdgX0JniUYtXrx4nZmtaEGsjekLrGu2VDzkU6yQX/HmU6ygeNtS\nPsUKHTPeIZkUyiRBLQIOMbNyQiKaBJyfUmYWcDGwADgXeKq560/unpUqlJlVZtJdMQ7yKVbIr3jz\nKVZQvG0pn2IFxduUZhOUu9eY2dXAHEI386nuvtTMbgQq3X0WcA/wezNbBnxISGIiIiKtltF9UO4+\nG5id8toNSevVwPjshiYiIh1Ze5hh5s6oA2iBfIoV8ivefIoVFG9byqdYQfE2KrKhjkRERJrSHmpQ\nIiLSDilBiYhILOVFgjKz0Wb2bzNbZmaT02zvamYPJbY/Z2ZluY+yXjzNxXuJma01syWJ5ctRxJmI\nZaqZrTGzVxrZbmZ2a+K7vGRmx+Y6xpR4mov3VDPbmHRsb0hXLhfMbJCZzTWzV81sqZldm6ZMLI5v\nhrHG6dgWmdnzZvZiIt4fpSkTm/NChvHG5ryQiKfAzP5lZo+n2ZabY+vusV4IXdvfAg4EugAvAoen\nlPkacEdifRLwUMzjvQT4TdTHNhHLKcCxwCuNbB8DPAEYMBJ4Lubxngo8HvVxTcQyADg2sd4TeCPN\n30Isjm+Gscbp2BrQI7FeCDwHjEwpE6fzQibxxua8kIjnG8Af0v2b5+rY5kMNas9gte6+A6gdrDbZ\nOOC+xPpM4HSzyEZszCTe2HD3+YR71xozDpjmwUKgl5kNyE10DWUQb2y4+3vu/kJifTPwGg3HsYzF\n8c0w1thIHK8tiaeFiSW1x1dszgsZxhsbZlYKfB64u5EiOTm2+ZCgsjZYbY5kEi/AOYkmnZlmNijN\n9rjI9PvEyYmJppQnzGxY1MEAJJpAjiH8ck4Wu+PbRKwQo2ObaIJaAqwB/u7ujR7bGJwXMokX4nNe\nuAX4NrC7ke05Obb5kKDaoz8DZe5+FPB36n6JyN57ARji7kcD/w38KeJ4MLMewMPAde6+Kep4mtJM\nrLE6tu6+y92HE8YHHWFmR0QZT3MyiDcW5wUzOxNY4+6Lo9h/snxIUC0ZrBbLcLDaNtRsvO6+3t23\nJ57eTZiJOK4yOf6x4e6baptSPIyAUmhmfaOKx8wKCSf8B9z9kTRFYnN8m4s1bse2lrtvAOYCo1M2\nxem8sEdj8cbovDAKGGtmywmXKE4zs/tTyuTk2OZDgtozWK2ZdSFckJuVUqZ2sFrIcLDaNtRsvCnX\nGMYS2vvjahZwUaK32Uhgo7u/F3VQjTGz/rVt4WY2gvA3HslJKRHHPcBr7v7LRorF4vhmEmvMjm0/\nM+uVWO9GmPH79ZRisTkvZBJvXM4L7v5ddy919zLC+espd78wpVhOjm1GY/FFyfNssNoM473GzMYC\nNYl4L4kqXjN7kNA7q6+ZVQE/JFzAxd3vIIzBOAZYBmwFLo0m0iCDeM8FvmpmNcA2YFKEP1ZGAV8C\nXk5cewD4v8BgiN3xzSTWOB3bAcB9ZlZASJQz3P3xuJ4XyCze2JwX0oni2GqoIxERiaV8aOITEZEO\nSAlKRERiSQlKRERiSQlKRERiSQlKRERiSQlKRERiSQlKRERi6f8HdjWRcTDn3pEAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fcb305717d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(211)\n",
    "plt.title(\"accuracy\")\n",
    "plt.plot(history.history[\"acc\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_acc\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.subplot(212)\n",
    "plt.title(\"loss\")\n",
    "plt.plot(history.history[\"loss\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_loss\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy score: 0.504\n",
      "\n",
      "confusion matrix\n",
      "\n",
      "[[5595 4872]\n",
      " [5577 5012]]\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(linewidth=120)\n",
    "test_gen = datagen(Xtest, Ytest, docid2mat, batch_size=BATCH_SIZE)\n",
    "num_test_steps = len(Xtest) // BATCH_SIZE\n",
    "\n",
    "Ytest_ = model.predict_generator(test_gen, num_test_steps)\n",
    "\n",
    "ytest_ = np.argmax(Ytest_, axis=1)\n",
    "ytest = np.argmax(Ytest, axis=1)[0:len(ytest_)]\n",
    "\n",
    "print(\"accuracy score: {:.3f}\".format(accuracy_score(ytest, ytest_)))\n",
    "print(\"\\nconfusion matrix\\n\")\n",
    "print(confusion_matrix(ytest, ytest_))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}