src/05a-ng-sim-eep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Document Similarity - w/o Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division, print_function\n",
    "from keras import backend as K\n",
    "from keras.layers import Input\n",
    "from keras.layers.core import Dense, Dropout\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.layers.pooling import GlobalMaxPooling1D\n",
    "from keras.layers.wrappers import TimeDistributed, Bidirectional\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.models import Model\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "import logging\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import os\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_DIR = \"../data\"\n",
    "\n",
    "VOCAB_FILE = os.path.join(DATA_DIR, \"ng-vocab.tsv\")\n",
    "MIN_OCCURS = 5\n",
    "\n",
    "GLOVE_FILE = os.path.join(DATA_DIR, \"glove.840B.300d.txt\")\n",
    "\n",
    "DOCSIM_IDLABELS = os.path.join(DATA_DIR, \"docsim-idlabels.tsv\")\n",
    "DOCSIM_TEXTS = os.path.join(DATA_DIR, \"docsim-texts.tsv\")\n",
    "\n",
    "# covers about 95% of input data\n",
    "MAX_SENTS = 40 # maximum number of sentences per document\n",
    "MAX_WORDS = 60 # maximum number of words per sentence\n",
    "\n",
    "WORD_EMBED_SIZE = 300\n",
    "SENT_EMBED_SIZE = 100\n",
    "DOC_EMBED_SIZE = 50\n",
    "NUM_CLASSES = 2\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "NUM_EPOCHS = 5\n",
    "\n",
    "logging.basicConfig()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_size: 40730\n"
     ]
    }
   ],
   "source": [
    "word2id = {\"PAD\": 0, \"UNK\": 1}\n",
    "fvocab = open(VOCAB_FILE, \"rb\")\n",
    "for i, line in enumerate(fvocab):\n",
    "    word, count = line.strip().split(\"\\t\")\n",
    "    if int(count) <= MIN_OCCURS:\n",
    "        break\n",
    "    word2id[word] = i\n",
    "fvocab.close()\n",
    "id2word = {v:k for k, v in word2id.items()}\n",
    "vocab_size = len(word2id)\n",
    "print(\"vocab_size: {:d}\".format(vocab_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load GloVe Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40730, 300)\n"
     ]
    }
   ],
   "source": [
    "E = np.zeros((vocab_size, WORD_EMBED_SIZE))\n",
    "E[1] = np.random.random(WORD_EMBED_SIZE)\n",
    "fglove = open(GLOVE_FILE, \"rb\")\n",
    "for line in fglove:\n",
    "    cols = line.strip().split(\" \")\n",
    "    word = cols[0]\n",
    "    if not word2id.has_key(word):\n",
    "        continue\n",
    "    vec = np.array([float(x) for x in cols[1:]])\n",
    "    idx = word2id[word]\n",
    "    E[idx] = vec\n",
    "fglove.close()\n",
    "print(E.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute Document Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['like', 'a', 'mouse']\n",
      "['PAD', 'The', 'cat', 'fought', 'like', 'a', 'mouse']\n"
     ]
    }
   ],
   "source": [
    "def pad_or_truncate(xs, maxlen):\n",
    "    if len(xs) > maxlen:\n",
    "        xs = xs[len(xs) - maxlen:]\n",
    "    elif len(xs) < maxlen:\n",
    "        xs = [\"PAD\"] * (maxlen - len(xs)) + xs\n",
    "    return xs\n",
    "\n",
    "xs = [\"The\", \"cat\", \"fought\", \"like\", \"a\", \"mouse\"]\n",
    "print(pad_or_truncate(xs, 3))\n",
    "print(pad_or_truncate(xs, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1885 (40, 60)\n"
     ]
    }
   ],
   "source": [
    "docid2mat = {}\n",
    "ftext = open(DOCSIM_TEXTS, \"rb\")\n",
    "for line in ftext:\n",
    "    rec_id, text = line.strip().split(\"\\t\")\n",
    "    M = np.zeros((MAX_SENTS, MAX_WORDS))\n",
    "    sents = pad_or_truncate(nltk.sent_tokenize(text), MAX_SENTS)\n",
    "    for sid, sent in enumerate(sents):\n",
    "        words = pad_or_truncate(nltk.word_tokenize(sent), MAX_WORDS)\n",
    "        for wid, word in enumerate(words):\n",
    "            try:\n",
    "                word_id = word2id[word]\n",
    "            except KeyError:\n",
    "                word_id = word2id[\"UNK\"]\n",
    "            M[sid, wid] = word_id\n",
    "    docid2mat[int(rec_id)] = M\n",
    "ftext.close()\n",
    "print(len(docid2mat), docid2mat[list(docid2mat.keys())[0]].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract Label and DocID pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70253, 2) (70253, 2)\n"
     ]
    }
   ],
   "source": [
    "xdata, ydata = [], []\n",
    "fidl = open(DOCSIM_IDLABELS, \"rb\")\n",
    "for line in fidl:\n",
    "    label, docid_left, docid_right = line.strip().split(\"\\t\")\n",
    "    xdata.append((int(docid_left), int(docid_right)))\n",
    "    ydata.append(int(label))\n",
    "X = np.array(xdata)\n",
    "Y = to_categorical(np.array(ydata), num_classes=NUM_CLASSES)\n",
    "print(X.shape, Y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Partition into training, validation and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(44259, 2) (44259, 2) (4918, 2) (4918, 2) (21076, 2) (21076, 2)\n"
     ]
    }
   ],
   "source": [
    "Xtv, Xtest, Ytv, Ytest = train_test_split(X, Y, train_size=0.7)\n",
    "Xtrain, Xval, Ytrain, Yval = train_test_split(Xtv, Ytv, train_size=0.9)\n",
    "print(Xtrain.shape, Ytrain.shape, Xval.shape, Yval.shape, \n",
    "      Xtest.shape, Ytest.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Data Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 40, 60) (64, 40, 60) (64, 2)\n"
     ]
    }
   ],
   "source": [
    "def datagen(X, Y, docid2mat, batch_size=BATCH_SIZE):\n",
    "    while True:\n",
    "        num_recs = X.shape[0]\n",
    "        indices = np.random.permutation(np.arange(num_recs))\n",
    "        num_batches = num_recs // batch_size\n",
    "        for bid in range(num_batches):\n",
    "            batch_ids = indices[bid * batch_size : (bid + 1) * batch_size]\n",
    "            Xbatch_l = np.zeros((batch_size, MAX_SENTS, MAX_WORDS))\n",
    "            Xbatch_r = np.zeros((batch_size, MAX_SENTS, MAX_WORDS))\n",
    "            for idx, (docid_l, docid_r) in enumerate(X[batch_ids, :]):\n",
    "                Xbatch_l[idx] = docid2mat[docid_l]\n",
    "                Xbatch_r[idx] = docid2mat[docid_r]\n",
    "            Ybatch = Y[batch_ids, :]\n",
    "            yield [Xbatch_l, Xbatch_r], Ybatch\n",
    "\n",
    "train_gen = datagen(Xtrain, Ytrain, docid2mat)\n",
    "[Xbatch_left, Xbatch_right], Ybatch = train_gen.next()\n",
    "print(Xbatch_left.shape, Xbatch_right.shape, Ybatch.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Network\n",
    "\n",
    "### Sentence Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 60)                0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 60, 300)           12219000  \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 200)               240600    \n",
      "=================================================================\n",
      "Total params: 12,459,600\n",
      "Trainable params: 12,459,600\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "sent_in = Input(shape=(MAX_WORDS,), dtype=\"int32\")\n",
    "\n",
    "sent_emb = Embedding(input_dim=vocab_size,\n",
    "                       output_dim=WORD_EMBED_SIZE,\n",
    "                       weights=[E])(sent_in)\n",
    "\n",
    "sent_enc = Bidirectional(GRU(SENT_EMBED_SIZE,\n",
    "                               return_sequences=False))(sent_emb)\n",
    "\n",
    "sent_model = Model(inputs=sent_in, outputs=sent_enc)\n",
    "sent_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Document Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sum_over_axis(X, axis):\n",
    "    return K.mean(X, axis=axis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_2 (InputLayer)             (None, 40, 60)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "input_3 (InputLayer)             (None, 40, 60)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistribu (None, 40, 200)       12459600    input_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistribu (None, 40, 200)       12459600    input_3[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional)  (None, 40, 100)       75300       time_distributed_1[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional)  (None, 40, 100)       75300       time_distributed_2[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 40, 200)       0           bidirectional_2[0][0]            \n",
      "                                                                   bidirectional_3[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalMa (None, 200)           0           concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 200)           0           global_max_pooling1d_1[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 50)            10050       dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 50)            0           dense_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 2)             102         dropout_2[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 12,620,352\n",
      "Trainable params: 12,620,352\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# LHS document\n",
    "doc_in_l = Input(shape=(MAX_SENTS, MAX_WORDS), dtype=\"int32\")\n",
    "\n",
    "doc_emb_l = TimeDistributed(sent_model)(doc_in_l)\n",
    "\n",
    "doc_enc_l = Bidirectional(GRU(DOC_EMBED_SIZE,\n",
    "                              return_sequences=True))(doc_emb_l)\n",
    "\n",
    "# RHS document\n",
    "doc_in_r = Input(shape=(MAX_SENTS, MAX_WORDS), dtype=\"int32\")\n",
    "\n",
    "doc_emb_r = TimeDistributed(sent_model)(doc_in_r)\n",
    "\n",
    "doc_enc_r = Bidirectional(GRU(DOC_EMBED_SIZE,\n",
    "                              return_sequences=True))(doc_emb_r)\n",
    "\n",
    "# Merge without attention\n",
    "doc_mrg = concatenate([doc_enc_l, doc_enc_r], axis=2)\n",
    "doc_mrg = GlobalMaxPooling1D()(doc_mrg)\n",
    "\n",
    "# Prediction\n",
    "fc1_dropout = Dropout(0.2)(doc_mrg)\n",
    "fc1 = Dense(50, activation=\"relu\")(fc1_dropout)\n",
    "fc2_dropout = Dropout(0.2)(fc1)\n",
    "doc_pred = Dense(NUM_CLASSES, activation=\"softmax\")(fc2_dropout)\n",
    "\n",
    "model = Model(inputs=[doc_in_l, doc_in_r], outputs=doc_pred)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\",\n",
    "             metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "691/691 [==============================] - 940s - loss: 0.1767 - acc: 0.9084 - val_loss: 0.0454 - val_acc: 0.9860\n",
      "Epoch 2/5\n",
      "691/691 [==============================] - 935s - loss: 0.0378 - acc: 0.9901 - val_loss: 0.0399 - val_acc: 0.9883\n",
      "Epoch 3/5\n",
      "691/691 [==============================] - 935s - loss: 0.0332 - acc: 0.9913 - val_loss: 0.0414 - val_acc: 0.9870\n",
      "Epoch 4/5\n",
      "691/691 [==============================] - 934s - loss: 0.0292 - acc: 0.9925 - val_loss: 0.0356 - val_acc: 0.9912\n",
      "Epoch 5/5\n",
      "691/691 [==============================] - 933s - loss: 0.0262 - acc: 0.9933 - val_loss: 0.0397 - val_acc: 0.9883\n"
     ]
    }
   ],
   "source": [
    "train_gen = datagen(Xtrain, Ytrain, docid2mat, batch_size=BATCH_SIZE)\n",
    "val_gen = datagen(Xval, Yval, docid2mat, batch_size=BATCH_SIZE)\n",
    "\n",
    "num_train_steps = len(Xtrain) // BATCH_SIZE\n",
    "num_val_steps = len(Xval) // BATCH_SIZE\n",
    "\n",
    "history = model.fit_generator(train_gen, \n",
    "                              steps_per_epoch=num_train_steps,\n",
    "                              epochs=NUM_EPOCHS,\n",
    "                              validation_data=val_gen,\n",
    "                              validation_steps=num_val_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPW5x/HPQ/YAIptiCRqsXhW1FzVSet2oSwUXsKKi\nVaveXu2i16W1Fttbba322uWqtVfryq0rSnGjFktdQHtv0RIsKq6A1RJQCSgYlEBCnvvH7wSGYUIm\nmcmcSeb7fr3OK2fO+Z2ZZ04y881Zf+buiIiI5JtecRcgIiKSigJKRETykgJKRETykgJKRETykgJK\nRETykgJKRETykgJKRETykgJKRETykgJKJM9ZoM+qFBz90Yukycwmm9kSM2sws9fM7MsJ8841s9cT\n5u0fTR9mZg+bWb2ZrTKz/46m/8jM7k1YvtrM3MyKo8dzzOwaM/s/4FNgVzM7J+E13jazryfVN8HM\nFpjZx1GdY83sZDObn9Tu22b2WNetKZHsKI67AJFuZAlwCPA+cDJwr5ntBhwM/Ag4AagFPgs0mVkR\n8DjwDHAmsBGo6cDrnQmMA94EDNgDOA54GzgUeMLM5rn7i2Y2CrgbOAl4GtgJ6Av8HbjVzPZy99cT\nnvfqzqwAkVzSFpRImtz9d+6+3N1b3P1BYBEwCvg34OfuPs+Dxe7+bjTvM8B33f0Td2909//twEv+\n1t1fdfdmd29y9z+4+5LoNZ4F/kQITICvAVPc/cmovmXu/oa7rwceBM4AMLO9gWpCcIrkNQWUSJrM\n7KvRLrTVZrYa2AcYBAwjbF0lGwa86+7NnXzJpUmvP87MnjezD6PXPyZ6/dbXSlUDwF3AV8zMCFtP\n06LgEslrCiiRNJjZLsDtwAXAQHffHlhI2PW2lLBbL9lSYOfW40pJPgEqEx4PSdFmU1cDZlYGPAT8\nEtgxev2Z0eu3vlaqGnD354ENhK2trwD3pH6XIvlFASWSnt6EwKgHMLNzCFtQAHcAl5rZAdEZd7tF\ngfZX4D3gWjPrbWblZnZQtMwC4FAz29nM+gGXt/P6pUBZ9PrNZjYO+FLC/DuBc8zsCDPrZWZDzWzP\nhPl3A/8NNHVwN6NIbBRQImlw99eA/wLmAh8A+wL/F837HXANcD/QADwKDHD3jcDxwG7AP4A6YFK0\nzJOEY0MvA/Np55iQuzcAFwLTgI8IW0IzEub/FTgHuB5YAzwL7JLwFPcQAvVeRLoJU4eFIj2fmVUA\nK4D93X1R3PWIpENbUCKF4ZvAPIWTdCe6DkqkhzOzdwgnU5wQcykiHaJdfCIikpe0i09ERPJSt9rF\nN2jQIK+uro67DBERycD8+fNXuvvg9tp1q4Cqrq6mtrY27jJERCQDZvZuOu20i09ERPJSt9qCEhGR\nHGtpCcPGjZvHe/fOyUsroEQkf7W0QHNzGDZu3Ho81bTk+a1D8pds8rSOtMmneV39usnKyqCxMSe/\nfgWUSNzc2/+i7ciXcke/wPN5fnfUq1cYiopS/8zWvJKSzJ+zM8sX5y42FFASH/fN/yEnfjlta0in\nXTafKxftNm6M+zexpV69wpdQ65dR4niqaW3Nr6jo+PIJ4xutmLUtlTRsrGRtczkNzRU0NFewtrmM\nhqYKGjaUsbapjIYNpTSsL2PthhIaGktpWF/C2sYSGhqLWdtYjBlUlLVQUe5UlLVQXubRuFNR4VSU\ns+lneXkou6ICKipt889oKK+Ixnv3Cj/7FFHRuxdFpQlf5pI1CijpvBtugOnTM/vSzgetX8jJQ+KX\n5raGoqKw2yOddh1pk2lAdHa+WfvrLIWmJmhogLVrw8+0xte03WbduvRfu3dv6NMH+vYNQ5/+MKRv\nmOYenquxMfxcvQ7WfRTGW4fGRtiwoVNvGwgbM1uEW4phW/M7M68QslABJR3nDldcAVdfDfvtBzvs\n0Lkv9Wy26+xztu66KDDusH59FAgfdiJYUoyvT7MLRLMoRBIDpQ/svHPq6e2NV1aGX2OmNm7cHGLJ\n4ZU8LZ15ifM//DD1vKamztdbWppZ8HV02fLy3H9UFFDdkHv4b2/9+uwMHXsuZ/37H7F+zTdZX/Y9\nWv7em+Kl1mY+pDOts8vF/ZoZbGx06ne+bl1mAZI8nu4GbHHxlqHQ+nPHHTseJn36hEDJ1XrriKKi\nsCWWoxPUgPA7yGYots775BNYuTL1/Ex2XJSVhd9jfX321sG2KKDa4R5+odkKg2wERia7IpL16hX+\n6Nob+vSBsjKn7PWXKFvzMmV77krZ4QfRq8jaPM69rWmtH5SOLpdvx85T7TnLJADNwpdLqlBJdUJV\nKq2/r8RQ6NcPqqo6FyhlZfkZKD1BcXFYx3365O41m5s7H3zr1uX2kGnBBNSnn8J553UuLLJ1P12z\nrb/4S0u3ntavX3qhkelQWho+IGlxh0sugcd+BeefD78+M7ZvrcRzK9o656AjYdfRadl+/sSwbr3E\nZPBgGD6842HSt284HiLSltYt4r59466kfQUTUABz5279Jd23LwwcmJtAaP0PudtpaYELL4SbboKL\nLoLrr4/1jZiFLY/WcxNEpGcqmICqrIQlS+KuohtqaYFvfQtuvRUuvRR+/vNumrIi0t0U3ulLkr6W\nlrBf9NZbYfJkhZOI5JQCSlLbuBHOOQfuvBN++EP46U8VTiKSUwWzi086oLkZzjoL7r8ffvzjcM2T\niEiOKaBkS01NcOaZ8OCDcM018P3vx12RiBQoBZRs1tQEp50GDz0EP/sZXHZZ3BWJSAFTQEmwYQNM\nmgSPPgrXXReueRIRiVFaJ0mY2Vgze9PMFpvZ5BTzdzGzp83sZTObY2ZV0fQvmtmChKHRzE6I5v3W\nzP6eMG9kdt+apG39epg4MYTTjTcqnEQkL7S7BWVmRcBNwFFAHTDPzGa4+2sJzX4J3O3ud5nZ4cB/\nAme6+2xgZPQ8A4DFwJ8Slvuuu0/PzluRTmlshBNPhCeegJtvhm9+M+6KRESA9LagRgGL3f1td98A\nPABMSGozAngmGp+dYj7AScAT7v5pZ4uVLFu3DsaPhz/+EW67TeEkInklnYAaCixNeFwXTUv0EnBi\nNP5loK+ZDUxqcyowNWnaNdFuwevNLOVNa8zsPDOrNbPa+lzdQrcQfPIJHHccPPVUuNbp3HPjrkhE\nZAvZulD3UuAwM/sbcBiwDNh0z1sz2wnYF5iVsMzlwJ7AgcAA4Hupntjdb3P3GnevGTx4cJbKLXBr\n18Ixx8CcOXDXXeGCXBGRPJPOWXzLgGEJj6uiaZu4+3KiLSgz6wNMdPfVCU1OAR5x96aEZd6LRteb\n2f8QQk66WkMDjBsHzz8P994bTisXEclD6WxBzQN2N7PhZlZK2FU3I7GBmQ0ys9bnuhyYkvQcp5G0\ney/aqsLMDDgBWNjx8qVD1qyBL30phNPUqQonEclr7QaUuzcDFxB2z70OTHP3V83sKjMbHzUbA7xp\nZm8BOwLXtC5vZtWELbBnk576PjN7BXgFGARcndE7kW1bvRqOOgpqa2HaNDj55LgrEhHZJvNs9caX\nAzU1NV5bWxt3Gd3Phx+GcHrlFZg+PZy5JyISEzOb7+417bXTnSR6upUrQzi99ho88ggce2zcFYmI\npEUB1ZOtWAFHHglvvQUzZsDRR8ddkYhI2hRQPdX778MRR8Df/w6PPx6CSkSkG1FA9UTLl8Phh8PS\npTBzJowZE3dFIiIdpoDqaerqQjgtXx5uYXTIIXFXJCLSKQqonuQf/4AvfhHq6+FPf4J/+Ze4KxIR\n6TQFVE/xzjshnD78EJ58Ej7/+bgrEhHJiAKqJ3j77RBOH38MTz8NNe1eXiAikvcUUN3dokXhmNOn\nn4Zw2n//uCsSEckKBVR39uabYcupqQmeeQb++Z/jrkhEJGsUUN3Va6+FLSd3mD0b9tkn7opERLIq\nW/1BSS4tXBiubTILfTopnESkB1JAdTcvvRR265WUhHDaa6+4KxIR6RIKqO7kxRfDbr3ycnj2Wdhj\nj7grEhHpMgqo7qK2Ntxbr0+fEE677RZ3RSIiXUoB1R288EK42Wv//iGcdt017opERLpcWgFlZmPN\n7E0zW2xmk1PM38XMnjazl81sjplVJczbaGYLomFGwvThZvZC9JwPRt3JS7L/+7/Qn9OgQeGYU3V1\n3BWJiOREuwFlZkXATcA4YARwmpmNSGr2S+Bud/8ccBXwnwnz1rn7yGhI7Mr1Z8D17r4b8BHwtQze\nR8/03HOhD6chQ8KW0847x12RiEjOpLMFNQpY7O5vu/sG4AFgQlKbEcAz0fjsFPO3YGYGHA5Mjybd\nBZyQbtEFYfZsGDcOhg0L4TR0aNwViYjkVDoBNRRYmvC4LpqW6CXgxGj8y0BfMxsYPS43s1oze97M\nWkNoILDa3Zu38ZyF66mnQtfs1dVht95OO8VdkYhIzmXrJIlLgcPM7G/AYcAyYGM0bxd3rwG+Atxg\nZp/tyBOb2XlRwNXW19dnqdw8NmsWHH98OEtv9mzYcce4KxIRiUU6AbUMGJbwuCqatom7L3f3E919\nP+AH0bTV0c9l0c+3gTnAfsAqYHszK27rOROe+zZ3r3H3msGDB6f7vrqnmTNh/HjYc89wb70ddoi7\nIhGR2KQTUPOA3aOz7kqBU4EZiQ3MbJCZtT7X5cCUaHp/MytrbQMcBLzm7k44VnVStMxZwGOZvplu\nbcYMOOEE2HffcFfyQYPirkhEJFbtBlR0nOgCYBbwOjDN3V81s6vMrPWsvDHAm2b2FrAjcE00fS+g\n1sxeIgTSte7+WjTve8C3zWwx4ZjUnVl6T93PI4/AxImw337h+NOAAXFXJCISOwsbM91DTU2N19bW\nxl1Gdv3ud3DaaTBqFDzxBPTrF3dFIiJdyszmR+cmbJPuJBGnqVNDOH3hC+HkCIWTiMgmCqi43HMP\nnHEGHHxw2HLq2zfuikRE8ooCKg7/8z9w1lmhT6c//CHcAFZERLagHnVz7fbb4bzzwv31Hn0UKivj\nrkhEcqypqYm6ujoaGxvjLqVLlZeXU1VVRUlJSaeWV0Dl0m9+A9/6VriF0cMPh36dRKTg1NXV0bdv\nX6qrqwl3fut53J1Vq1ZRV1fH8OHDO/Uc2sWXK7/+dQin444Lp5UrnEQKVmNjIwMHDuyx4QRgZgwc\nODCjrUQFVC5cfz1ceGG4EPehh6CsLO6KRCRmPTmcWmX6HhVQXe3nP4dvfztciDttGpSq2ysRkXQo\noLrST38K3/seTJoUrnnq5IFCEZFsWr16NTfffHOHlzvmmGNYvXp1F1SUmgKqq1x1FfzgB3D66XDv\nvQonEckbbQVUc3NzitabzZw5k+23376rytqKzuLLNne48kr4yU/CtU533glFRXFXJSKyyeTJk1my\nZAkjR46kpKSE8vJy+vfvzxtvvMFbb73FCSecwNKlS2lsbOSiiy7ivPPOA6C6upra2lrWrl3LuHHj\nOPjgg/nLX/7C0KFDeeyxx6ioqMhqnQqobHKH738frr0WvvY1uO026KWNVBHZhosvhgULsvucI0fC\nDTe0Ofvaa69l4cKFLFiwgDlz5nDssceycOHCTaeDT5kyhQEDBrBu3ToOPPBAJk6cyMCBA7d4jkWL\nFjF16lRuv/12TjnlFB566CHOOOOMrL4NBVS2uMNll8Evfwlf/zrcfLPCSUS6hVGjRm1xrdKNN97I\nI488AsDSpUtZtGjRVgE1fPhwRo4cCcABBxzAO++8k/W6FFDZ4A6XXAK/+hWcf3645qkATiEVkSzY\nxpZOrvTu3XvT+Jw5c3jqqaeYO3culZWVjBkzJuW1TGUJl8sUFRWxbt26rNelf/Ez1dIC//7vIZwu\nukjhJCJ5r2/fvjQ0NKSct2bNGvr3709lZSVvvPEGzz//fI6r20xbUJloaQl3h7j1Vrj00nDNk8JJ\nRPLcwIEDOeigg9hnn32oqKhgxx133DRv7Nix3HLLLey1117ssccejB49OrY61WFhZ7W0hJu+3nkn\nXH45XHONwklE0vL666+z1157xV1GTqR6r1ntsNDMxprZm2a22Mwmp5i/i5k9bWYvm9kcM6uKpo80\ns7lm9mo0b1LCMr81s7+b2YJoGJlOLXlh40b4138N4fTDHyqcRES6QLsBZWZFwE3AOGAEcJqZjUhq\n9kvgbnf/HHAV8J/R9E+Br7r73sBY4AYzS7zK67vuPjIasnyeZRdpbg7XN911F/z4x+GCXIWTiEjW\npbMFNQpY7O5vu/sG4AFgQlKbEcAz0fjs1vnu/pa7L4rGlwMrgMHZKDwWzc2hF9z77gtbTVdcEXdF\nIiI9VjoBNRRYmvC4LpqW6CXgxGj8y0BfM9vipHkzGwWUAksSJl8T7fq73sxS3uLbzM4zs1ozq62v\nr0+j3C7S1ASnngoPPhhOhvj+9+OrRUSkAGTrNPNLgcPM7G/AYcAyYGPrTDPbCbgHOMfdW6LJlwN7\nAgcCA4DvpXpid7/N3WvcvWbw4Jg2vjZsgFNOCV1lXHcdfPe78dQhIlJA0jnNfBkwLOFxVTRtk2j3\n3YkAZtYHmOjuq6PH2wF/AH7g7s8nLPNeNLrezP6HEHL5Z/16OOkkePxxuPHGcM2TiIh0uXS2oOYB\nu5vZcDMrBU4FZiQ2MLNBZtb6XJcDU6LppcAjhBMopicts1P004ATgIWZvJEu0dgIX/5yCKff/Ebh\nJCIFqU+fPrG8brsB5e7NwAXALOB1YJq7v2pmV5nZ+KjZGOBNM3sL2BG4Jpp+CnAocHaK08nvM7NX\ngFeAQcDV2XpTWbFuHYwfD3/8I9x+O3zjG3FXJCJSUNK6k4S7zwRmJk27ImF8OjA9xXL3Ave28ZyH\nd6jSXPrkkxBOs2fDlClw9tlxVyQikjWTJ09m2LBhnH/++QD86Ec/ori4mNmzZ/PRRx/R1NTE1Vdf\nzYQJySds55ZudZRs7Vo47jj485/h7rvDaeUiIl0kht42mDRpEhdffPGmgJo2bRqzZs3iwgsvZLvt\ntmPlypWMHj2a8ePHYzFe56mAStTQAMccA3Pnhl5wTzst7opERLJuv/32Y8WKFSxfvpz6+nr69+/P\nkCFDuOSSS3juuefo1asXy5Yt44MPPmDIkCGx1amAarVmDYwbB/PmwdSpcPLJcVckIgUgrt42Tj75\nZKZPn87777/PpEmTuO+++6ivr2f+/PmUlJRQXV2dspuNXFJAAaxeDUcfDS++CNOmhTP3RER6sEmT\nJnHuueeycuVKnn32WaZNm8YOO+xASUkJs2fP5t133427RAUUH34IRx0Fr7wSLsQdP779ZUREurm9\n996bhoYGhg4dyk477cTpp5/O8ccfz7777ktNTQ177rln3CUWeECtXBnC6fXX4dFHw/EnEZEC8cor\nr2waHzRoEHPnzk3Zbu3atbkqaQuFG1ArVsCRR8KiRfDYY2EXn4iI5I3CDKgPPoDDD4e//x1+//sQ\nVCIiklcKL6Deey+E0z/+ATNnwpgxcVckIgXI3WO9xigXMu2xPVt3M+8eli2Dww6DurpwCyOFk4jE\noLy8nFWrVmX8BZ7P3J1Vq1ZRXl7e6econC2o5ctDOK1YAbNmwb/8S9wViUiBqqqqoq6ujlj7uMuB\n8vJyqqqqOr184QTUgAGw//7wne/A5z8fdzUiUsBKSkoYPnx43GXkvcIJqPLycBGuiIh0C4V1DEpE\nRLoNBZSIiOQl605nkZhZPZDpDaIGASuzUE5XU53ZpTqzr7vUqjqzKxt17uLug9tr1K0CKhvMrNbd\na+Kuoz2qM7tUZ/Z1l1pVZ3blsk7t4hMRkbykgBIRkbxUiAF1W9wFpEl1ZpfqzL7uUqvqzK6c1Vlw\nx6BERKR7KMQtKBER6QYUUCIikpd6ZECZ2Vgze9PMFpvZ5BTzy8zswWj+C2ZWnfsqN9XSXq1nm1m9\nmS2Ihn+LocYpZrbCzBa2Md/M7MboPbxsZvvnusaojvbqHGNmaxLW5RW5rjGqY5iZzTaz18zsVTO7\nKEWb2NdpmnXmyzotN7O/mtlLUa0/TtEm9s99mnXG/plPqKXIzP5mZo+nmNf169Pde9QAFAFLgF2B\nUuAlYERSm28Bt0TjpwIP5nGtZwP/HfM6PRTYH1jYxvxjgCcAA0YDL+RpnWOAx+Ncl1EdOwH7R+N9\ngbdS/N5jX6dp1pkv69SAPtF4CfACMDqpTeyf+zTrjP0zn1DLt4H7U/2Oc7E+e+IW1Chgsbu/7e4b\ngAeACUltJgB3RePTgSMsnp7D0qk1du7+HPDhNppMAO724HlgezPbKTfVbZZGnXnB3d9z9xej8Qbg\ndWBoUrPY12madeaFaD2tjR6WREPyGWCxf+7TrDMvmFkVcCxwRxtNunx99sSAGgosTXhcx9Yfqk1t\n3L0ZWAMMzEl1bdQRSVUrwMRoN890MxuWm9I6JN33kQ++EO1eecLM9o67mGi3yH6E/6QT5dU63Uad\nkCfrNNodtQBYATzp7m2u0zg/92nUCfnxmb8BuAxoaWN+l6/PnhhQPc3vgWp3/xzwJJv/Y5GOe5Fw\nD7B/Bn4NPBpnMWbWB3gIuNjdP46zlm1pp868WafuvtHdRwJVwCgz2yeuWrYljTpj/8yb2XHACnef\nn+vXTtQTA2oZkPgfR1U0LWUbMysG+gGrclJdG3VEtqrV3Ve5+/ro4R3AATmqrSPSWeexc/ePW3ev\nuPtMoMTMBsVRi5mVEL7073P3h1M0yYt12l6d+bROE2paDcwGxibNypfPPdB2nXnymT8IGG9m7xAO\nPRxuZvcmteny9dkTA2oesLuZDTezUsLBuxlJbWYAZ0XjJwHPeHSkL8farTXpuMN4wnGAfDMD+Gp0\n5tloYI27vxd3UcnMbEjrPnIzG0X4+8/5F1RUw53A6+5+XRvNYl+n6dSZR+t0sJltH41XAEcBbyQ1\ni/1zn06d+fCZd/fL3b3K3asJ30vPuPsZSc26fH32uB513b3ZzC4AZhHOkpvi7q+a2VVArbvPIHzo\n7jGzxYSD6qfmca0Xmtl4oDmq9exc12lmUwlnaw0yszrgSsLBXdz9FmAm4ayzxcCnwDm5rjHNOk8C\nvmlmzcA64NSY/jE5CDgTeCU6FgHwfWDnhFrzYZ2mU2e+rNOdgLvMrIgQktPc/fE8/NynU2fsn/m2\n5Hp96lZHIiKSl3riLj4REekBFFAiIpKXFFAiIpKXFFAiIpKXFFAiIpKXFFAiIpKXFFAiIpKXFFAi\nIpKXFFAiIpKXFFAiIpKXFFAiIpKXFFAiIpKXFFAiOWBm75jZkXHXIdKdKKBERCQvKaBERCQvKaBE\ncsjMyszsBjNbHg03mFlZNG+QmT1uZqvN7EMz+7OZ9Yrmfc/MlplZg5m9aWZHxPtORLpej+tRVyTP\n/QAYDYwEHHgM+A/gh8B3gDpgcNR2NOBmtgdwAXCguy83s2pCD8wiPZq2oERy63TgKndf4e71wI8J\n3aoDNBG6BN/F3Zvc/c9R9+kbgTJghJmVuPs77r4klupFckgBJZJbnwHeTXj8bjQN4BfAYuBPZva2\nmU0GcPfFwMXAj4AVZvaAmX0GkR5OASWSW8uBXRIe7xxNw90b3P077r4rMB74duuxJne/390PjpZ1\n4Ge5LVsk9xRQIrk1FfgPMxtsZoOAK4B7AczsODPbzcwMWEPYtddiZnuY2eHRyRSNwDqgJab6RXJG\nASWSW1cDtcDLwCvAi9E0gN2Bp4C1wFzgZnefTTj+dC2wEngf2AG4PLdli+SehWOwIiIi+UVbUCIi\nkpcUUCIikpcUUCIikpcUUCIikpe61a2OBg0a5NXV1XGXISIiGZg/f/5Kdx/cXrtuFVDV1dXU1tbG\nXYaIiGTAzN5tv5V28YmISJ4qnIByh5/9DJYvj7sSERFJQ+EE1DvvwDXXwMEHw9tvx12NiIi0o1sd\ng8rI8OHw9NMwdiwccgg8+SSMGBF3VSJSgJqamqirq6OxsTHuUrpUeXk5VVVVlJSUdGr5wgkogAMP\nhGefhaOOgkMPhVmz4IAD4q5KRApMXV0dffv2pbq6mnBv4J7H3Vm1ahV1dXUMHz68U89ROLv4Wu2z\nD/zv/0LfvnD44fDnP8ddkYgUmMbGRgYOHNhjwwnAzBg4cGBGW4mFF1AAn/1sCKbPfAaOPhr++Me4\nKxKRAtOTw6lVpu+xMAMKoKoKnnsO9twTxo+Hhx6KuyIREUlQuAEFMHgwPPMMjBoFp5wCv/1t3BWJ\niHS51atXc/PNN3d4uWOOOYbVq1d3QUWpFXZAAWy/fThZ4ogj4Jxz4Ne/jrsiEZEu1VZANTc3b3O5\nmTNnsv3223dVWVsprLP42tK7N/z+93DaaXDhhbBmDfzgB1AA+4hFpPBMnjyZJUuWMHLkSEpKSigv\nL6d///688cYbvPXWW5xwwgksXbqUxsZGLrroIs477zxg8+3m1q5dy7hx4zj44IP5y1/+wtChQ3ns\nsceoqKjIap0KqFZlZTBtGvzrv8IPfxhC6uc/V0iJSNe6+GJYsCC7zzlyJNxwQ5uzr732WhYuXMiC\nBQuYM2cOxx57LAsXLtx0OviUKVMYMGAA69at48ADD2TixIkMHDhwi+dYtGgRU6dO5fbbb+eUU07h\noYce4owzzsjq21BAJSouDsehttsOfvlLaGiAm26CoqK4KxMR6TKjRo3a4lqlG2+8kUceeQSApUuX\nsmjRoq0Cavjw4YwcORKAAw44gHfeeSfrdSmgkvXqFY5D9esHP/0pfPwx3HUXdPJKaBGRbdrGlk6u\n9O7de9P4nDlzeOqpp5g7dy6VlZWMGTMm5bVMZWVlm8aLiopYt25d1uvK6CQJMxtrZm+a2WIzm5xi\n/qFm9qKZNZvZSUnzNprZgmiYkUkdWWcW7tt37bUwdSpMnAg9/JYkIlI4+vbtS0NDQ8p5a9asoX//\n/lRWVvLGG2/w/PPP57i6zTq9BWVmRcBNwFFAHTDPzGa4+2sJzf4BnA1cmuIp1rn7yM6+fk5873th\nd9/558Oxx8Jjj0GfPnFXJSKSkYEDB3LQQQexzz77UFFRwY477rhp3tixY7nlllvYa6+92GOPPRg9\nenRsdWayi28UsNjd3wYwsweACcCmgHL3d6J5LRm8Try++c1wW6Szz4Yjj4SZM2HAgLirEhHJyP33\n359yelkunZNoAAASAklEQVRZGU888UTKea3HmQYNGsTChQs3Tb/00lTbIJnLZBffUGBpwuO6aFq6\nys2s1syeN7MTMqij651xRrjTxN/+BmPGwPvvx12RiEiPF+eFuru4ew3wFeAGM/tsqkZmdl4UZLX1\n9fW5rTDRhAnwhz/AkiXhTuj/+Ed8tYiIFIBMAmoZMCzhcVU0LS3uviz6+TYwB9ivjXa3uXuNu9cM\nHjy489Vmw5FHwlNPQX196PjwrbfirUdEpAfLJKDmAbub2XAzKwVOBdI6G8/M+ptZWTQ+CDiIhGNX\nee0LX4DZs8NZfYccAi+9FHdFIiI9UqcDyt2bgQuAWcDrwDR3f9XMrjKz8QBmdqCZ1QEnA7ea2avR\n4nsBtWb2EjAbuDbp7L/8NnJk6K6jtDQck4rxNEwRkZ4qowt13X0mMDNp2hUJ4/MIu/6Sl/sLsG8m\nrx27PfYIHR8eeWQYHnss3HBWRESyQnczz8Quu4QtqeHD4ZhjYEZ+XW8sIpINfWK6/lMBlakhQ+DZ\nZ8NuvxNPhDauLRARkY7RvfiyYcCAcHbf+PHhmqmGBvj61+OuSkQkpcmTJzNs2DDOP/98AH70ox9R\nXFzM7Nmz+eijj2hqauLqq69mwoQJsdapgMqWvn3DXSZOOQW+8Y3QXcdll8VdlYjkuRh622DSpElc\nfPHFmwJq2rRpzJo1iwsvvJDtttuOlStXMnr0aMaPH4/F2OWQAiqbKirg4Yfhq18N9/Fbswauvlp9\nSolIXtlvv/1YsWIFy5cvp76+nv79+zNkyBAuueQSnnvuOXr16sWyZcv44IMPGDJkSGx1KqCyraQE\n7r03bFG1dtfxq1+FbjxERJLE1dvGySefzPTp03n//feZNGkS9913H/X19cyfP5+SkhKqq6tTdrOR\nSwqorlBUBLfeGu6E/l//FY5J3XFH6BBRRCQPTJo0iXPPPZeVK1fy7LPPMm3aNHbYYQdKSkqYPXs2\n7777btwlKqC6jBn84heh48Mrrgghdf/9oWt5EZGY7b333jQ0NDB06FB22mknTj/9dI4//nj23Xdf\nampq2HPPPeMuUQHVpczghz8MW1IXXxzO8nv4YUjovVJEJC6vvPLKpvFBgwYxd+7clO3Wrl2bq5K2\noAMjuXDRRTBlSjgV/eijw8kTIiKyTQqoXDnnHHjgAfjrX+Hww8Md0UVEpE0KqFw6+eRwz77XXgt9\nSi1Lu3cSEelh3D3uErpcpu9RAZVr48bBrFkhnA4+OHSAKCIFpby8nFWrVvXokHJ3Vq1aRXl5eaef\nQydJxOHQQ+GZZ2Ds2NCn1JNPwt57x12ViORIVVUVdXV1xNpLeA6Ul5dTVbVVhxZpU0DFpaYm3GT2\nqKNCYM2aFaaJSI9XUlLC8OHD4y4j72kXX5z23jv0KdWvXzhx4rnn4q5IRCRvKKDituuuoU+pqqpw\nCvrMme0vIyJSABRQ+WDo0LC7b8QImDABfve7uCsSEYmdAipfDB4cTpwYPRpOPTVc2CsiUsAUUPmk\nX79wssRRR8HXvhbfbY5FRPJARgFlZmPN7E0zW2xmk1PMP9TMXjSzZjM7KWneWWa2KBrOyqSOHqWy\nMlzMO3EiXHIJXHUV9OBrJURE2tLpgDKzIuAmYBwwAjjNzEYkNfsHcDZwf9KyA4Argc8Do4Arzax/\nZ2vpccrKwm2RzjoLrrwSLr1UISUiBSeT66BGAYvd/W0AM3sAmAC81trA3d+J5rUkLXs08KS7fxjN\nfxIYC0zNoJ6epbg4HIfabju47rrQ8eEtt4S+pkRECkAmATUUWJrwuI6wRdTZZYemamhm5wHnAey8\n884dr7I769Ur9Mbbr1/oOr6hAe65J/TaKyLSw+X9nSTc/TbgNoCamprC289lBj/5SdiSuuwyWLs2\nnIZeURF3ZSIiXSqTkySWAcMSHldF07p62cL03e+GXXwzZ8Ixx4StKRGRHiyTgJoH7G5mw82sFDgV\nmJHmsrOAL5lZ/+jkiC9F02Rbvv51uPfecOeJI46AVavirkhEpMt0OqDcvRm4gBAsrwPT3P1VM7vK\nzMYDmNmBZlYHnAzcamavRst+CPyEEHLzgKtaT5iQdnzlK6Hb+JdfhjFj4L334q5IRKRLWHfqj6Sm\npsZra2vjLiM/PP10uC3SkCGhK/nq6rgrEhFJi5nNd/d2u2/QnSS6qyOOCMG0alXoU+rNN+OuSEQk\nqxRQ3dno0eEmsxs2hJBasCDuikREskYB1d197nPhpIny8nBM6i9/ibsiEZGsUED1BP/0T6Hjwx12\nCDeafeqpuCsSEcmYAqqn2HnnsCW1225w7LHw6KNxVyQikhEFVE+y444wZw7svz+cdFK4ZkpEpJtS\nQPU0/fvDk0/CYYfBmWfCzTfHXZGISKcooHqiPn3gD3+A44+H88+Ha6+NuyIRkQ5TQPVU5eXw0EPh\nzhOXXx6GbnRRtohI3t/NXDJQUgJ33w19+4atqI8/hl//OnTjISKS5xRQPV1REfzmN6G7jl/8ItwF\nfcqU0CGiiEge07dUITCDn/0sdHz4H/8RQuqBB0LX8iIieUr7egqFGfzgB3DjjeEaqeOPh08+ibsq\nEZE2KaAKzb//O/z2t+Fu6F/6EqxeHXdFIiIpKaAK0VlnwbRpMG8efPGLsGJF3BWJiGxFAVWoJk6E\n3/8+dNNx6KGwdGncFYmIbEEBVciOPhpmzQq98h5yCCxeHHdFIiKbKKAK3SGHwOzZsHZtGF+4MO6K\nREQABZRAuLnsc8+FC3gPOwz++te4KxIRUUBJZMSI0KfU9tuH7uTnzIm7IhEpcBkFlJmNNbM3zWyx\nmU1OMb/MzB6M5r9gZtXR9GozW2dmC6LhlkzqkCwZPjz0KbXzzjBuXLjhrIhITDodUGZWBNwEjANG\nAKeZ2YikZl8DPnL33YDrgZ8lzFvi7iOj4RudrUOy7DOfgWefhb33hhNOgAcfjLsiESlQmWxBjQIW\nu/vb7r4BeACYkNRmAnBXND4dOMLMLIPXlFwYNAieeQa+8AU47TS44464KxKRApRJQA0FEi+eqYum\npWzj7s3AGmBgNG+4mf3NzJ41s0PaehEzO8/Mas2str6+PoNypUO22w7++MdwKvq558J118VdkYgU\nmLhuFvsesLO7rzKzA4BHzWxvd/84uaG73wbcBlBTU9PpDo02bgyHVCorw1BRsfV4ebl6othCZSU8\n9hicfjp85zuhu44rrwz39RMR6WKZBNQyYFjC46poWqo2dWZWDPQDVrm7A+sB3H2+mS0B/gmozaCe\nbfrkE5iQvAMyhfLybYdYtsZLSrrqnWZZaSlMnRr6lPrxj2HNmrA1pZASkS6WSUDNA3Y3s+GEIDoV\n+EpSmxnAWcBc4CTgGXd3MxsMfOjuG81sV2B34O0MamlXZSXU1sKnn4Zh3bqOj3/0Uerpnemotrg4\nvUDLNBDLy7OQJcXF4TjUdtvBDTeELanbbgt9TYmIdJFOB5S7N5vZBcAsoAiY4u6vmtlVQK27zwDu\nBO4xs8XAh4QQAzgUuMrMmoAW4Bvu/mEmb6Q9xcVwwAHZf153WL++86GXavyjj2DZsq2nNzV1rsbW\nwMpsy68XlUdfT+Xqz1Ex5TdU1n2XyhuvpbiylI0boaUl7EZNHt/WvHTbZeM58r1d62P3rf9BqayE\n3r23ntbZeRUV+t9COq+lJXeHQsw78+9/TGpqary2tsv2Aua95ubNgZWtMGxrvLvr1St8Cbf+TB7v\n7LyubgfQ2Bh+B598svn3kTy0zvvkkxBuHZW4K7urwlCdNmfHxo3hn+D2hsbGrp3f2qa4OPzMhJnN\nd/ea9trpT6gbKS4Oh4L69u3a10ncKmwrxNb9/kk+ve0+mir7UdS7nKLSInqVFFFUunnoVVpMUVlx\nGC8rCeNlxWG8vISi8mKKykroVV4aHleUhvGK0i3HK8soqiyjV0VZeFxWTK8ia/NL3qywDpE1NW0d\nXNsKtW1N/+QTqK/fevqGDR2vq6Ska7YAE+eVlGT/d93S0rkv7q6a39ycvfdWVhaG8vLN48lDnz5t\nz2tdNlcUULIVs/BHWF4OAwa00ei4o2D8Bnj4YVi/KnzCkoeGRqhPmrZuXfgGyESvXpsLzNXQ+sks\nL++ab8UMlJRAv35h6CqJW+/pht225rUez02c3pn/youK2g+25MBpLxy6IhBSfcm3jvfvv+357S2f\n7vw8+7NNiwJKOu/YY8PQUc3NqQMtcWj95ujs8PHHbc/L9BsoMcG7IgiTh9LS1NNzeE1ELrbeW1rS\nC8F0A/KDD8LjXr22XG3bb991IZD8a+tugZBvFFCSe8XFYT9Cnz7xvH5zc+YB2F6QrlzZ9jKdPdsl\nWXFx2+G1rWDrzDLpPl9xcae/lXv1Cls+vXtnZ/VI96eAksJTXByGuL4JW496byv0kocNG9reR9XW\nvNbpa9due5lsBSaEcMpm4GVjXkmJrsDvphRQIrmWeOAkH7S0bBlY7QVeNuatWbPtZdavz+57LCoK\nQVVaGn52p/ECDlgFlEihSzzpJF+4b94V29EtxuShqSnMa2pKf3zdunAcM932XS2fArasDMaM6fr3\njAJKRPKR2eYvxLiOVaarNUw7EoBdPd6VAVtamv0t3DYooEREMpEYpt1RRwO2M1eGd5ICSkSkkOVx\nwBbmkTcREcl7CigREclL3epmsWZWD7yb4dMMAlZmoZyupjqzS3VmX3epVXVmVzbq3MXdB7fXqFsF\nVDaYWW06d9GNm+rMLtWZfd2lVtWZXbmsU7v4REQkLymgREQkLxViQN0WdwFpUp3ZpTqzr7vUqjqz\nK2d1FtwxKBER6R4KcQtKRES6AQWUiIjkpR4ZUGY21szeNLPFZjY5xfwyM3swmv+CmVXnvspNtbRX\n69lmVm9mC6Lh32KocYqZrTCzhW3MNzO7MXoPL5vZ/rmuMaqjvTrHmNmahHV5Ra5rjOoYZmazzew1\nM3vVzC5K0Sb2dZpmnfmyTsvN7K9m9lJU649TtIn9c59mnbF/5hNqKTKzv5nZ4ynmdf36dPceNQBF\nwBJgV6AUeAkYkdTmW8At0fipwIN5XOvZwH/HvE4PBfYHFrYx/xjgCcCA0cALeVrnGODxONdlVMdO\nwP7ReF/grRS/99jXaZp15ss6NaBPNF4CvACMTmoT++c+zTpj/8wn1PJt4P5Uv+NcrM+euAU1Cljs\n7m+7+wbgAWBCUpsJwF3R+HTgCLNO9lOdmXRqjZ27Pwd8uI0mE4C7PXge2N7MdspNdZulUWdecPf3\n3P3FaLwBeB0YmtQs9nWaZp15IVpPa6OHJdGQfAZY7J/7NOvMC2ZWBRwL3NFGky5fnz0xoIYCSxMe\n17H1h2pTG3dvBtYAA3NSXRt1RFLVCjAx2s0z3cyG5aa0Dkn3feSDL0S7V54ws73jLibaLbIf4T/p\nRHm1TrdRJ+TJOo12Ry0AVgBPunub6zTOz30adUJ+fOZvAC4DWtqY3+XrsycGVE/ze6Da3T8HPMnm\n/1ik414k3APsn4FfA4/GWYyZ9QEeAi5294/jrGVb2qkzb9apu29095FAFTDKzPaJq5ZtSaPO2D/z\nZnYcsMLd5+f6tRP1xIBaBiT+x1EVTUvZxsyKgX7AqpxU10Ydka1qdfdV7t7afeUdwAE5qq0j0lnn\nsXP3j1t3r7j7TKDEzAbFUYuZlRC+9O9z94dTNMmLddpenfm0ThNqWg3MBsYmzcqXzz3Qdp158pk/\nCBhvZu8QDj0cbmb3JrXp8vXZEwNqHrC7mQ03s1LCwbsZSW1mAGdF4ycBz3h0pC/H2q016bjDeMJx\ngHwzA/hqdObZaGCNu78Xd1HJzGxI6z5yMxtF+PvP+RdUVMOdwOvufl0bzWJfp+nUmUfrdLCZbR+N\nVwBHAW8kNYv9c59OnfnwmXf3y929yt2rCd9Lz7j7GUnNunx99rgedd292cwuAGYRzpKb4u6vmtlV\nQK27zyB86O4xs8WEg+qn5nGtF5rZeKA5qvXsXNdpZlMJZ2sNMrM64ErCwV3c/RZgJuGss8XAp8A5\nua4xzTpPAr5pZs3AOuDUmP4xOQg4E3glOhYB8H1g54Ra82GdplNnvqzTnYC7zKyIEJLT3P3xPPzc\np1Nn7J/5tuR6fepWRyIikpd64i4+ERHpARRQIiKSlxRQIiKSlxRQIiKSlxRQIiKSlxRQIiKSlxRQ\nIiKSl/4f1s3XHWp9jLEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fee1466bed0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(211)\n",
    "plt.title(\"accuracy\")\n",
    "plt.plot(history.history[\"acc\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_acc\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.subplot(212)\n",
    "plt.title(\"loss\")\n",
    "plt.plot(history.history[\"loss\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_loss\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy score: 0.506\n",
      "\n",
      "confusion matrix\n",
      "\n",
      "[[5403 5118]\n",
      " [5276 5259]]\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(linewidth=120)\n",
    "test_gen = datagen(Xtest, Ytest, docid2mat, batch_size=BATCH_SIZE)\n",
    "num_test_steps = len(Xtest) // BATCH_SIZE\n",
    "\n",
    "Ytest_ = model.predict_generator(test_gen, num_test_steps)\n",
    "\n",
    "ytest_ = np.argmax(Ytest_, axis=1)\n",
    "ytest = np.argmax(Ytest, axis=1)[0:len(ytest_)]\n",
    "\n",
    "print(\"accuracy score: {:.3f}\".format(accuracy_score(ytest, ytest_)))\n",
    "print(\"\\nconfusion matrix\\n\")\n",
    "print(confusion_matrix(ytest, ytest_))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}