src/04c-ng-clf-eeap.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embed, Encode, Attend and Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division, print_function\n",
    "from keras.layers import Input\n",
    "from keras.layers.core import Dense, Dropout\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.layers.pooling import GlobalAveragePooling1D, GlobalMaxPooling1D\n",
    "from keras.layers.wrappers import TimeDistributed, Bidirectional\n",
    "from keras.models import Model\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "import custom_attn\n",
    "import logging\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import os\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_DIR = \"../data\"\n",
    "\n",
    "VOCAB_FILE = os.path.join(DATA_DIR, \"ng-vocab.tsv\")\n",
    "MIN_OCCURS = 5\n",
    "\n",
    "GLOVE_FILE = os.path.join(DATA_DIR, \"glove.840B.300d.txt\")\n",
    "\n",
    "# covers about 95% of input data\n",
    "MAX_SENTS = 40 # maximum number of sentences per document\n",
    "MAX_WORDS = 60 # maximum number of words per sentence\n",
    "\n",
    "WORD_EMBED_SIZE = 300\n",
    "SENT_EMBED_SIZE = 100\n",
    "DOC_EMBED_SIZE = 50\n",
    "NUM_CLASSES = 20\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "NUM_EPOCHS = 10\n",
    "\n",
    "logging.basicConfig()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Load Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_size: 40730\n"
     ]
    }
   ],
   "source": [
    "word2id = {\"PAD\": 0, \"UNK\": 1}\n",
    "fvocab = open(VOCAB_FILE, \"rb\")\n",
    "for i, line in enumerate(fvocab):\n",
    "    word, count = line.strip().split(\"\\t\")\n",
    "    if int(count) <= MIN_OCCURS:\n",
    "        break\n",
    "    word2id[word] = i\n",
    "fvocab.close()\n",
    "id2word = {v:k for k, v in word2id.items()}\n",
    "vocab_size = len(word2id)\n",
    "print(\"vocab_size: {:d}\".format(vocab_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load GloVe Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40730, 300)\n"
     ]
    }
   ],
   "source": [
    "E = np.zeros((vocab_size, WORD_EMBED_SIZE))\n",
    "E[1] = np.random.random(WORD_EMBED_SIZE)\n",
    "fglove = open(GLOVE_FILE, \"rb\")\n",
    "for line in fglove:\n",
    "    cols = line.strip().split(\" \")\n",
    "    word = cols[0]\n",
    "    if not word2id.has_key(word):\n",
    "        continue\n",
    "    vec = np.array([float(x) for x in cols[1:]])\n",
    "    idx = word2id[word]\n",
    "    E[idx] = vec\n",
    "fglove.close()\n",
    "print(E.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute Document Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18846\n"
     ]
    }
   ],
   "source": [
    "ng_data = fetch_20newsgroups(subset='all',\n",
    "                             data_home=DATA_DIR,\n",
    "                             shuffle=True, \n",
    "                             random_state=42)\n",
    "num_docs = len(ng_data.data)\n",
    "print(num_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['like', 'a', 'mouse']\n",
      "['PAD', 'The', 'cat', 'fought', 'like', 'a', 'mouse']\n"
     ]
    }
   ],
   "source": [
    "def pad_or_truncate(xs, maxlen):\n",
    "    if len(xs) > maxlen:\n",
    "        xs = xs[len(xs) - maxlen:]\n",
    "    elif len(xs) < maxlen:\n",
    "        xs = [\"PAD\"] * (maxlen - len(xs)) + xs\n",
    "    return xs\n",
    "\n",
    "xs = [\"The\", \"cat\", \"fought\", \"like\", \"a\", \"mouse\"]\n",
    "print(pad_or_truncate(xs, 3))\n",
    "print(pad_or_truncate(xs, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18846, 40, 60)\n"
     ]
    }
   ],
   "source": [
    "X = np.zeros((num_docs, MAX_SENTS, MAX_WORDS))\n",
    "for docid in range(num_docs):\n",
    "    text = ng_data.data[docid]\n",
    "    sents = pad_or_truncate(nltk.sent_tokenize(text), MAX_SENTS)\n",
    "    for sid, sent in enumerate(sents):\n",
    "        words = nltk.word_tokenize(sent)\n",
    "        words = pad_or_truncate(words, MAX_WORDS)\n",
    "        for wid, word in enumerate(words):\n",
    "            try:\n",
    "                word_id = word2id[word]\n",
    "            except KeyError:\n",
    "                word_id = word2id[\"UNK\"]\n",
    "            X[docid, sid, wid] = word_id\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18846, 20)\n"
     ]
    }
   ],
   "source": [
    "y = ng_data.target\n",
    "Y = to_categorical(y, num_classes=NUM_CLASSES)\n",
    "print(Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13192, 40, 60) (13192, 20) (5654, 40, 60) (5654, 20)\n"
     ]
    }
   ],
   "source": [
    "Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, train_size=0.7)\n",
    "print(Xtrain.shape, Ytrain.shape, Xtest.shape, Ytest.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Network\n",
    "\n",
    "### Sentence Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 60)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)          (None, 60, 300)       12219000    input_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional)  (None, 60, 200)       240600      embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalMa (None, 200)           0           bidirectional_1[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "attention_mv_1 (AttentionMV)     (None, 200)           12260       bidirectional_1[0][0]            \n",
      "                                                                   global_max_pooling1d_1[0][0]     \n",
      "====================================================================================================\n",
      "Total params: 12,471,860\n",
      "Trainable params: 12,471,860\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "sent_inputs = Input(shape=(MAX_WORDS,), dtype=\"int32\")\n",
    "\n",
    "sent_emb = Embedding(input_dim=vocab_size,\n",
    "                     output_dim=WORD_EMBED_SIZE,\n",
    "                     weights=[E])(sent_inputs)\n",
    "sent_enc = Bidirectional(GRU(SENT_EMBED_SIZE,\n",
    "                             return_sequences=True))(sent_emb)\n",
    "\n",
    "sent_vec = GlobalMaxPooling1D()(sent_enc)\n",
    "sent_att = custom_attn.AttentionMV()([sent_enc, sent_vec])\n",
    "\n",
    "sent_model = Model(inputs=sent_inputs, outputs=sent_att)\n",
    "\n",
    "# sent_enc = Bidirectional(GRU(SENT_EMBED_SIZE,\n",
    "#                              return_sequences=False))(sent_emb)\n",
    "\n",
    "# sent_model = Model(inputs=sent_inputs, outputs=sent_enc)\n",
    "\n",
    "sent_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_2 (InputLayer)             (None, 40, 60)        0                                            \n",
      "____________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistribu (None, 40, 200)       12471860    input_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional)  (None, 40, 100)       75300       time_distributed_1[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalMa (None, 100)           0           bidirectional_2[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "attention_mv_2 (AttentionMV)     (None, 100)           4140        bidirectional_2[0][0]            \n",
      "                                                                   global_max_pooling1d_2[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 100)           0           attention_mv_2[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 50)            5050        dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 50)            0           dense_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 20)            1020        dropout_2[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 12,557,370\n",
      "Trainable params: 12,557,370\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "doc_inputs = Input(shape=(MAX_SENTS, MAX_WORDS), dtype=\"int32\")\n",
    "\n",
    "doc_emb = TimeDistributed(sent_model)(doc_inputs)\n",
    "\n",
    "doc_enc = Bidirectional(GRU(DOC_EMBED_SIZE,\n",
    "                            return_sequences=True))(doc_emb)\n",
    "\n",
    "doc_vec = GlobalMaxPooling1D()(doc_enc)\n",
    "doc_att = custom_attn.AttentionMV()([doc_enc, doc_vec])\n",
    "\n",
    "fc1_dropout = Dropout(0.2)(doc_att)\n",
    "fc1 = Dense(50, activation=\"relu\")(fc1_dropout)\n",
    "fc2_dropout = Dropout(0.2)(fc1)\n",
    "doc_pred = Dense(NUM_CLASSES, activation=\"softmax\")(fc2_dropout)\n",
    "\n",
    "model = Model(inputs=doc_inputs, outputs=doc_pred)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11872 samples, validate on 1320 samples\n",
      "Epoch 1/10\n",
      "11872/11872 [==============================] - 143s - loss: 2.6603 - acc: 0.1515 - val_loss: 1.7490 - val_acc: 0.4455\n",
      "Epoch 2/10\n",
      "11872/11872 [==============================] - 142s - loss: 1.2612 - acc: 0.5948 - val_loss: 0.9111 - val_acc: 0.7242\n",
      "Epoch 3/10\n",
      "11872/11872 [==============================] - 141s - loss: 0.6531 - acc: 0.7953 - val_loss: 0.7710 - val_acc: 0.7682\n",
      "Epoch 4/10\n",
      "11872/11872 [==============================] - 141s - loss: 0.3494 - acc: 0.8985 - val_loss: 0.7463 - val_acc: 0.7864\n",
      "Epoch 5/10\n",
      "11872/11872 [==============================] - 141s - loss: 0.1852 - acc: 0.9506 - val_loss: 0.7559 - val_acc: 0.8038\n",
      "Epoch 6/10\n",
      "11872/11872 [==============================] - 141s - loss: 0.0943 - acc: 0.9752 - val_loss: 0.8884 - val_acc: 0.8061\n",
      "Epoch 7/10\n",
      "11872/11872 [==============================] - 141s - loss: 0.0494 - acc: 0.9902 - val_loss: 0.9145 - val_acc: 0.7917\n",
      "Epoch 8/10\n",
      "11872/11872 [==============================] - 141s - loss: 0.0316 - acc: 0.9935 - val_loss: 0.9929 - val_acc: 0.8053\n",
      "Epoch 9/10\n",
      "11872/11872 [==============================] - 140s - loss: 0.0251 - acc: 0.9945 - val_loss: 1.0296 - val_acc: 0.8030\n",
      "Epoch 10/10\n",
      "11872/11872 [==============================] - 140s - loss: 0.0194 - acc: 0.9965 - val_loss: 1.0857 - val_acc: 0.7886\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE,\n",
    "                   epochs=NUM_EPOCHS, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VOX5///XRQgkLAoEUCSEIFrFFTAqilqrtgWrSOuC\nVq21rXzaj1Zo7YJalbrV7dPW/qptXbBaqNSiVbR+S7WCWsWWIKgoWpayhEVCWGRJJIHr98c9IZOQ\nZZJMciaT9/PxOI+ZOefMnCuj5J1zn/vct7k7IiIiqaZD1AWIiIjURgElIiIpSQElIiIpSQElIiIp\nSQElIiIpSQElIiIpSQElIiIpSQElIiIpSQElkoIs0L9Padf0D0CkHmY2ycyWmdk2M/vAzL4ct+0q\nM1sct214bP0AM3vGzIrNrMTMfh1bP9nMpsa9P9/M3Mw6xl7PMbM7zOwNYCdwsJldGXeM5Wb2PzXq\nO8/MFprZJ7E6R5nZhWY2v8Z+3zez51rumxJJvo5RFyCS4pYBpwLrgQuBqWZ2CHAKMBkYCxQCg4Fy\nM8sAXgBeAS4HdgMFjTje5cBo4CPAgMOAc4DlwGnA/zOzee7+tpmdADwBXAD8A+gHdAf+C/zOzIa4\n++K4z729KV+ASFR0BiVSD3f/s7uvdfc97v4nYAlwAvAt4B53n+fBUndfGdt2EPBDd9/h7mXu/s9G\nHPL37v6+u1e4e7m7/9Xdl8WO8Srwd0JgAnwTmOLuL8XqW+PuH7r7p8CfgMsAzOxIIJ8QnCJthgJK\npB5m9rVYE9oWM9sCHAX0BgYQzq5qGgCsdPeKJh5ydY3jjzazt8xsU+z4Z8eOX3ms2moAeBz4qpkZ\n4ezpqVhwibQZCiiROpjZQOBh4Bogx917AIsITW+rCc16Na0G8iqvK9WwA+gS9/rAWvbZO72AmXUG\nngbuAw6IHf/F2PErj1VbDbj7W8AuwtnWV4E/1P5TiqQuBZRI3boSAqMYwMyuJJxBATwC/MDMjov1\nuDskFmj/BtYBd5lZVzPLMrORsfcsBE4zszwz2x+4voHjdwI6x45fYWajgS/EbX8UuNLMzjSzDmbW\n38wOj9v+BPBroLyRzYwiKUEBJVIHd/8A+D9gLvAxcDTwRmzbn4E7gD8C24BngV7uvhs4FzgEWAUU\nAeNi73mJcG3oXWA+DVwTcvdtwLXAU8BmwpnQzLjt/wauBH4BbAVeBQbGfcQfCIE6FZE2yDRhoUh6\nMrNsYAMw3N2XRF2PSGPpDEokfX0HmKdwkrZK90GJpCEzW0HoTDE24lJEmkxNfCIikpLUxCciIikp\n5Zr4evfu7fn5+VGXISIiLWT+/Pkb3b1PQ/s1GFBmNoUwFtgGdz+qlu0G3E+4w30n8HV3fzu27Qrg\nJ7Fdb3f3xxs6Xn5+PoWFhQ3tJiIibZSZrUxkv0Sa+H4PjKpn+2jg0NgyHvhNrIBewC3AiYTxyW4x\ns56JFCUiItJgQLn7a8CmenY5D3giNpjlW0APM+sHfBF4yd03uftm4CXqDzoREYmaO+zZAxUVUF4O\nu3ZBWRmUlsLOneF5K0nGNaj+VB/gsii2rq71+zCz8YSzL/Ly8pJQkoi0it27wy+uHTvCL6+dO6s/\nr3xdWhr23bMnLPHP61rX0Otk7eNefYHan9e3LdH9GrMtfomvM5HnTX1PIr26x46Fv/yl6f/PNEJK\ndJJw94eAhwAKCgrU710kGfbs2Tcomvq6rm2fttAA6R06hCUjo+p5Xesael3fPmbhEcLzyiX+daLb\nkvEZtb2urDPR5y39nkMPbZn/5rVIRkCtIQz7Xyk3tm4NcHqN9XOScDyR9qG0FDZvrr5s2bLvuvj1\nW7dWhUdTmmIyM6FLF+jaNTxWLl27Qk5O3dtqex7/OjsbOnZMLEgqf0lLu5eMgJoJXGNm0wkdIra6\n+zozmwXcGdcx4gs0PHqzSPpwh+3bEwuW2tY3dHbSvTv07BmWHj3gkENg//2hW7fGhUf8kpnZOt+N\nSAIS6Wb+JOFMqLeZFRF65mUCuPtvCfPTnA0sJXQzvzK2bZOZ3QbMi33Ure5eX2cLkdRWXg4rV8Ky\nZbB6dcNhs2VLuNBcF7MQLD16VAVN//7VQ6fyec11PXqEMxKRNJZyQx0VFBS47oOSyJSVwfLlsHRp\nCKKlS6uer1gRLrDHy8jYN0TqCpea6/fbr+r6h0g7Ymbz3b2gof30J5i0P9u27Rs+lc/XrKnek6my\n6ez44+Hii8PzQw6BvDzo1Ss0k+maiUiLUEBJ+nGHTZvqDqENG6rv37dvCJ0zzoDBg6tC6JBDQgiJ\nSCQUUNI2ucP69bU3xS1dGq7/xMvNDYEzZkx4rAyiwYNDZwMRSTkKKElde/aEzgi1nQUtWxa6UlfK\nyID8/BA4X/1q9RAaNCh0cxaRNkUBJallzRr4+9/D8tJLUFJSta1TpxA6gwfDmWdWD6GBA9VFWiTN\nKKAkWqWl8PrrMGtWCKVFi8L6Aw6As8+GU04Jd64PHhy6YGdkRFuviLQaBZS0Lnf44IMQSLNmwWuv\nha7dnTrBqafC5ZfDF78Ixxyj3nEi7ZwCSlpeSQm8/HLVWdKaNWH94YfD//xPCKTTTgtdtkVEYhRQ\nknzl5fDWWyGMZs2CwsJw5tSjB5x1VgikL3wh3EskIlIHBZQkx/LlVWdIr7wCn3wSRkkYMQImTw6B\ndPzxuoYkIglTQEnTbNsGs2dXXUtatiysHzgwjLjwxS+GG1979Ii2ThFpsxRQkpg9e+Dtt6ua7d58\nMwyE2rUrfO5zMGFCCKVDD1XnBhFJCgWU1G3t2ur3JG3cGNYPGwY/+EFotjv5ZOjcOdo6RSQtKaCk\nSllZ9XuS3nsvrD/gABg9OpwhnXVWeJ3G3MNUTNu377vs2BEeS0urZsuOf6xtXX3bGrt/Q9ugamqn\n7Ox9p3tqaFtWVtsbYL3yv1dpaRhcpLQ08eelpeEz4ieOre2xvm0ttW/nzvX/92sP96UroNq78nJ4\n/HGYMQNefbX6PUl3353y9yTt3l0VGpWPiS717V9zVo2WVHNm7YZ+cdW3zb36L+GmyMpqfLA1tK1z\n5xAijQ2QREOmqbMGZWWF766+wE9VHTs2/r9DU7ZHOe2YAqq92rMnhNKNN4ax7Q47rMXvSdqzJ4TC\ntm0hBGo+1rWuvqUxv4QzMsJkszWXAw+sfX3XrrWv79at6kyjuaFSubSEPXvC3xuVv9BrWxq7bePG\nfbc1NQhrk5FR/ZdkdnbV8+7dw8DztW1ryvPKcKqPe2JntI05+030zDo+0Bv736+4eN9tO3aEz22s\nzMzq4TV6NPz6103779tYCqj26OWXYdIkmD8fjj4aXnghDCsU96+1stmkvvBIJGDiH+PHdm1IVlb4\nhdS1a3isL0zqC5L4pVOnlD0RbBEdOlT9YsnJabnjVAZhfb80y8rCf9OGwiPVmq3iz27bOvfQYNKU\nwItfP2hQ69WcUECZ2SjgfiADeMTd76qx/RfA52IvuwB93b1HbNtuIHYxg1XuPiYZhUvj7Xrrbdb8\n4BesemMVq3JOZdUFD7Oq57Gs+nUHSn66b9jUN1t5vA4dQohUBknlY15eVTjU3FbfY9eums28LYkP\nQkldZuGPtE6dYP/9o64mMQ3+GjCzDOAB4PNAETDPzGa6+weV+7j79+L2/y4wLO4jSt19aPJKltq4\nw+bNsGrVvsvKj8pY9dFO1pUOxflDeEMJMAP69IEBA0LTycCBDQdIbesSaSoREWmsRP5OPQFY6u7L\nAcxsOnAe8EEd+18C3JKc8qTSrl1QVFR7AFUuO3ZUf0/nzk5e1gbyPlnEFzusIe+0AeRdNIK8w7LJ\nywvBpGmSRCRVJRJQ/YHVca+LgBNr29HMBgKDgFfiVmeZWSFQAdzl7s/W8r7xwHiAvHY4PlvlDOX1\nhc+6dfv2KurbNzSjDRkS+jbk5YWzoLxe28l77v+jz+9ux3bsgu+Mh5tuChdwRETaiGS39F8MzHD3\n+E66A919jZkdDLxiZu+5+7L4N7n7Q8BDAAUFBSneubPp3n0XFi6MNbutrB5ANTsQZGWFwMnLg1Gj\nqp5XLrm5tZz9fPopPPggjL8jjCB+8cVw221hQj8RkTYmkYBaAwyIe50bW1ebi4Gr41e4+5rY43Iz\nm0O4PrVs37emr4oKuPlm+NnPqtYdcEAImiOPDN029579xAKod+9GXNfZvRumTg0HWbUKPv/5cLDj\njmuRn0dEpDUkElDzgEPNbBAhmC4GvlpzJzM7HOgJzI1b1xPY6e6fmllvYCRwTzIKbyuKiuCSS+Cf\n/4RvfQt+9KNw7ScrKwkf7g5//Stcf32YibagAKZMCdOhi4i0cQ0GlLtXmNk1wCxCN/Mp7v6+md0K\nFLr7zNiuFwPT3atdKRkC/M7M9gAdCNeg6upckXb++le44orQ8jZtGnx1n1hvhjffhB//OCTfIYfA\nU0/BBReoO52IpA3zFBvPo6CgwAsLC6Muo1nKy+GGG+C+++DYY0N2fOYzSfrw998PHz5zZuj0cMst\n8M1vpt4djiIidTCz+e5e0NB+uh0yyVauDH0T3noLvvMd+PnPk9Sct3p1CKPHHw83H91xR5jiQtOk\ni0iaUkAl0XPPwde/HoZ++dOf4KKLkvChJSWhw8Ovfx2uOU2cGM6gWnLsGhGRFJAGI0xFb9eukBtj\nx8LgwWFev2aH086dIZgGD4Zf/CL0tFiyBP7v/xROItIu6AyqmZYvh3HjoLAQvvtduPfeZs7fV14e\neuL99Kfh7twxY+DOO0N/dBGRdkQB1QxPPw3f+EYYLPOZZ+DLX27Gh7lXTX+xZAmMHBl6V5xyStLq\nFRFpS9TE1wRlZXDNNaFX9+GHw4IFzQynV16BE08M7YKdOoUeeq+/rnASkXZNAdVIS5bAySfDAw/A\nddeFHMnPb+KHLVgQBtE780xYvx4eewzeeQfOPVf3M4lIu6cmvkaYPh3Gjw+3HM2cGXKkSZYtg5/8\nJHxgr16h48P//m+S+qOLiKQHnUEloLQ0zIZ+ySVhAtoFC5oYTh9/HNoGDz889Em/4YbQy+L731c4\niYjUoDOoBnz0Ubg09O67YWSh225r4qAN//lPaBvcsgWuuioM7NqvX9LrFRFJFwqoekydCt/+dpjW\n4sUXw6jjTbJpE5xzTriu9M476jIuIpIANfHVYufO0H388sth+PAwh1OTw2nXrtDdb+VKePZZhZOI\nSIIUUDW8/z4cfzz8/vehH8Mrr0D//k38MPfQ+WH2bHj00XBvk4iIJERNfDHuIZSuvhq6d4dZs8K8\nf83y85+HYPrJT+Cyy5JRpohIu6EzKGD79jBv0ze+ASNGhCa9ZofTzJnwwx/ChReGYYtERKRR2n1A\nvftuaNKbNi3kyEsvJaFz3cKFYXbCgoJwWtah3X/NIiKNltBvTjMbZWYfmdlSM5tUy/avm1mxmS2M\nLd+K23aFmS2JLVcks/jmcIeHHgojDG3ZAv/4R+j5nZHRzA9ety7cJNWzZ7jXqUuXpNQrItLeNHgN\nyswygAeAzwNFwDwzm1nL1O1/cvdrary3F3ALUAA4MD/23s1Jqb6JPvkk3Hg7fXpoyps6Ffr2TcIH\n79wJ550HmzeHqdh1n5OISJMlcgZ1ArDU3Ze7+y5gOnBegp//ReAld98UC6WXgFFNKzU5FiyA444L\nA4XfcQf87W9JCqc9e8KFrMJC+OMfYejQJHyoiEj7lUhA9QdWx70uiq2r6Xwze9fMZpjZgMa818zG\nm1mhmRUWFxcnWHrjuMODD4ZOEKWlMGdOGGkoaZeHbrklTJdx771hDicREWmWZP16fh7Id/djCGdJ\njzfmze7+kLsXuHtBnz59klRSla1bw3BFV18dBg5fuBBOPTWJB5g6FW6/Hb71rTCunoiINFsi90Gt\nAQbEvc6NrdvL3UviXj4C3BP33tNrvHdOY4tsjsLCEE6rVsE994QpMpLaqe6NN+Cb34TTTw9zcGia\nDBFpQHl5OUVFRZSVlUVdSovKysoiNzeXzCYNYJpYQM0DDjWzQYTAuRj4avwOZtbP3dfFXo4BFsee\nzwLuNLOesddfAK5vUqWN5A6/+lW4FalfvzBv00knJfkgy5fD2LEwcGCYXrdTpyQfQETSUVFREd27\ndyc/Px9L0z9q3Z2SkhKKiooYNGhQkz6jwYBy9wozu4YQNhnAFHd/38xuBQrdfSZwrZmNASqATcDX\nY+/dZGa3EUIO4FZ339SkShth8+Zw0+2zz4bLQY89FqZdSqqtW0N38t274YUXWuAAIpKuysrK0jqc\nAMyMnJwcmtOvIKGhjtz9ReDFGutujnt+PXWcGbn7FGBKkytspA8/hFGjYO3aMNLQxIkt0OpWUQHj\nxoUpNGbNgs98JskHEJF0l87hVKm5P2PajcWXmwtHHBG6kZ9wQgsd5HvfC8H08MNwxhktdBARkfYt\n7cbg6dYtzN3UYuH061+H5brrQq89EZE2ZsuWLTz44IONft/ZZ5/Nli1bWqCi2qVdQLWov/0NJkwI\n157uvjvqakREmqSugKqoqKj3fS+++CI9evRoqbL2kXZNfC3m/ffDdaejjw4jRTR70D4RkWhMmjSJ\nZcuWMXToUDIzM8nKyqJnz558+OGH/Oc//2Hs2LGsXr2asrIyJkyYwPjx4wHIz8+nsLCQ7du3M3r0\naE455RTefPNN+vfvz3PPPUd2dnZS61RAJaK4OEzZ3qULPP98aEcUEUmGiRPD6AHJNHQo/PKXdW6+\n6667WLRoEQsXLmTOnDl86UtfYtGiRXu7g0+ZMoVevXpRWlrK8ccfz/nnn09OTk61z1iyZAlPPvkk\nDz/8MBdddBFPP/00lyV53jsFVEM+/RS+/GVYvx5efRUGDGj4PSIibcgJJ5xQ7V6lX/3qV/zlL38B\nYPXq1SxZsmSfgBo0aBBDY2OOHnfccaxYsSLpdSmg6uMOV10VRov4059asOeFiLRb9ZzptJauXbvu\nfT5nzhxefvll5s6dS5cuXTj99NNrHfGic+fOe59nZGRQWlqa9LrUSaI+P/sZ/OEPcNttYbwkEZE0\n0L17d7Zt21brtq1bt9KzZ0+6dOnChx9+yFtvvdXK1VXRGVRdZsyAG2+ESy8NjyIiaSInJ4eRI0dy\n1FFHkZ2dzQEHHLB326hRo/jtb3/LkCFDOOywwxgxYkRkdZq7R3bw2hQUFHhhYWG0RcybB5/9LAwb\nFqbazcqKth4RSSuLFy9myJAhUZfRKmr7Wc1svrsXNPReNfHVtHp1GMDvgAPgL39ROImIRERNfPG2\nbw/htGMHvPxykqbaFRGRplBAVdq9O1xvevdd+Otf4cgjo65IRKRdU0BVuv56mDkzTCI1alTU1YiI\ntHu6BgXw6KNw773wv/8L11wTdTUiIoICCubMgW9/G77wBbj/fk3ZLiKSItp3QC1ZAl/5Chx6aBgp\noqNaPEVEauoW0fijCQWUmY0ys4/MbKmZTapl+/fN7AMze9fM/mFmA+O27TazhbFlZjKLb5ZNm8IA\nsB06hCnbW3EIeRERaViDpwxmlgE8AHweKALmmdlMd/8gbrcFQIG77zSz7wD3AONi20rdfWiS626e\n8nK44AJYsSLciHvwwVFXJCLSaiZNmsSAAQO4+uqrAZg8eTIdO3Zk9uzZbN68mfLycm6//XbOO++8\nSOtMpE3rBGCpuy8HMLPpwHnA3oBy99lx+78FJHfM9WRyD50hZs+Gxx+HU06JuiIRaccimG2DcePG\nMXHixL0B9dRTTzFr1iyuvfZa9ttvPzZu3MiIESMYM2YMFuF1+UQCqj+wOu51EXBiPft/E/h/ca+z\nzKwQqADucvdna77BzMYD4wHy8vISKKkZfvELeOQRuOEG+NrXWvZYIiIpaNiwYWzYsIG1a9dSXFxM\nz549OfDAA/ne977Ha6+9RocOHVizZg0ff/wxBx54YGR1JrVXgJldBhQAn41bPdDd15jZwcArZvae\nuy+Lf5+7PwQ8BGEsvmTWVM3zz8MPfgDnnx9GKBcRiVhUs21ceOGFzJgxg/Xr1zNu3DimTZtGcXEx\n8+fPJzMzk/z8/Fqn2WhNiXSSWAPEz9KXG1tXjZmdBdwIjHH3TyvXu/ua2ONyYA4wrBn1Nt0778Al\nl8Dw4fDEE6FzhIhIOzVu3DimT5/OjBkzuPDCC9m6dSt9+/YlMzOT2bNns3LlyqhLTCig5gGHmtkg\nM+sEXAxU641nZsOA3xHCaUPc+p5m1jn2vDcwkrhrV61m3To499zQU2/mzDB1u4hIO3bkkUeybds2\n+vfvT79+/bj00kspLCzk6KOP5oknnuDwww+PusSGm/jcvcLMrgFmARnAFHd/38xuBQrdfSZwL9AN\n+HPsgtoqdx8DDAF+Z2Z7CGF4V43efy2vtBTOOw9KSuCf/4SDDmrVw4uIpKr33ntv7/PevXszd+7c\nWvfbvn17a5VUTULXoNz9ReDFGutujnt+Vh3vexM4ujkFNsuePXDFFVBYCM88E+Z3EhGRNiG9h06Y\nPBn+/Ge45x4YOzbqakREpBHSt6fAtGmhp943vhF67omIpJBUm828JTT3Z0zPgHrjjRBMn/0s/OY3\nGgBWRFJKVlYWJSUlaR1S7k5JSQlZzZiVPP2a+FasgC9/GfLy4OmnoVOnqCsSEakmNzeXoqIiiouL\noy6lRWVlZZGbm9vk96dfQHXvDiefDHffDTk5UVcjIrKPzMxMBg0aFHUZKS/9AionB57dZzQlERFp\nY9LzGpSIiLR5CigREUlJlmq9SMysGEjGIFC9gY1J+Jz2Qt9X4+j7ahx9X42Xzt/ZQHfv09BOKRdQ\nyWJmhe5eEHUdbYW+r8bR99U4+r4aT9+ZmvhERCRFKaBERCQlpXNAPRR1AW2Mvq/G0ffVOPq+Gq/d\nf2dpew1KRETatnQ+gxIRkTZMASUiIikp7QLKzEaZ2UdmttTMJkVdTyozswFmNtvMPjCz981sQtQ1\ntRVmlmFmC8zshahrSXVm1sPMZpjZh2a22MxOirqmVGZm34v9e1xkZk+aWdOHA2/j0iqgzCwDeAAY\nDRwBXGJmR0RbVUqrAK5z9yOAEcDV+r4SNgFYHHURbcT9wN/c/XDgWPS91cnM+gPXAgXufhSQAVwc\nbVXRSauAAk4Alrr7cnffBUwHzou4ppTl7uvc/e3Y822EXxz9o60q9ZlZLvAl4JGoa0l1ZrY/cBrw\nKIC773L3LdFWlfI6Atlm1hHoAqyNuJ7IpFtA9QdWx70uQr9wE2Jm+cAw4F/RVtIm/BL4EbAn6kLa\ngEFAMfBYrEn0ETPrGnVRqcrd1wD3AauAdcBWd/97tFVFJ90CSprAzLoBTwMT3f2TqOtJZWZ2DrDB\n3edHXUsb0REYDvzG3YcBOwBdG66DmfUktPoMAg4CuprZZdFWFZ10C6g1wIC417mxdVIHM8skhNM0\nd38m6nragJHAGDNbQWhCPsPMpkZbUkorAorcvfLMfAYhsKR2ZwH/dfdidy8HngFOjrimyKRbQM0D\nDjWzQWbWiXBxcWbENaUsMzPCtYHF7v7zqOtpC9z9enfPdfd8wv9fr7h7u/0LtyHuvh5YbWaHxVad\nCXwQYUmpbhUwwsy6xP59nkk77lSSVjPqunuFmV0DzCL0fpni7u9HXFYqGwlcDrxnZgtj625w9xcj\nrEnSz3eBabE/GpcDV0ZcT8py93+Z2QzgbUIv2wW04yGPNNSRiIikpHRr4hMRkTShgBIRkZSkgBIR\nkZSkgBIRkZSkgBIRkZSkgBIRkZSkgBIRkZSkgBIRkZSkgBIRkZSkgBIRkZSkgBIRkZSkgBIRkZSk\ngBJpIWa2wszOiroOkbZKASUiIilJASUiIilJASXSwsyss5n90szWxpZfmlnn2LbeZvaCmW0xs01m\n9rqZdYht+7GZrTGzbWb2kZmdGe1PItK60mpGXZEUdSMwAhgKOPAc8BPgJuA6oAjoE9t3BOCxKdKv\nAY5397Vmlk+YJVqk3dAZlEjLuxS41d03uHsx8FPg8ti2cqAfMNDdy939dQ/TXO8GOgNHmFmmu69w\n92WRVC8SEQWUSMs7CFgZ93plbB3AvcBS4O9mttzMJgG4+1JgIjAZ2GBm083sIETaEQWUSMtbCwyM\ne50XW4e7b3P369z9YGAM8P3Ka03u/kd3PyX2Xgfubt2yRaKlgBJpeU8CPzGzPmbWG7gZmApgZueY\n2SFmZsBWQtPeHjM7zMzOiHWmKANKgT0R1S8SCQWUSMu7HSgE3gXeA96OrQM4FHgZ2A7MBR5099mE\n6093ARuB9UBf4PrWLVskWhaux4qIiKQWnUGJiEhKUkCJiEhKUkCJiEhKUkCJiEhKSrmhjnr37u35\n+flRlyEiIi1k/vz5G929T0P7pVxA5efnU1hYGHUZIiLSQsxsZcN7qYlPRERSVPoFVEUF3HYbbNoU\ndSUiItIM6RdQixaFgLroIigvj7oaERFpopS7BtVsQ4fCQw/BlVfCtdfCgw+CWdRViYjsVV5eTlFR\nEWVlZVGX0qKysrLIzc0lMzOzSe9Pv4AC+PrXYfFiuOceOPJIuOaaqCsSEdmrqKiI7t27k5+fj6Xp\nH9DuTklJCUVFRQwaNKhJn5F+TXyV7rwTxoyBCRPg73+PuhoRkb3KysrIyclJ23ACMDNycnKadZaY\nvgGVkQFTp8JRR4XrUYsXR12RiMhe6RxOlZr7M6ZvQAF07w4zZ0LnznDuuVBSEnVFIiKSoPQOKICB\nA+HZZ2H1arjgAti1K+qKREQitWXLFh588MFGv+/ss89my5YtLVBR7dI/oABOOgkefRTmzAkdJjQH\nloi0Y3UFVEVFRb3ve/HFF+nRo0dLlbWP9OzFV5vLLgvXoe68M/TsmzAh6opERCIxadIkli1bxtCh\nQ8nMzCQrK4uePXvy4Ycf8p///IexY8eyevVqysrKmDBhAuPHjweqhqLbvn07o0eP5pRTTuHNN9+k\nf//+PPfcc2RnZye1zvYTUBBu4F28GL7/ffjMZ2D06KgrEpH2buJEWLgwuZ85dCj88pd1br7rrrtY\ntGgRCxcuZM6cOXzpS19i0aJFe7uDT5kyhV69elFaWsrxxx/P+eefT05OTrXPWLJkCU8++SQPP/ww\nF110EU+Wwi4yAAATiUlEQVQ//TSXXXZZUn+M9tHEV6lDB3jiCTjmGBg3Dt5/P+qKREQid8IJJ1S7\nV+lXv/oVxx57LCNGjGD16tUsWbJkn/cMGjSIoUOHAnDcccexYsWKpNfVvs6gALp1Cz37jj8+9Oz7\n97+hd++oqxKR9qqeM53W0rVr173P58yZw8svv8zcuXPp0qULp59+eq33MnXu3Hnv84yMDEpLS5Ne\nV/s6g6o0YAA89xysXQtf+Yp69olIu9K9e3e2bdtW67atW7fSs2dPunTpwocffshbb73VytVVaZ8B\nBXDiifDYY/D66/Dtb6tnn4i0Gzk5OYwcOZKjjjqKH/7wh9W2jRo1ioqKCoYMGcKkSZMYMWJERFWC\neYr9Yi4oKPBWnbDw5ptD54n77oPrrmu944pIu7V48WKGDBkSdRmtoraf1czmu3tBQ+9tf9egapo8\nOfTs++EP4bDD4Jxzoq5IRERoz018lTp0gMcfh+HD4ZJLwnxSIiISOQUUQJcuodNE9+6hZ9+GDVFX\nJCLS7imgKvXvH0Jq/frQs+/TT6OuSESkXWvxgDKzAWY228w+MLP3zSx1xxg6/vjQ3PfGGzB+vHr2\niYhEqDU6SVQA17n722bWHZhvZi+5+wetcOzGq5w7avLkMGbfj34UdUUiIu1Si59Bufs6d3879nwb\nsBjo39LHbZabbw5DIU2aFJr9RETasW7dukVy3Fa9BmVm+cAw4F811o83s0IzKywuLm7NkmpnFm7i\nLSiASy+Fd96JuiIRkXan1QLKzLoBTwMT3f2T+G3u/pC7F7h7QZ8+fVqrpPplZ4eJDnv0CD37Pv44\n6opERJJi0qRJPPDAA3tfT548mdtvv50zzzyT4cOHc/TRR/NcCrQetcqNumaWSQinae7+TGscMykO\nOigMLHvKKfDlL8Mrr0BWVtRViUgaiWC2DcaNG8fEiRO5+uqrAXjqqaeYNWsW1157Lfvttx8bN25k\nxIgRjBkzBjNLbnGN0Bq9+Ax4FFjs7j9v6eMl3fDh8Ic/wNy5cNVV6tknIm3esGHD2LBhA2vXruWd\nd96hZ8+eHHjggdxwww0cc8wxnHXWWaxZs4aPI245ao0zqJHA5cB7Zlb5d8IN7v5iKxw7Oc4/H26/\nHX7yEzjiCLj++qgrEpE0EdVsGxdeeCEzZsxg/fr1jBs3jmnTplFcXMz8+fPJzMwkPz+/1mk2WlOL\nB5S7/xOI7hwxWW64AT74IDwedli4mVdEpI0aN24cV111FRs3buTVV1/lqaeeom/fvmRmZjJ79mxW\nrlwZdYkaLDZhZvDII7BsGVx+OQwaBMOGRV2ViEiTHHnkkWzbto3+/fvTr18/Lr30Us4991yOPvpo\nCgoKOPzww6MuMf0C6pNPQqe7yZPhc59L8odX9uw74QQYMybMxtuvX5IPIiLSOt577729z3v37s3c\nuXNr3W/79u2tVVI1aTcW38cfh+H0zjwzXCoqL0/yAQ48MPTs27QJxo6FFpjmWERE0jCgDj0U3n4b\nvvUtuOsuGDkSli5N8kGGDoVp08IZ1De+oZ59IiItIO0CCqBrV3joIXj66RBOQ4fC73+f5BwZOxZ+\n9jOYPj308BMRaYRUm828JTT3Z0zLgKr0la/Au++GQcqvvBIuvhi2bEniAX7849Bh4uabYcaMJH6w\niKSzrKwsSkpK0jqk3J2SkhKymjG4gaXaF1RQUOCFhYVJ/czdu+Hee+Gmm8LgEFOnwqmnJunDP/0U\nzjgDFiyA11+H445L0geLSLoqLy+nqKgo8vuMWlpWVha5ublkZmZWW29m8929oKH3t4uAqjRvHnz1\nq7B8Odx4Yzjx6ZiMfowbNoTTtIqKcJCDDkrCh4qIpKdEAyqtm/hqOv740IHia1+D226D006D//43\nCR/cty88/zxs3QrnnQc7dybhQ0VE2rd2FVAA3buHmTSmTw8DQxx7bGjya7ZjjoE//hHmz4evfx32\n7EnCh4qItF/tLqAqjRsXpnk69tjQz+Gyy8IJULOMGQN33w1//jPcemtS6hQRaa/abUABDBwIs2eH\nLJk+PXRHr+NG6sT94AfhDOqnPw0fKiIiTdKuAwpCJ4mbbgod8MxC777bbgs9/5rEDH772zCH1JVX\nhpt5RUSk0dp9QFU66aQwadgll4TefaefDk0ezLdzZ3jmmTAs0nnnQVFRMksVEWkXFFBx9tsvzE04\ndWrV9ak//amJH9anT+jZt2NHCKkdO5Jaq4hIulNA1eLSS8PZ1JAhYfSJK6+Ebdua8EFHHRWuQy1c\nCFdcoZ59IiKNoICqw8EHh+tSN90ETzwRpn5q0uWks8+G++4LAwPeckvS6xQRSVcKqHp07Bh6+M2Z\nE6btGDkyjJDe6A4UEyfCN79ZNW38J5+0RLkiImlFAZWAU08N16S+8pUwx9RZZzWy34MZPPhgGGfp\njjvC6dm992rECRGReiigEtSjR7ic9NhjYbi9Y44JrXYJ69QpzCE1bx4UFMCPfgSHHAIPPAC7drVY\n3SIibZUCqhHMwj24CxbA4MFwwQVw1VWN7KBXUAB/+xu89loIqGuugcMOCxNWVVS0UOUiIm2PAqoJ\nDj0U3nwzNPc9+igMHx4GoW2UU0+FV18NYZWTE7oKHnUUPPWUevuJiKCAarLMTLjzTnjllXAGNWJE\n6KzXqGwxgy9+MTT7PfNM6JUxblxIvBde0FTyIpIy3MNl8ybdctNELT4flJlNAc4BNrj7UQ3t35Lz\nQbWUTZtCU98zz4QOFI8/3sQpoXbvDhe6br45TFp10kmhU8XnPpf0mkUkfbmHS9vbtsH27WGp63lj\ntu3ZEzqLNer6ey1SZsJCMzsN2A48ka4BBeF/iEcfhQkTIDsbpkwJg5s3SXl56I1x662wZg2ceWYI\nqhNPTGrNIhKdyhApK9t3KS0NLTPNCZdEL2mbQbduYSqibt2qlvjX8c+POALOPbd5P3vKBFSsmHzg\nhXQOqEoffRTG81uwAL7zndDs16VLEz+srCwMPHvnnVBcHP6vuP320IVQRJqtvDw0W9UWEq2xNEXX\nrg2HSH3baj7Pzg4h1ZraVECZ2XhgPEBeXt5xK5s8Smtq+PTTcD/uffeFvzaefLKZmbJ9O9x/f7h3\nauvWMP7ST38Kn/lM0moWaavKy2Hz5n2XTZsaXt/cWxEzMyErKyzZ2VXPm7t07lx72HTpAhkZyfne\notSmAipeWz+DivfSS2EIvpKSMI/htddCh+Z0S9m8OaTe/feHP7+uuCJcrxo4MGk1i0ShogK2bEk8\nWOKX7dvr/+yuXaFnz6qlV6+q5z16hO1NDZF0CIsoKKBSRHFxGOXo+edh1Kgw6tHAgZCX14ymv48/\nhp/9DH7zm/B6/Hi48cYwvYdIM+3ZE85KysvDNZJdu6o/r/k6kW1lZVUBVFvoNNQzLDu7erDUFTg1\n1/foEe6Rl9SigEoh7uFS0ve/X73duXfvEFaVgVXzeU5OA23Dq1eH2RWnTAn/Cr/73TBCRU5Oi/9M\nEi33cGa+enX15eOP6w6LRMOkpe4Xz8pqXLjEv+7cuWVqkmikTECZ2ZPA6UBv4GPgFnd/tK790zGg\nKm3cCB9+GCZCXLkSVq2qer5y5b7t4V261B5clc/79w+3TrF0KUyeDH/8Y2isvu66cKq2335R/JiS\nBFu37hs+8UtRUejpFS8zEw44IARBZmb4m6VyiX8dxbbOnRUyUiVlAqqx0jmg6uMemjtqBlf88+Li\n6u/JyAghtTe4umwg799PM/Cd5xi4/1byfnQxXb83PrSPSMrYuTMETM3QWbWq6nnNJq8OHaBfPxgw\noO7lgAOaeY1TpJUooNJQaWlVYNUWYkVF+zbP5NgmBubuJm9YDgMHddjnTKx379bvYprOdu0Kt67V\nd/ZTUrLv+/r2rT98+vULZyIi6SDRgOrYGsVIcmRnh3FlDzus9u27d8O6dXHB9ep/WTnzHVau7sSS\n9YN5OeNgtpdl7vOZeXmhnb+yOaayeSb+MdnrEtk/I6MqPN3Dxfv6lt27G94nWftt3157+Hz88b4j\nVPXoEUImLy8MiVUzfPr3D81yIlKdzqDSnTvMmgU33oi//TabDz2RVVfdyspDzmLl6g57z762bq26\nSB7/WN+61hh8PSOjCRNEtqKuXes/8xkwINy/IiJV1MQn1bmHwQJvugkWLw5z2N9+O4we3eQ2Pveq\nwGpsuCW6bvfuEFIZGeH6SkNLsvera9/sbMjNDWdHaiIVaRwFlNRu9+7Q22/y5DAg7cknh3H+Tj89\n6spEpJ1INKDU56e9yciAyy8P/d1/+1tYsSKMln7MMeFGrb/+FT75JOoqRUR0BtXulZbCI4/As8/C\nG2+EgQQzMuCEE+CMM8JI6iedpKv4IpI0auKTxisthblz4R//CDMxzpsXmgSzsmDkyKrAOu642B3C\nIiKNp4CS5vvkE3jttarAevfdsH6//eCznw1hdcYZYap69RQQkQTpPihpvv32g3POCQvAhg0wZ05V\nYD3/fFjft2+4jlUZWAcfrMASkWbTGZQ03apVIaj+8Y+wrFsX1g8cWBVWZ5wRhkEQEYlRE5+0Lvcw\nnXBlYM2eHeZRABgypCqwTj89DFshIu2WAkqitXs3vPNOVWC99loYJdUMhg+vCqxTTgnDMYhIu6GA\nktSyaxf8+99V16/mzg3DRWRmhgHqKgPrxBM1w5xImlNASWrbsSPcd1UZWPPnh2bCrl3h1FOrrl8N\nHap5tUXSjAJK2pbNm+HVV6sC64MPwvrOnWHwYDjkkH2XAQN0P5ZIG6Ru5tK29OwJY8eGBUKPwFde\nCdexli4Ny0svVZ9GNjMTBg2qPbwGDlRToUgbpzMoaTvcQ3BVBlbNJX4a2oyMMAFTbeF18MEaukkk\nQjqDkvRjBgcdFJbTTqu+zR2Ki2sPriefhC1bqn9Obm7t4TV4sHoViqQIBZSkB7MwokXfvmEKkZo2\nbao9vJ59NgRbvH796g6v/fdvnZ9HRNTEJ8LWrbBsWQisysfKZe3a6vv27h3CKj8f+vQJr3v3hpyc\nqueVS+fOkfw4IqlOTXwiidp//3Dz8PDh+27bsSNM7FjzzGvePCgpqd50WFO3bvuGVvxSM9RyckLH\nDxEBFFAi9evaFY4+Oiy1KS8PzYcbN1YtJSXVX1cuH30UHuM7c9S0//6JB1rv3tCrl+4Tk7SlgBJp\njsxMOOCAsCTq009DiNUVZJXLunXw3nvh+c6dtX+WWeiiXxlYPXuGUO3SZd8lOzvx9dnZ0EETbku0\nWiWgzGwUcD+QATzi7ne1xnFFUlLnzlW9ERNVWtpwoG3cGK6ZlZaGQItfmlpnc4Oucn12dgjzjh1r\nXzIy6t7WsaPCsp1q8YAyswzgAeDzQBEwz8xmuvsHLX1skbSRnR26xufmNv697lBWVntwVS6N3bZ5\nc/KCMFH1BVgiIVfXdrMQgPU9Nnefpr6/5vqmvk7me/r2rbvJO9n/yVvhGCcAS919OYCZTQfOAxRQ\nIq3BrOosplevljtOZRDWFWoVFWGU+4qKupfmbq9vn08/DZ1e4reXl4e63WHPnvofk7FPOhg7Fv7y\nl1Y5VGsEVH9gddzrIuDE+B3MbDwwHiAvL68VShKRpIsPQqlbIgEX/7pm8DX1dbLe05J/5NSQEp0k\n3P0h4CEI90FFXI6ISMsxU8/LBLXGlcc1wIC417mxdSIiInVqjYCaBxxqZoPMrBNwMTCzFY4rIiJt\nWKsMdWRmZwO/JHQzn+Lud9SzbzGwMgmH7Q1sTMLntBf6vhpH31fj6PtqvHT+zga6e5+Gdkq5sfiS\nxcwKExnrSQJ9X42j76tx9H01nr6z1mniExERaTQFlIiIpKR0DqiHoi6gjdH31Tj6vhpH31fjtfvv\nLG2vQYmISNuWzmdQIiLShimgREQkJaVdQJnZKDP7yMyWmtmkqOtJZWY2wMxmm9kHZva+mU2Iuqa2\nwswyzGyBmb0QdS2pzsx6mNkMM/vQzBab2UlR15TKzOx7sX+Pi8zsSTPLirqmqKRVQMVN7TEaOAK4\nxMyOiLaqlFYBXOfuRwAjgKv1fSVsArA46iLaiPuBv7n74cCx6Hurk5n1B64FCtz9KMLgBhdHW1V0\n0iqgiJvaw913AZVTe0gt3H2du78de76N8Iujf7RVpT4zywW+BDwSdS2pzsz2B04DHgVw913uviXa\nqlJeRyDbzDoCXYC1EdcTmXQLqNqm9tAv3ASYWT4wDPhXtJW0Cb8EfgSkyQQ/LWoQUAw8FmsSfcTM\nukZdVKpy9zXAfcAqYB2w1d3/Hm1V0Um3gJImMLNuwNPARHf/JOp6UpmZnQNscPf5UdfSRnQEhgO/\ncfdhwA5A14brYGY9Ca0+g4CDgK5mdlm0VUUn3QJKU3s0kpllEsJpmrs/E3U9bcBIYIyZrSA0IZ9h\nZlOjLSmlFQFF7l55Zj6DEFhSu7OA/7p7sbuXA88AJ0dcU2TSLaA0tUcjmJkRrg0sdvefR11PW+Du\n17t7rrvnE/7/esXd2+1fuA1x9/XAajM7LLbqTOCDCEtKdauAEWbWJfbv80zacaeSlJhRN1ncvcLM\nrgFmUTW1x/sRl5XKRgKXA++Z2cLYuhvc/cUIa5L0811gWuyPxuXAlRHXk7Lc/V9mNgN4m9DLdgHt\neMgjDXUkIiIpKd2a+EREJE0ooEREJCUpoEREJCUpoEREJCUpoEREJCUpoEREJCUpoEREJCX9/6Pp\nOadZZWauAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6ef54a04d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(211)\n",
    "plt.title(\"accuracy\")\n",
    "plt.plot(history.history[\"acc\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_acc\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.subplot(212)\n",
    "plt.title(\"loss\")\n",
    "plt.plot(history.history[\"loss\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_loss\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy score: 0.820\n",
      "\n",
      "confusion matrix\n",
      "\n",
      "[[183   1   0   0   0   1   1   0   2   0   0   0   1   0   0   2   3   3   4  38]\n",
      " [  2 206  21  14   4  12  15   3   2   2   1   0   8   1   5   1   0   0   0   2]\n",
      " [  1  23 207  17   5   8   1   0   0   0   2   0   4   1   1   0   1   0   0   0]\n",
      " [  0  16  26 213  16   3  10   1   2   1   0   2   9   0   0   0   4   1   2   0]\n",
      " [  0   5   8  29 224   1  15   1   1   1   0   3   9   1   0   1   1   0   2   0]\n",
      " [  2  35  16   9   0 229   0   2   1   0   2   1   0   1   1   0   0   0   1   0]\n",
      " [  1   3   8   8  14   5 231   5   2   0   3   0   5   0   3   0   0   0   3   2]\n",
      " [  2   0   2   0   5   1   7 238   6   2   3   1   5   0   1   0   1   0   1   2]\n",
      " [  0   0   0   2   0   0   2   6 297   2   4   0   4   0   1   0   3   0   1   0]\n",
      " [  0   0   0   3   1   0   1   2   3 261  33   0   0   2   0   3   0   1   3   2]\n",
      " [  0   0   1   3   1   0   5   0   3   6 261   0   0   0   2   0   1   0   3   0]\n",
      " [  1   3   2   2   0   1   1   1   1   0   0 272   0   1   1   0   4   1   5   0]\n",
      " [  0   3   6  10   8   1  16   5   3   2   0   1 236   1   4   0   2   0   1   0]\n",
      " [  1   3   0   0   1   3   2   2   0   0   0   3   7 265   2   0   2   0   5   0]\n",
      " [  3   6   0   1   0   0   3   1   1   0   1   4   5   2 253   1   0   2   3   5]\n",
      " [ 10   2   0   0   1   2   1   0   0   0   3   1   2   1   3 231   0   4   5  36]\n",
      " [  0   0   1   0   1   0   2   0   2   0   1   4   0   1   0   1 247   4   7   3]\n",
      " [  1   0   0   0   1   0   0   0   0   1   0   0   0   0   2   0   1 272   5   8]\n",
      " [  1   1   0   0   1   0   1   0   3   1   4   0   2   2   2   1  11   3 171   5]\n",
      " [ 13   1   0   0   0   0   0   0   1   0   1   0   1   3   1   8   7   3   9 138]]\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(linewidth=120)\n",
    "Ytest_ = model.predict(Xtest)\n",
    "ytest_ = np.argmax(Ytest_, axis=1)\n",
    "ytest = np.argmax(Ytest, axis=1)\n",
    "print(\"accuracy score: {:.3f}\".format(accuracy_score(ytest, ytest_)))\n",
    "print(\"\\nconfusion matrix\\n\")\n",
    "print(confusion_matrix(ytest, ytest_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}