src/04a-ng-clf-eeap.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embed, Encode, Attend and Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division, print_function\n",
    "from keras.layers import Input\n",
    "from keras.layers.core import Dense, Dropout\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.layers.pooling import GlobalMaxPooling1D\n",
    "from keras.layers.wrappers import TimeDistributed, Bidirectional\n",
    "from keras.models import Model\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "import custom_attn\n",
    "import logging\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import os\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_DIR = \"../data\"\n",
    "\n",
    "VOCAB_FILE = os.path.join(DATA_DIR, \"ng-vocab.tsv\")\n",
    "MIN_OCCURS = 5\n",
    "\n",
    "GLOVE_FILE = os.path.join(DATA_DIR, \"glove.840B.300d.txt\")\n",
    "\n",
    "# covers about 95% of input data\n",
    "MAX_SENTS = 40 # maximum number of sentences per document\n",
    "MAX_WORDS = 60 # maximum number of words per sentence\n",
    "\n",
    "WORD_EMBED_SIZE = 300\n",
    "SENT_EMBED_SIZE = 100\n",
    "DOC_EMBED_SIZE = 50\n",
    "NUM_CLASSES = 20\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "NUM_EPOCHS = 10\n",
    "\n",
    "logging.basicConfig()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Load Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_size: 40730\n"
     ]
    }
   ],
   "source": [
    "word2id = {\"PAD\": 0, \"UNK\": 1}\n",
    "fvocab = open(VOCAB_FILE, \"rb\")\n",
    "for i, line in enumerate(fvocab):\n",
    "    word, count = line.strip().split(\"\\t\")\n",
    "    if int(count) <= MIN_OCCURS:\n",
    "        break\n",
    "    word2id[word] = i\n",
    "fvocab.close()\n",
    "id2word = {v:k for k, v in word2id.items()}\n",
    "vocab_size = len(word2id)\n",
    "print(\"vocab_size: {:d}\".format(vocab_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load GloVe Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40730, 300)\n"
     ]
    }
   ],
   "source": [
    "E = np.zeros((vocab_size, WORD_EMBED_SIZE))\n",
    "E[1] = np.random.random(WORD_EMBED_SIZE)\n",
    "fglove = open(GLOVE_FILE, \"rb\")\n",
    "for line in fglove:\n",
    "    cols = line.strip().split(\" \")\n",
    "    word = cols[0]\n",
    "    if not word2id.has_key(word):\n",
    "        continue\n",
    "    vec = np.array([float(x) for x in cols[1:]])\n",
    "    idx = word2id[word]\n",
    "    E[idx] = vec\n",
    "fglove.close()\n",
    "print(E.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute Document Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18846\n"
     ]
    }
   ],
   "source": [
    "ng_data = fetch_20newsgroups(subset='all',\n",
    "                             data_home=DATA_DIR,\n",
    "                             shuffle=True, \n",
    "                             random_state=42)\n",
    "num_docs = len(ng_data.data)\n",
    "print(num_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['like', 'a', 'mouse']\n",
      "['PAD', 'The', 'cat', 'fought', 'like', 'a', 'mouse']\n"
     ]
    }
   ],
   "source": [
    "def pad_or_truncate(xs, maxlen):\n",
    "    if len(xs) > maxlen:\n",
    "        xs = xs[len(xs) - maxlen:]\n",
    "    elif len(xs) < maxlen:\n",
    "        xs = [\"PAD\"] * (maxlen - len(xs)) + xs\n",
    "    return xs\n",
    "\n",
    "xs = [\"The\", \"cat\", \"fought\", \"like\", \"a\", \"mouse\"]\n",
    "print(pad_or_truncate(xs, 3))\n",
    "print(pad_or_truncate(xs, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18846, 40, 60)\n"
     ]
    }
   ],
   "source": [
    "X = np.zeros((num_docs, MAX_SENTS, MAX_WORDS))\n",
    "for docid in range(num_docs):\n",
    "    text = ng_data.data[docid]\n",
    "    sents = pad_or_truncate(nltk.sent_tokenize(text), MAX_SENTS)\n",
    "    for sid, sent in enumerate(sents):\n",
    "        words = nltk.word_tokenize(sent)\n",
    "        words = pad_or_truncate(words, MAX_WORDS)\n",
    "        for wid, word in enumerate(words):\n",
    "            try:\n",
    "                word_id = word2id[word]\n",
    "            except KeyError:\n",
    "                word_id = word2id[\"UNK\"]\n",
    "            X[docid, sid, wid] = word_id\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18846, 20)\n"
     ]
    }
   ],
   "source": [
    "y = ng_data.target\n",
    "Y = to_categorical(y, num_classes=NUM_CLASSES)\n",
    "print(Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13192, 40, 60) (13192, 20) (5654, 40, 60) (5654, 20)\n"
     ]
    }
   ],
   "source": [
    "Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, train_size=0.7)\n",
    "print(Xtrain.shape, Ytrain.shape, Xtest.shape, Ytest.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Network\n",
    "\n",
    "### Sentence Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 60)                0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 60, 300)           12219000  \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 60, 200)           240600    \n",
      "_________________________________________________________________\n",
      "attention_m_1 (AttentionM)   (None, 200)               260       \n",
      "=================================================================\n",
      "Total params: 12,459,860\n",
      "Trainable params: 12,459,860\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "sent_inputs = Input(shape=(MAX_WORDS,), dtype=\"int32\")\n",
    "sent_emb = Embedding(input_dim=vocab_size,\n",
    "                     output_dim=WORD_EMBED_SIZE,\n",
    "                     weights=[E])(sent_inputs)\n",
    "\n",
    "sent_enc = Bidirectional(GRU(SENT_EMBED_SIZE,\n",
    "                            return_sequences=True))(sent_emb)\n",
    "\n",
    "sent_att = custom_attn.AttentionM()(sent_enc)\n",
    "\n",
    "sent_model = Model(inputs=sent_inputs, outputs=sent_att)\n",
    "sent_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 40, 60)            0         \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, 40, 200)           12459860  \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 40, 100)           75300     \n",
      "_________________________________________________________________\n",
      "attention_m_2 (AttentionM)   (None, 100)               140       \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 50)                5050      \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 20)                1020      \n",
      "=================================================================\n",
      "Total params: 12,541,370\n",
      "Trainable params: 12,541,370\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "doc_inputs = Input(shape=(MAX_SENTS, MAX_WORDS), dtype=\"int32\")\n",
    "\n",
    "doc_emb = TimeDistributed(sent_model)(doc_inputs)\n",
    "\n",
    "doc_enc = Bidirectional(GRU(DOC_EMBED_SIZE,\n",
    "                           return_sequences=True))(doc_emb)\n",
    "\n",
    "doc_att = custom_attn.AttentionM()(doc_enc)\n",
    "\n",
    "fc1_dropout = Dropout(0.2)(doc_att)\n",
    "fc1 = Dense(50, activation=\"relu\")(fc1_dropout)\n",
    "fc2_dropout = Dropout(0.2)(fc1)\n",
    "doc_pred = Dense(NUM_CLASSES, activation=\"softmax\")(fc2_dropout)\n",
    "\n",
    "model = Model(inputs=doc_inputs, outputs=doc_pred)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11872 samples, validate on 1320 samples\n",
      "Epoch 1/10\n",
      "11872/11872 [==============================] - 140s - loss: 2.9606 - acc: 0.0784 - val_loss: 2.6775 - val_acc: 0.1235\n",
      "Epoch 2/10\n",
      "11872/11872 [==============================] - 137s - loss: 2.1478 - acc: 0.2532 - val_loss: 1.4592 - val_acc: 0.5386\n",
      "Epoch 3/10\n",
      "11872/11872 [==============================] - 139s - loss: 1.1226 - acc: 0.6060 - val_loss: 0.8516 - val_acc: 0.7386\n",
      "Epoch 4/10\n",
      "11872/11872 [==============================] - 139s - loss: 0.5574 - acc: 0.8210 - val_loss: 0.7516 - val_acc: 0.7765\n",
      "Epoch 5/10\n",
      "11872/11872 [==============================] - 139s - loss: 0.2927 - acc: 0.9130 - val_loss: 0.7602 - val_acc: 0.7894\n",
      "Epoch 6/10\n",
      "11872/11872 [==============================] - 138s - loss: 0.1517 - acc: 0.9586 - val_loss: 0.8532 - val_acc: 0.7932\n",
      "Epoch 7/10\n",
      "11872/11872 [==============================] - 137s - loss: 0.0917 - acc: 0.9773 - val_loss: 0.9222 - val_acc: 0.7932\n",
      "Epoch 8/10\n",
      "11872/11872 [==============================] - 138s - loss: 0.0516 - acc: 0.9894 - val_loss: 0.9803 - val_acc: 0.7962\n",
      "Epoch 9/10\n",
      "11872/11872 [==============================] - 138s - loss: 0.0394 - acc: 0.9916 - val_loss: 1.0664 - val_acc: 0.7939\n",
      "Epoch 10/10\n",
      "11872/11872 [==============================] - 138s - loss: 0.0292 - acc: 0.9944 - val_loss: 1.0975 - val_acc: 0.8015\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE,\n",
    "                   epochs=NUM_EPOCHS, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VdX59vHvQwiEeQiDSsDgjIqCBEWxlVfUoiiiLeJc\nrYqts21t0dqfQ51abVUUtThTEVTQatVWpXUWlUFUFBUHkIBMQRCEQIb1/vEkJkDmnHP2OSf357r2\nlZ0z7L1yNLlZa6/9LAshICIikmyaRd0AERGRqiigREQkKSmgREQkKSmgREQkKSmgREQkKSmgREQk\nKSmgREQkKSmgREQkKSmgRJKEOf1OipTRL4PIVsxsrJl9YWbrzOxjMzuu0nPnmNn8Ss/tV/Z4TzN7\n0sxWmlmBmd1Z9vjVZvZIpffnmlkws+Zl379iZteb2ZvABmAnMzuz0jm+NLNzt2rfsWY218y+K2vn\nMDMbZWazt3rdr83s6fh9UiLx1TzqBogkoS+AHwHLgFHAI2a2C3AwcDUwEpgF7AwUmVkG8CzwP+A0\noATIq8f5TgOOBD4FDNgdOBr4Evgx8G8zmxlCmGNm+wMTgZ8B/wW2B9oBXwF/N7M+IYT5lY57XUM+\nAJFkoB6UyFZCCE+EEJaGEEpDCI8BC4D9gbOBv4QQZgb3eQhhUdlzOwCXhRC+DyEUhhDeqMcpHwoh\nfBRCKA4hFIUQngshfFF2jleBF/HABDgLeCCE8FJZ+5aEED4JIWwCHgNOBTCzvYBcPDhFUpICSmQr\nZnZ62RDaGjNbA+wNdAF64r2rrfUEFoUQiht4ysVbnf9IM3vbzFaXnf+osvOXn6uqNgA8DJxsZob3\nnh4vCy6RlKSAEqnEzHYE7gUuALJDCB2BefjQ22J8WG9ri4Fe5deVtvI90LrS99tV8ZoflhQws5bA\nNOAWoHvZ+Z8vO3/5uapqAyGEt4HNeG/rZOAfVf+UIqlBASWypTZ4YKwEMLMz8R4UwH3Ab81sQNmM\nu13KAu1d4BvgJjNrY2ZZZja47D1zgR+bWS8z6wBcXsv5WwAty85fbGZHAkdUev5+4EwzG2pmzcys\nh5ntUen5icCdQFE9hxlFko4CSqSSEMLHwF+BGcByoC/wZtlzTwDXA48C64B/Ap1DCCXAMcAuwNdA\nPjC67D0v4deGPgBmU8s1oRDCOuAi4HHgW7wn9Eyl598FzgRuBdYCrwI7VjrEP/BAfQSRFGdasFAk\nfZhZK2AFsF8IYUHU7RFpDPWgRNLLr4CZCidJB7oPSiRNmNlCfDLFyIibIhITGuITEZGkpCE+ERFJ\nSpEN8XXp0iXk5uZGdXoREYnI7NmzV4UQutb2uloDyswewOuCrQgh7F3F8wbcjt/tvgE4I4Qwp7bj\n5ubmMmvWrNpeJiIiacbMFtXldXUZ4nsIGFbD80cCu5ZtY4C763JiERGRmtTagwohvGZmuTW85Fhg\nYvDZFm+bWUcz2z6E8E2M2igiIokUgm+lpb5V3m/RAjIzE9KMWFyD6sGWxS7zyx7bJqDMbAzey6JX\nr14xOLWINBkhQFERbNoEhYXbfq3qsZpeW1xc8ce3/A9wTfv1eW0s9rcOhsY+Vp/31TS7e8IEOOec\nhPwnT+gkiRDCBGACQF5e3jafQFFREfn5+RQWFiayWQmXlZVFTk4OmQn6V4hIzIUAmzfD+vW+ff99\n9V9jFSibNtX8h7OumjeHrCz/2qwZmFV8rct+fV5bn/2MjIrvy/ebNdtyq8tjDX1fXR874IDG/zeo\n63+qGBxjCb4EQLmcssfqLT8/n3bt2pGbm4vPvUg/IQQKCgrIz8+nd+/eUTdH0l0I/oe9cmhUFSQN\neaykpH5tKQ+Gli23/Fp5v02bql9T29e6vrZlS//jLykhFgH1DHCBmU0BDgDWNvT6U2FhYVqHE4CZ\nkZ2dzcqVK6NuiqSaoiJYuhQWL4b8/Ipt6VJYt676ICktrfs5WrSAtm19a9OmYn+HHbZ9rHy/psda\nt64IBwWD1FNdpplPBoYAXcwsH7gKyAQIIdyDr1VzFPA5Ps38zMY0KJ3DqVxT+BmlnjZt8qApD53K\nIVS+v3z5tkNcbdtCjx7Qvr3v5+TUP0TK99u0SdjFb5G6qMssvpNqeT4A58esRSLpZtMmWLJk255P\n5e+XL9/2fe3bQ8+eHjr77utfy78v32/fPvE/j0iCqFhsJWvWrOHRRx/lvPPOq9f7jjrqKB599FE6\nduwYp5ZJ0ios3DJ0qur9VDWc26FDRdj0779t8JT3ikSaMAVUJWvWrOGuu+7aJqCKi4tp3rz6j+r5\n55+Pd9MkKiHAV1/BO+/4161DaNWqbd/TqVNF2OTlbRk8OTkePu3aJf5nEUkxCqhKxo4dyxdffEG/\nfv3IzMwkKyuLTp068cknn/DZZ58xcuRIFi9eTGFhIRdffDFjxowBKso2rV+/niOPPJKDDz6Yt956\nix49evD000/TqlWriH8yqbMNG2DWLJgxo2JbsaLi+c6dK8LmgAO2HXYrvwYkIo2WvAF1ySUwd25s\nj9mvH9x2W7VP33TTTcybN4+5c+fyyiuvMHz4cObNm/fDdPAHHniAzp07s3HjRgYOHMhPf/pTsrOz\ntzjGggULmDx5Mvfeey8nnHAC06ZN49RTT43tzyGxEQIsWgRvvVURRu+/7zdwAuy6KwwbBgceCIMG\nwW67+aw0EUmI5A2oJLD//vtvca/SuHHjeOqppwBYvHgxCxYs2CagevfuTb9+/QAYMGAACxcuTFh7\npRYbN8Ls2Vv2jpYt8+dat4b994fLLoODDvJA6tIl2vaKNHHJG1A19HQSpU2loZpXXnmF6dOnM2PG\nDFq3bs2QIUOqrHjRsmXLH/YzMjLYuHFjQtoqWwkBvv56yzCaO9fvJQLYeWc47DDvHR14IPTt6zeS\nikjS0G9kJe3atWPdunVVPrd27Vo6depE69at+eSTT3j77bcT3DqpUWEhzJmzZSAtXerPtWoFAwfC\nr39d0Tvq1i3a9opIrRRQlWRnZzN48GD23ntvWrVqRffu3X94btiwYdxzzz306dOH3XffnUGDBkXY\nUmHx4i3D6L33vDYcQO/eMGRIRe9on310A6pICrIQi+KLDZCXlxe2XrBw/vz59OnTJ5L2JFpT+lkb\nbdMmD6DyMHrrLb/xFbyMTl5eRRgdeCBst1207RWRGpnZ7BBCXm2vUw9Kks+SJVv2jmbPrugd7bgj\n/OhHFWG0775eP05E0o4CSpLDokVw443w73/75AbwAqMDBsCFF1YE0g47RNtOEUkYBZREa9kyuP56\n+Pvffd2ZESPg0ks9jPr185ASkSZJASXRWL0abr4Zbr/dh+9+8Qv44x+9KkMTV3lR1ZKSioVOq9uv\nz2oaiVbXRVzr83ysjgG1r0NY02PxfH1Jid8RUVzsX7fer+m5ur6uocf4y1/g5JMT8/+PAkoSa906\nD6Wbb/b9k06Cq6/2qg2NUL7Aa/niq/HYylcIr2tw1LafioEjqaV5c5/AWv61pv3K37duXf3rcnIS\n2P7EnUqatMJCuPtuuOEGWLWK9cNHs/y8a1jWYXeWfwjLXvIVJ777ruEB0lgtWlQszrr11rKl/3Jm\nZFSsfB3r/Ya+r/xf4MmqcpurWlG8thXH6/uauh4Dtuytbr2/9df6PtfQ15eW+mdWW6BUFy7l+82b\nJ/f/F3WhgGqEtm3bsn79+qibkTQ2bPCQWb7cLy0tXw7LlpSw/LVPWPbO1ywv3J9lrd5neavufP9c\nBjy35fvNvMh3dSHRvr3fX1vd843ZWrb0P1wikjwUUFKjTZu2CpwavlZdhCODbLqzXatWbLdfRwbt\n0ZnttoPu3dnma5cuqjYkIhX056CSsWPH0rNnT84/3xcIvvrqq2nevDkvv/wy3377LUVFRVx33XUc\ne+yxEbe0cTZv9hUkqgubyvtr1lR9jE6dKsJlv/0qhU33QPev3ma7ybfS/Ys36bbP9mRefzUMH576\n4w0iklBJG1ARrLbB6NGjueSSS34IqMcff5wXXniBiy66iPbt27Nq1SoGDRrEiBEjsBT7YxsCvPii\nz+h+/fWqX9O+fUXo9O0Lhx9e8X3l3k63blXM/g4Bpk+HP/wBZs70pSmm/A1GjdLYmYg0SNIGVBT6\n9+/PihUrWLp0KStXrqRTp05st912XHrppbz22ms0a9aMJUuWsHz5crZLkXI6paXwr3/Bddf5Onw5\nOZ4hPXtuO8TW4HUV33zTD/rqq9CrF9x/P5x+usbrRKRRkvYvSFSrbYwaNYqpU6eybNkyRo8ezaRJ\nk1i5ciWzZ88mMzOT3NzcKpfZSDYlJTBtmveYPvgAdtoJ7r3XcyNmlYHmzoUrr4TnnvOEGzcOxozR\nzbUiEhMae9nK6NGjmTJlClOnTmXUqFGsXbuWbt26kZmZycsvv8yiRYuibmKNiopg4kTYay8YPdqv\nN/3jH/Dpp3D22TEKp08/9YP37++9pxtvhC++8JJECicRiZGk7UFFZa+99mLdunX06NGD7bffnlNO\nOYVjjjmGvn37kpeXxx577BF1E6u0aRM8/DDcdBN89ZXXUH3iCTjuOL+nIiYWLYJrrvETtWrlvaff\n/AY6dozRCUREKiigqvDhhx/+sN+lSxdmzJhR5euS4R6oDRvgvvu8/MiSJb5q+e23w9FHx3DSXOV6\nec2awcUXw9ixWvRPROJKAZWi1q2De+6BW27xKeM/+hE8+KCvYh6zYFq92pNv3DgfKzzrLK+Xl8ha\nJyLSZCmgUsyaNXDHHT6JZPVqnwp+5ZXw4x/H8CTr1vkJbrnF908+2evl7bJLDE8iIlKzpAuoEELK\n3WNUXw1ZxXjVKs+MO+7wenXHHOMzuw84IIYN27jR6+XdeKOfcORI+NOfYO+9Y3gSEZG6SapZfFlZ\nWRQUFDToD3iqCCFQUFBAVlZWnV7/zTfw29/6QrI33AA/+Ymvfv7MMzEMp6Iiv760664+6aF/f3jn\nHXjqKYWTiEQmqXpQOTk55Ofns3LlyqibEldZWVnk1HId5+uv/fLPfff5Mg8nnwyXXw59+sSwISUl\nMHkyXHUVfPklHHQQPPIIDBkSw5OIiDRMUgVUZmYmvXv3jroZkfriCx9he/hhn+zw85/7hLmdd47h\nSUKAf/7TJzx89JHXgHr2WTjqKNXLE5GkkVQB1ZTNn+9DeI8+6uu5/PKXcNllXjkopt5+Gy66yOvl\n7b47PPYY/OxnqpcnIklHARWxuXP9FqNp0/ze10sv9ctA228fh5M98ohPFe/eHR54AE47TfXyRCRp\n6a9TRN55x4PpX//yKuJXXOEV3Lt0icPJQvAKENdc49eXnnzS18sQEUliCqgEe+01ryz+0kvQubPP\n4r7ggjhWC9q0yXtNkybBGWf4bL2YVYsVEYkfBVQClC+V9Kc/+VpM3bvDzTf7daa2beN44lWrvBjf\nG294d+3yyzUJQkRShgIqjkLwyXHXXQfvvusVgsaN86riDV57qa4++8xn5eXnw5QpXn1cRCSFKKDi\noLS0Yi2m99+H3r1hwgRfiykhq1G8+qr3nJo3h5dfhgMPTMBJRURiS3OLY6ykxO9dOuEEKCz0+5k+\n/RTOOSdB4TRxYsVa7W+/rXASkZRVp4Ays2Fm9qmZfW5mY6t4/gwzW2lmc8u2s2Pf1ORXUgJnnumz\nua+91u+BPf10v68p7kLwG29//nMvbT5jhi+jKyKSomod4jOzDGA8cDiQD8w0s2dCCB9v9dLHQggX\nxKGNKaGkBH7xC1+99rrrvJBrwhQWejJOmeKNuPtuzdQTkZRXlx7U/sDnIYQvQwibgSnAsfFtVmop\nKfGJDxMnes8poeG0ciUMHerhdNNNXrxP4SQiaaAuAdUDWFzp+/yyx7b2UzP7wMymmlnPqg5kZmPM\nbJaZzUqXgrClpX596aGH/D7YP/4xgSf/5BMYNAjmzPH13X//e00jF5G0EatJEv8CckMI+wAvAQ9X\n9aIQwoQQQl4IIa9r164xOnV0SkthzBhfyfaqq+D//i+BJ//f/3wCxPr18MorXk9PRCSN1CWglgCV\ne0Q5ZY/9IIRQEELYVPbtfcCA2DQveZWWwrnnwv33e6/pqqsSePIHH/SFoXbYwWsmxXTVQhGR5FCX\ngJoJ7Gpmvc2sBXAi8EzlF5hZ5dKmI4D5sWti8ikthV/9yi/3/OEPPrSXkJG10lIv2veLX3hNvTff\nhNzcBJxYRCTxap3FF0IoNrMLgBeADOCBEMJHZnYtMCuE8AxwkZmNAIqB1cAZcWxzpEpL4fzz/cbb\nyy/38kUJCaeNG72W3uOP+0Wv8eMTNH9dRCQaFtXy6nl5eWHWrFmRnLuhQvBwuvtuX0TwhhsSFE4r\nVsCxx/pw3l/+4utxaDKEiKQoM5sdQsir7XUqdVRHIcCFF3o4/e53CQynjz+Go4+GZctg6lQ4/vgE\nnFREJHoKqDoIAS6+2EfVfvtbv90oIeE0fbrPzsvK8vp6Awcm4KQiIslBtfhqEYIvJHjHHfDrX/sI\nW0LC6b774MgjoWdPH9pTOIlIE6OAqkEIvgT7uHH+9ZZbEhBOpaV+geucc7xCxJtvwo47xvmkIiLJ\nR0N81QjB5yLcfrsP7/31rwkIpw0bvLrstGm+muEdd/iSGSIiTZD++lUhBLjsMrj1Vp8YceutCQin\nZct8pt7MmZ6Gl16qmXoi0qQpoLYSgpe0++tf4YILvAcV95z46CMYPtwLvz75JIwcGecTiogkP12D\nqiQEv/xz881w3nl+7Snu4fTSS3DQQbB5M7z2msJJRKSMAqpMCF5F6C9/8TJGd96ZgHCaMMFn6uXm\n+ky9AWlfwlBEpM4UUHg4XXml39907rkJCKfSUr/Ide65cMQR8MYbPp1cRER+0OSvQYXgy2TccIPP\n7L7rLmgWz9jesAFOPRWeesrrJt12m2bqiYhUocn/Zbz6al+i/eyz4Z574hxO33wDI0bA7NkeTBdd\npJl6IiLVaNIBdc01vkT7L34Bf/97nMPpww99pt7q1fD003DMMXE8mYhI6muy16CuvdZ7T2ecAffe\nG+dw+s9/YPBgKCmB119XOImI1EGTDKjrrvMVcH/+cy95F9dwuvtur0a+884+U69//zieTEQkfTS5\ngLrhBl+i/bTTfLn2jIw4naikxKvLnnceDBvmPaecnDidTEQk/TSpgLrxRl+i/dRT4cEH4xhO337r\n6zbdeqtPhHj6aWjbNk4nExFJT01mksSf/+w34p58Mjz0UJzC6YMP/CaqRx6BTZu82OsFF8ThRCIi\n6a9JBNTNN3sJo5NOgocfjnE4FRV5D+mOO7xUUatWcMopXmV2n31ieCIRkaYl7QPqr3/1JdpHj4aJ\nE2N4T+yKFT797+67YckSL1d0880+Z71z5xidRESk6UrrgPrb33yJ9hNO8FG3mITTzJneW3rsMS/w\nevjhXn5i+PA4XtQSEWl60jagbrvNFxwcNQomTWpkOG3aBE884cH07rs+4WHMGC9VtMceMWuziIhU\nSMuAKl+i/ac/bWQ45ed7iYkJE3xIb/fdPaROPx3at49pm0VEZEtpF1B33ulLtB93HEyeDJmZ9TxA\nCH7P0p13+uKBpaV+o+2FF8LQoXG+q1dERMqlVUCNH+85MnIkTJlSz3DasMG7W3fe6dPFO3Xybth5\n50Hv3nFrs4iIVC1tAuquu/yWo2OP9fkLLVrU8Y1ffulvvv9+WLPGp4bfe6/fMNW6dVzbLCIi1UuL\ngLrnHp+vcMwx8PjjdQin0lKYPt2vJz33nA/bHX+8d78OPlhLYIiIJIGUD6gJE3yJ9qOP9ol2NYbT\nd9/5nbp33gmffQbdunnto1/+Enr0SFibRUSkdikdUPfe66umDx8OU6dCy5bVvHD+fL9A9fDDsH49\nHHAA/OMfPge92jeJiEiUUjagXnrJb0U66iiYNq2KnCkpgWef9d7S9OnetTrxRL9QNXBgJG0WEZG6\nS9mAGjLEC8BedNFW4VRQ4BMe7roLFi3yJS6uv97XdO/WLarmiohIPaVsQGVmeo29H8yd672lSZOg\nsBAOOcQL8R17bAwL8ImISKKk9l/uoiK/mfbOO+GNN7yS+Omn+zBe375Rt05ERBohdQNqxgyvZfTN\nN7DTTt5bOvNMv8FWRERSXuoG1O67Q16eT+MbNkyVxEVE0kzqBlTnzvDMM1G3QkRE4kSVT0VEJCkp\noEREJClZCCGaE5utBBbF4FBdgFUxOE5Tos+sYfS51Z8+s4ZJ989txxBC19peFFlAxYqZzQoh5EXd\njlSiz6xh9LnVnz6zhtHn5jTEJyIiSUkBJSIiSSkdAmpC1A1IQfrMGkafW/3pM2sYfW6kwTUoERFJ\nT+nQgxIRkTSkgBIRkaSUsgFlZsPM7FMz+9zMxkbdnlRgZj3N7GUz+9jMPjKzi6NuU6owswwze8/M\nno26LanCzDqa2VQz+8TM5pvZgVG3KdmZ2aVlv5vzzGyymWVF3aYopWRAmVkGMB44EtgTOMnM9oy2\nVSmhGPhNCGFPYBBwvj63OrsYmB91I1LM7cB/Qgh7APuiz69GZtYDuAjICyHsDWQAJ0bbqmilZEAB\n+wOfhxC+DCFsBqYAx0bcpqQXQvgmhDCnbH8d/gejR7StSn5mlgMMB+6Lui2pwsw6AD8G7gcIIWwO\nIayJtlUpoTnQysyaA62BpRG3J1KpGlA9gMWVvs9Hf2jrxcxygf7AO9G2JCXcBvwOKI26ISmkN7AS\neLBsaPQ+M2sTdaOSWQhhCXAL8DXwDbA2hPBitK2KVqoGlDSCmbUFpgGXhBC+i7o9yczMjgZWhBBm\nR92WFNMc2A+4O4TQH/ge0LXiGphZJ3wkqDewA9DGzE6NtlXRStWAWgL0rPR9TtljUgszy8TDaVII\n4cmo25MCBgMjzGwhPpR8qJk9Em2TUkI+kB9CKO+hT8UDS6p3GPBVCGFlCKEIeBI4KOI2RSpVA2om\nsKuZ9TazFviFRK1eWAszM/yawPwQwt+ibk8qCCFcHkLICSHk4v+f/S+E0KT/VVsXIYRlwGIz273s\noaHAxxE2KRV8DQwys9Zlv6tDaeITS1JyRd0QQrGZXQC8gM90eSCE8FHEzUoFg4HTgA/NbG7ZY1eE\nEJ6PsE2Svi4EJpX9I/JL4MyI25PUQgjvmNlUYA4+4/Y9mnjJI5U6EhGRpJSqQ3wiIpLmFFAiIpKU\nFFAiIpKUFFAiIpKUFFAiIpKUFFAiIpKUFFAiIpKUFFAiIpKUFFAiIpKUFFAiIpKUFFAiIpKUFFAi\nIpKUFFAiMWRmC83ssKjbIZIOFFAiIpKUFFAiIpKUFFAicWBmLc3sNjNbWrbdZmYty57rYmbPmtka\nM1ttZq+bWbOy535vZkvMbJ2ZfWpmQ6P9SUSik5Ir6oqkgD8Ag4B+QACeBq4E/gj8BsgHupa9dhAQ\nypZHvwAYGEJYama5+IrRIk2SelAi8XEKcG0IYUUIYSVwDXBa2XNFwPbAjiGEohDC68GXti4BWgJ7\nmllmCGFhCOGLSFovkgQUUCLxsQOwqNL3i8oeA7gZ+Bx40cy+NLOxACGEz4FLgKuBFWY2xcx2QKSJ\nUkCJxMdSYMdK3/cqe4wQwroQwm9CCDsBI4Bfl19rCiE8GkI4uOy9AfhzYpstkjwUUCLxMRm40sy6\nmlkX4P+ARwDM7Ggz28XMDFiLD+2VmtnuZnZo2WSKQmAjUBpR+0Uip4ASiY/rgFnAB8CHwJyyxwB2\nBaYD64EZwF0hhJfx6083AauAZUA34PLENlskeZhfmxUREUku6kGJiEhSUkCJiEhSUkCJiEhSUkCJ\niEhSilmpIzPLAl7DZyI1B6aGEK6q7vVdunQJubm5sTq9iIikiNmzZ68KIXSt7XWxrMW3CTg0hLDe\nzDKBN8zs3yGEt6t6cW5uLrNmzYrh6UVEJBWY2aLaXxXDgCqrJba+7NvMsk1z2EVEpEFieg3KzDLM\nbC6wAngphPDOVs+PMbNZZjZr5cqVjTvZ5s1w1VXw7beNO46IiCSlmAZUCKEkhNAPyAH2N7O9t3p+\nQgghL4SQ17VrrcOPNXv9dbj+ethjD3jkEdANxyIiaSUu60GFENaY2cvAMGBePM7B0KEwaxb88pdw\n2mnwwANw110eWCIiSayoqIj8/HwKCwujbkpcZWVlkZOTQ2ZmZoPeH8tZfF2BorJwagUcTrwrMffr\nB2+9BffeC2PHwj77wO9/D1dcAa1axfXUIiINlZ+fT7t27cjNzcVrBqefEAIFBQXk5+fTu3fvBh0j\nlkN82wMvm9kHwEz8GtSzMTx+1Zo1g3PPhU8+gRNPhOuug733hv/8J+6nFhFpiMLCQrKzs9M2nADM\njOzs7Eb1EmMWUCGED0II/UMI+4QQ9g4hXBurY9dJ9+4wcSL897+QmQlHHgknnABLliS0GSIidZHO\n4VSusT9j+lWSOPRQeP9970n961/Qpw+MGwfFxVG3TERE6iH9AgqgZUv4wx9g3jwYPBguvhj23x/e\nfTfqlomIRG7NmjXcdddd9X7fUUcdxZo1a+LQoqqldEDNnVvLC3beGZ5/Hh5/HJYvh0GD4PzzIYEf\nsIhIsqkuoIprGWl6/vnn6dixY7yatY2UDaiXXoL+/eGaa2q5BcoMRo2C+fPhoovgnnt8Kvqjj+re\nKRFpksaOHcsXX3xBv379GDhwID/60Y8YMWIEe+65JwAjR45kwIAB7LXXXkyYMOGH9+Xm5rJq1SoW\nLlxInz59OOecc9hrr7044ogj2LhxY8zbGdmKunl5eaExtfiKi+Gcc+Chh+CCC+D2231CX63mzPF7\np2bO9Hup7roLdtutwe0QEamv+fPn06dPH//mkkvqMBxUT/36wW23Vfv0woULOfroo5k3bx6vvPIK\nw4cPZ968eT9MB1+9ejWdO3dm48aNDBw4kFdffZXs7OwfaqiuX7+eXXbZhVmzZtGvXz9OOOEERowY\nwamnnlrzz1rGzGaHEPJq+zFStgfVvLnfm/ub38Cdd/q9ukVFdXjjfvvBjBkeTLNmQd++XjIpzW+Y\nExGpzv7777/FvUrjxo1j3333ZdCgQSxevJgFCxZs857evXvTr18/AAYMGMDChQtj3q64VJJIFDO4\n+Wbo0gXFlK6MAAAUq0lEQVQuv9wvLT3xBLRuXcsbMzLgV7+C446D3/4Wrr3Wh/zGj4cjjkhI20VE\ngBp7OonSpk2bH/ZfeeUVpk+fzowZM2jdujVDhgyp8l6mli1b/rCfkZERlyG+lO1BlTPzIhJ//zv8\n+9+eL3WeA7Hddl7Hb/p0Hx/8yU/8Zt9vvolrm0VEotSuXTvWrVtX5XNr166lU6dOtG7dmk8++YS3\n365yxaSESPmAKjdmjE/WmzkTDjmknhkzdCh88IH3pP75T59EceedUFISt/aKiEQlOzubwYMHs/fe\ne3PZZZdt8dywYcMoLi6mT58+jB07lkGDBkXUyhSeJFGd6dNh5EgvLPHSS7DTTvU8wOef+1T0F1+E\nAQN81l9erdfyRETqrKqJA+mqSU6SqM5hh8H//ufDfIMHe8eoXnbZxev4TZkCS5f6Db4XXghr18al\nvSIiUrW0CyjwTHn9dZ8Lccgh8Oab9TyAGYwe7fdOXXBBxTIeU6bo3ikRkQRJy4AC2HNPD6auXeHw\nw30CRb116OB1/N59F3Jy4KSTfCJFFVMuRUQkttI2oAB23BHeeMM7PyNGwOTJDTzQgAHw9ts+ceKd\nd/zeqWuu0b1TIiJxlNYBBdCtG7zyil+POuUUv9WpQTIyfPLEJ5/A8cfD1Vf7AonTp8ewtSIiUi7t\nAwqgfXuf93DMMX5J6dprG3Epafvt/abeF1/0gxx+OJx8MixbFtM2i4g0dU0ioACysmDaNPj5z72y\n0cUXQ2lpIw54+OHw4Yfek5o2zccRx4/XvVMiknbatm0byXmbTEBBRf2+X/8a7rgDTj+9jvX7qpOV\n5Wn34YcwcKB3zwYN8oK0IiLSKE0qoMArGt1yC9xwA0ya5OX4Nmxo5EF3282H/CZPhvx8D6tLLtEk\nChFJSmPHjmV8pQvyV199Nddddx1Dhw5lv/32o2/fvjz99NMRttClXSWJ+pgwwVfeGDzYV4ePyTpc\na9f6ar7jx3vl9GnTIDc3BgcWkXRRubpCBKtt8N5773HJJZfw6quvArDnnnvywgsv0KFDB9q3b8+q\nVasYNGgQCxYswMxo27Yt69evb1BbIq8kYWY9zexlM/vYzD4ys4tjcdx4GzMGHnvMZ44fckiM5jl0\n6ODT0Z95Br74wkOqQTdhiYjER//+/VmxYgVLly7l/fffp1OnTmy33XZcccUV7LPPPhx22GEsWbKE\n5cuXR9rOWC23UQz8JoQwx8zaAbPN7KUQwscxOn7cjBrlmXLccXDwwV6/r9KyKA13zDEwezb89Kcw\nfDj83//5VqdVFUWkqYhqtY1Ro0YxdepUli1bxujRo5k0aRIrV65k9uzZZGZmkpubW+UyG4kUk7+W\nIYRvQghzyvbXAfOBHrE4diIccQT897/w7bc+3PfhhzE68M47++KIp5/uN/YOHw4FBTE6uIhIw40e\nPZopU6YwdepURo0axdq1a+nWrRuZmZm8/PLLLFq0KOomxn6ShJnlAv2Bd2J97HgaNAhee83L8P34\nx/DWWzE6cKtW8OCDvmDV//7nVSlmz47RwUVEGmavvfZi3bp19OjRg+23355TTjmFWbNm0bdvXyZO\nnMgee+wRdRNjO0nCzNoCrwLXhxCerOL5McAYgF69eg1IhoTe2sKF3qPKz4cnn4Rhw2J48Jkz4Wc/\ng+XL/TrV2WfH8OAikiq03EaCl9sws0xgGjCpqnACCCFMCCHkhRDyunbtGqtTx1RurldC3313v4w0\nZUoMDz5woPeeDjkEzjkHzjoL4rBMsohIOojVLD4D7gfmhxD+FotjRql7d6/fd9BBXsXo7rtjePAu\nXeD55+GPf/S7hgcPhi+/jOEJRETSQ6x6UIOB04BDzWxu2XZUjI4diQ4dKur3nXce/OlPMVwKKiPD\nCwI++yx89ZVfl3ruuRgdXERSQVT3oCZSY3/GWM3ieyOEYCGEfUII/cq252Nx7Ci1auX32Z5+us8Q\nv+SSRtbv29rw4T7kl5sLRx/tJ1EtP5G0l5WVRUFBQVqHVAiBgoICsrKyGnyMWN0HlbaaN/dJeJ07\n+/0Kq1f7yFxmZoxOsNNOPmXw/PO9m/bOO16DqUuXGJ1ARJJNTk4O+fn5rFy5MuqmxFVWVhY5OTkN\nfr8Cqg6aNYO//c0z48orYc0aePxx72HFRKtWnnoHHeQFZwcMgKlTfVKFiKSdzMxMesekIkB6U1mD\nOjLzEnt33+2Xi37yEw+qmDr7bF8C2MzLWkyYEMMLXyIiqUUBVU+//KVPPX/7bRgyJA7rFObl+XWp\nQw+Fc8+FM8/UVHQRaZIUUA1wwgle/XzBAu/ofPVVjE+Qne0z/K66CiZOhAMP9MKzIiJNiAKqgX7y\nE6/ft3q138o0b16MT5CR4av1PvccfP21X5d69tkYn0REJHkpoBph0CCvOlFev2/GjDic5Mgjfchv\n5539pqwrr9RUdBFpEhRQjbTXXvDmmz4qd9hh8MILcThJ795+krPOguuv99BatSoOJxIRSR4KqBjI\nzfXJd7vt5p2cxx6Lw0mysuC++3x77TVfCPHdd+NwIhGR5KCAipHy+n2DBsFJJ8Edd8S46kS5s87y\nG3szMnyGxj33aCq6iKQlBVQMdejgQ3xHHw0XXQR9+8JDD8HmzTE+0X77+XWpww6DX/0KzjgDNmyI\n8UlERKKlgIqxVq3gqafgH//wMklnnunVjG65Bb77LoYn6tzZZ/Vdc42f7MAD4fPPY3gCEZFoKaDi\nICMDTj0V5s71iui77w6XXQY9e8LYsbB0aYxO1KyZF5h9/nlfYTEvD555JkYHFxGJlgIqjswq7pea\nOdNX5735Zp9UcdZZMH9+jE40bJgP+e2yCxx7LFxxhaaii0jKU0AlSF6ez+777DNfTPfRR2HPPT1P\n3nwzBicon0p4zjlw442ejGleKVlE0psCKsF23hnGj/fiEFdd5Zly8MFejeLppxs58y8rywvMPvCA\np95++/nyHSIiKUgBFZGuXb2S0ddf+5T0pUth5EjvVd13H2za1IiDn3mmT0XPzIQf/QjuuktT0UUk\n5SigItamjS8BtWABTJ4MrVv7KF1uLtx0UyOW9Ojf369LHXGEL4Z4+unw/fexbLqISFwpoJJE8+Zw\n4omeKS+95PdQXX459OoFv/2tT9Krt06dfFbfn/7kq/QeeKAnoYhIClBAJRkzv//2xRdhzhwvnXTb\nbV6O74wzGlA1vVkzLzD7n//4OGJens/Q0Cw/EUlyFiK6NpGXlxdmzZoVyblTzcKFcOutfm1qwwYY\nPhx+9zu/vGRWjwMtWgSjRvmc99xcH/o76yzvaYlIyikthaIir1ZT27ZpU91eV9t27rkwdGjj2m1m\ns0MIebW+LlYBZWYPAEcDK0IIe9f2egVU/RUU+HyHceO8mPkBB/gNwCNH+s3BdVJc7NMFx43zorOt\nW8Npp8GFF3ppdhGJuaIiWLsWvv22Yluzpvr9DRvqFhbFxfFpb2YmtGhR9XbNNf7v3MaIIqB+DKwH\nJiqg4mvjRq/x99e/+kK7u+zi16lOP91LLdXZ++/7FMJJk6Cw0JeZv+giLyZY58QTSX8h+K9IbeFS\n3XPr19d8/Kws6NjRBzM6dvTJU9UFRG1by5YNf2+LFh5O9RqZaYCEB1TZSXOBZxVQiVFS4nX//vxn\nmDULunXzfPnVr7xUX50VFPj44fjxsHixhv8k7Wze7JNY16/3r99/77Uxa+vJVA6e2oo+t2tXETCd\nOlVslb+vbj8rKzGfQ7JIyoAyszHAGIBevXoNWLRoUczO3ZSFAK++Cn/5C/z73/6vr3POgUsv9VmA\ndVY+/HfHHX5ADf9JAhUXV4RH+VY5UBrzfV2Gwpo1qz5AagqXTp18JYPmzeP/GaWLpAyoytSDio8P\nPvDK6ZMne3CddJJfp9pnn3oeSMN/UoOSEli3zrfvvtt2v/JjdQ2T+t6cnpUFbdv6P8jKt5q+3/q5\n8h5P+dauXfyHtsQpoJq4xYt9evqECf6HYNgwn/k3ZEg9fwk1/Jc2iotrDpKtH6vpubouP5aZWb8Q\nqS1Uyr9v3Vr/TkplCigBfOz8nnvg9tth+XIYMMAnU3TvDl26QHZ2xdcaJ1hUNfx36qk+/Ld3rf+5\npZFKS30WWEEBrF695fbtt3ULl40b63aurCzvTbRrB+3bb7tf1WPVPd+yZXw/F0lNUczimwwMAboA\ny4GrQgj3V/d6BVRiFRb6uoa33OIV1avSqtW2oVV5/4fHCj6jy9P3k/30A7TdtAo79FAPqmOO0T9r\na1Fc7IGydcjUtn37bc3lFFu1anyYlO9nZibu85CmKZIeVH0ooKIRgvekCgp8W7Vq2/2tH6vpj2OL\njGKyQwHZpSvokrWe7F2zyd5vR7r0aFlt0HXo4BekU9mmTfUPmdWra15V2cxHTTt3rt/WsaNCRVJL\nXQNK806aGDPYbjvf6qqkxEOq6iBrTsHKrqyaV0LBp6V8/GExBR+uocC6UBKq7k1lZPgf1qoCrGVL\nD8PSUt8as9/Y9299rLVrK4Kmpmsw5T9f+bb99j4Jsrag6dBBHVCRyhRQUquMDA+PLl2qe0UzYAff\n3n8f7vgD4ZFJrN3UkoJBR7PquHMo6HMwBWsyquylffmlV19atcrvNWnWzDez2OzH4lgZGV4PccCA\n2oNGs8FEYkNDfBIfmv0nItWo6xBfil8JkKSVnQ2//713j6ZNgx139BuyevTwapP1LssuIk2NAkri\nq3lzOP54eOUVmDsXTj4ZJk70Ba8OPRT++U8t/SEiVVJASeLsu68P++Xn+3LBn38Oxx3n1W5vvtln\nH4iIlNE1KIlOcbGv+DtunN/826qVL1Gfl+ezEQYM8Aq4IpJWNM1ckl/58N/xx3sRwfHjfSjw6acr\nXtOzpwdV5dDq2jWyJotI4iigJDnssw/8/e++v3YtvPcezJ7t64jMnu3Xqsr16lURVuXBVf0ceBFJ\nUQooST4dOnhV2yFDKh5buxbmzPGwKg+up56qeL5Xry17WQotkZSngJLU0KED/L//51u5NWu8p1Xe\ny5o1C558suL5HXfcdngwOzvxbReRBlFASerq2LHq0JozpyK0Zs/eMrRyc7cdHqzX8sMikigKKEkv\nHTv6/VWHHlrx2LffVgwPlgfXtGkVz+fmVoRVXh7st59CSyQJKKAk/XXqBEOH+lZu9eptQ2vq1Irn\ne/fe9pqWSjSJJJQCSpqmzp3hsMN8K1dQsOVEjJkz4YknKp7v1AlycrxcU48eVe937qxKsSIxooAS\nKZedDYcf7lu58tCaMwe+/hqWLPFKGHPn+sJaW9/onpVVc4D16OHrbzTXr55IbfRbIlKTqkKrXFER\nfPNNRWgtWbLl/owZ/nXz5i3f16wZdO9ee2+sTZvE/IwiSUoBJdJQmZl+/1WvXtW/JgTvhVUVYEuW\nwIIFXj1jzZpt39uhQ0VYVRdmXbpoSFHSlgJKJJ7MKlZ77Nev+td9/33VAVa+P28eLFvmS/tW1rKl\nDxm2b+89rrZtK77WZX/rx1q39h6eSBJQQIkkgzZtYLfdfKtOcbGHVOXgys/3Ycb16yu2Zcv86/ff\n+9ea1qevri2xCLvyr1lZvrVoofCTeolZQJnZMOB2IAO4L4RwU6yOLSL4xIqcHN8OOKDu7yst9ZAq\nD7Dy4Kpuv6rH1q2rCL7yx+sbfOAhVR5YLVtW7Df2sbq+r0ULDYmmkJgElJllAOOBw4F8YKaZPRNC\n+DgWxxeRRmjWrKJnE0slJR5SNQXb99/Dpk1QWLjltvVj5d9//71fs6vqNYWF2w5xNkR5cGVmQkaG\nfz7NmlXsV/VYLJ+v72tr2szq9rpYbn36wA47NP6/Qx3Eqge1P/B5COFLADObAhwLKKBE0lVGBrRr\n51uiFBfXHHQ1hd/Wj23e7IFXUuJf67Jf3fNFRX7Mhr6/uv2qtojW8PvBvffC2Wcn5FSxCqgewOJK\n3+cD24xBmNkYYAxAr5pmPomIVKV58/j0BlNJCL5tHVo1hVost112SdiPmtBJEiGECcAE8BV1E3lu\nEZG0YFYxtJfmYvUTLgF6Vvo+p+wxERGRBolVQM0EdjWz3mbWAjgReCZGxxYRkSbIQowuuJnZUcBt\n+DTzB0II19fy+pXAohicuguwKgbHaUr0mTWMPrf602fWMOn+ue0YQuha24tiFlBRMbNZIYS8qNuR\nSvSZNYw+t/rTZ9Yw+txc+l9lExGRlKSAEhGRpJQOATUh6gakIH1mDaPPrf70mTWMPjfS4BqUiIik\np3ToQYmISBpSQImISFJK2YAys2Fm9qmZfW5mY6NuTyows55m9rKZfWxmH5nZxVG3KVWYWYaZvWdm\nz0bdllRhZh3NbKqZfWJm883swKjblOzM7NKy3815ZjbZzLKiblOUUjKgKi3vcSSwJ3CSme0ZbatS\nQjHwmxDCnsAg4Hx9bnV2MTA/6kakmNuB/4QQ9gD2RZ9fjcysB3ARkBdC2BsvenBitK2KVkoGFJWW\n9wghbAbKl/eQGoQQvgkhzCnbX4f/wegRbauSn5nlAMOB+6JuS6owsw7Aj4H7AUIIm0MIa6JtVUpo\nDrQys+ZAa2BpxO2JVKoGVFXLe+gPbT2YWS7QH3gn2pakhNuA3wExWC2vyegNrAQeLBsavc/M2kTd\nqGQWQlgC3AJ8DXwDrA0hvBhtq6KVqgEljWBmbYFpwCUhhO+ibk8yM7OjgRUhhNlRtyXFNAf2A+4O\nIfQHvgd0rbgGZtYJHwnqDewAtDGzU6NtVbRSNaC0vEcDmVkmHk6TQghPRt2eFDAYGGFmC/Gh5EPN\n7JFom5QS8oH8EEJ5D30qHlhSvcOAr0IIK0MIRcCTwEERtylSqRpQWt6jAczM8GsC80MIf4u6Pakg\nhHB5CCEnhJCL/3/2vxBCk/5XbV2EEJYBi81s97KHhgIfR9ikVPA1MMjMWpf9rg6liU8sSeiKurES\nQig2swuAF6hY3uOjiJuVCgYDpwEfmtncsseuCCE8H2GbJH1dCEwq+0fkl8CZEbcnqYUQ3jGzqcAc\nfMbtezTxkkcqdSQiIkkpVYf4REQkzSmgREQkKSmgREQkKSmgREQkKSmgREQkKSmgREQkKSmgREQk\nKf1/yCyByK0CwDoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7b6a1d2e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(211)\n",
    "plt.title(\"accuracy\")\n",
    "plt.plot(history.history[\"acc\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_acc\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.subplot(212)\n",
    "plt.title(\"loss\")\n",
    "plt.plot(history.history[\"loss\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_loss\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy score: 0.824\n",
      "\n",
      "confusion matrix\n",
      "\n",
      "[[196   1   0   0   0   0   0   0   0   1   0   1   0   0   0   2   0   2   9  34]\n",
      " [  1 220  13   4   2  15   4   0   0   1   0   1   7   0   4   1   3   2   4   0]\n",
      " [  1   8 239  15  10  24   0   0   2   3   2   2   3   0   2   0   0   0   0   1]\n",
      " [  0  12  26 208  23   3   9   2   0   0   1   1  17   1   0   0   1   0   0   0]\n",
      " [  0   8   7  28 190   1   4   2   1   1   0   4  20   4   1   0   0   0   1   0]\n",
      " [  1  16  17   4   2 245   0   0   0   1   1   2   1   1   3   0   0   0   0   0]\n",
      " [  0   6   5  13   6   0 203   8   6   1   2   0  12   1   2   2   1   1   3   0]\n",
      " [  0   2   2   0   1   0   9 235  14   0   2   0   6   1   7   2   2   0   2   0]\n",
      " [  4   1   0   0   0   0   1   4 268   0   1   0   0   0  15   0   1   0   2   1]\n",
      " [  0   0   2   1   0   0   0   1   1 267  18   0   0   1   2   2   7   0   2   0]\n",
      " [  0   0   0   0   0   0   3   2   1  25 253   0   1   0   1   0   1   0   1   0]\n",
      " [  1   7   6   0   2   1   0   0   1   1   0 271   1   0   1   0   2   3   8   1]\n",
      " [  0   4   2   9   5   0   7   4   2   2   0   0 260   1   6   1   1   0   1   0]\n",
      " [  0   4   1   0   3   0   1   1   0   1   1   0   1 277   4   0   0   2   5   6]\n",
      " [  1   3   0   0   0   0   1   2   3   0   0   0   5   3 255   0   2   0   8   0]\n",
      " [  3   0   0   0   0   0   1   2   0   2   1   0   0   3   2 238   1   4   2  40]\n",
      " [  0   0   0   0   0   1   0   0   1   4   1   1   1   1   1   1 251   1  21   3]\n",
      " [  0   1   0   0   0   0   0   1   0   3   0   0   0   2   0   2   1 268   9   2]\n",
      " [  1   2   2   0   0   0   0   0   0   2   1   2   0   2   3   3  11   2 190   9]\n",
      " [ 14   1   0   0   0   0   0   1   2   1   0   0   1   2   3  19   4   4  13 125]]\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(linewidth=120)\n",
    "Ytest_ = model.predict(Xtest)\n",
    "ytest_ = np.argmax(Ytest_, axis=1)\n",
    "ytest = np.argmax(Ytest, axis=1)\n",
    "print(\"accuracy score: {:.3f}\".format(accuracy_score(ytest, ytest_)))\n",
    "print(\"\\nconfusion matrix\\n\")\n",
    "print(confusion_matrix(ytest, ytest_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}