Task3-Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import tensorflow as tf\n",
    "import tensornets as nets\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from tensorflow.python.saved_model import tag_constants\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## List of tasks:\n",
    "\n",
    "1. Load images as arrays\n",
    "2. Load labels\n",
    "3. Make x and y training + dev arrays\n",
    "4. Load pre-trained architecture (eg. VGG19)\n",
    "5. Train with new data\n",
    "6. Make predictions\n",
    "7. Set up metrics - accuracy, precision, recall. Output train and dev metrics.\n",
    "8. Tune hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_file = 'X_final.npy'\n",
    "X = np.load(X_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22740, 224, 224, 3)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Flip images and labels\n",
    "\n",
    "m, n_H0, n_W0, n_C0 = X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_file = 'Y_final.npy'\n",
    "Y = np.load(Y_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22740, 7)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_loc = '../ISIC2018_Task3_Training_GroundTruth/ISIC2018_Task3_Training_GroundTruth.csv'\n",
    "\n",
    "def get_Y_orig(labels_loc):\n",
    "    Y_df = pd.read_csv(labels_loc)\n",
    "    Y_orig = np.array(Y_df.iloc[:,1:])\n",
    "    return Y_orig\n",
    "\n",
    "Y_orig = get_Y_orig(labels_loc)\n",
    "\n",
    "class_weights = np.array(np.mean(Y_orig, axis = 0)).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_index = np.load('train_index_with_mirror.npy')\n",
    "dev_index = list(set(range(m)) - set(train_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make x and y training + dev arrays\n",
    "\n",
    "def get_train_index(seed, train_size, m):\n",
    "    np.random.seed(seed)\n",
    "    train_index = np.random.choice(m, size = int(train_size*m), replace = False)\n",
    "    dev_index = list(set(range(m)) - set(train_index))\n",
    "    return train_index, dev_index\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = 0.9\n",
    "seed = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_index, dev_index = get_train_index(seed, train_size, m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save train_index for use while prediction\n",
    "np.save('train_index_final.npy', train_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_train = train_index.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X[train_index]\n",
    "Y_train = Y[train_index]\n",
    "\n",
    "#X_train = np.empty([m_train, 224, 224, 3])\n",
    "#Y_train = np.empty([m_train, 224, 224, 3])\n",
    "\n",
    "#for i, j in enumerate(train_index):\n",
    "#    X_train[i] = X[j]\n",
    "#    Y_train[i] = Y[j]\n",
    "X_dev = X[dev_index]\n",
    "Y_dev = Y[dev_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_placeholders(n_H0,n_W0,n_C0,n_y):\n",
    "    # ImageNet input image shape is (244, 244, 3)\n",
    "    inputs = tf.placeholder(tf.float32, [None, n_H0,n_W0,n_C0], name = 'inputs_ph')\n",
    "\n",
    "    # Output is dependent on your situation (10 for CIFAR-10)\n",
    "    outputs = tf.placeholder(tf.float32, [None, n_y], name = 'outputs_ph')\n",
    "    return inputs, outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_prop(model, inputs):\n",
    "\n",
    "    # model to give the name\n",
    "    if model == 'resnet50':\n",
    "        logits = nets.ResNet50(inputs, is_training=True, classes=7)\n",
    "        middles = logits.get_outputs()[-4]\n",
    "    elif model == 'squeezenet':\n",
    "        logits = nets.SqueezeNet(inputs, is_training=True, classes=7)\n",
    "        middles = logits.get_outputs()[-3]\n",
    "\n",
    "    Z1 = tf.contrib.layers.flatten(middles)\n",
    "    # FULLY-CONNECTED \n",
    "    Z2 = tf.contrib.layers.fully_connected(Z1, 120, normalizer_fn=tf.layers.batch_normalization)\n",
    "    # FULLY-CONNECTED \n",
    "    Y_hat = tf.identity(tf.contrib.layers.fully_connected(Z2, 7, activation_fn = tf.nn.softmax, normalizer_fn=tf.layers.batch_normalization), name = 'Y_hat')\n",
    "    return Y_hat, logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_loss(outputs, Y_hat, class_weights):\n",
    "\n",
    "    weights = tf.reduce_sum(class_weights * outputs, axis=1)\n",
    "\n",
    "    # loss function applied to the last layer\n",
    "    # train on the loss (Adam Optimizer is used)\n",
    "\n",
    "    loss = tf.losses.softmax_cross_entropy(outputs, Y_hat)\n",
    "    weighted_loss = tf.reduce_mean(loss*weights)\n",
    "    return weighted_loss\n",
    "\n",
    "\n",
    "\n",
    "def optimizer(learning_rate, weighted_loss):\n",
    "    train = tf.train.AdamOptimizer(learning_rate).minimize(weighted_loss)\n",
    "    return train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_random_batch(batch_size, full_batch):\n",
    "    try:\n",
    "        batch_index = np.random.choice(full_batch, size=batch_size, replace = False)\n",
    "        full_batch = np.array(list(set(full_batch) - set(batch_index)))\n",
    "    except ValueError:\n",
    "        batch_index = full_batch\n",
    "    return batch_index, full_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts = time.strftime('%Y%m%d%H%M')\n",
    "\n",
    "def train_model(X_train, Y_train, model, class_weights, learning_rate, batch_size, num_epochs, print_loss = True):\n",
    "    tf.reset_default_graph()\n",
    "    n_y = Y_train.shape[1]\n",
    "    m_train, n_H0, n_W0, n_C0 = X_train.shape\n",
    "    num_batches = m_train//batch_size + 1\n",
    "    \n",
    "    losses = []\n",
    "    inputs, outputs = create_placeholders(n_H0,n_W0,n_C0,n_y)\n",
    "    Y_hat, logits = forward_prop(model, inputs)\n",
    "    weighted_loss = compute_loss(outputs, Y_hat, class_weights)\n",
    "    train = optimizer(learning_rate, weighted_loss)\n",
    "    init = tf.global_variables_initializer()\n",
    "    \n",
    "\n",
    "    with tf.Session() as sess:    \n",
    "        # Initializing the variables\n",
    "        sess.run(init)\n",
    "\n",
    "        # Loading the parameters\n",
    "        sess.run(logits.pretrained())\n",
    "\n",
    "        for epoch in range(num_epochs):\n",
    "            batch_loss = 0\n",
    "            full_batch = range(m_train)\n",
    "            for batch in range(num_batches):\n",
    "                batch_index, full_batch = get_random_batch(batch_size, full_batch)\n",
    "                X_train_batch = X_train[batch_index]\n",
    "                Y_train_batch = Y_train[batch_index]\n",
    "                sess.run(train, feed_dict = {inputs: X_train_batch, outputs: Y_train_batch})\n",
    "                temp_loss = sess.run(weighted_loss, feed_dict ={inputs: X_train_batch, outputs: Y_train_batch})\n",
    "                batch_loss += temp_loss/num_batches\n",
    "            losses.append(batch_loss)\n",
    "            if print_loss:\n",
    "                print('Loss after epoch {:2d}: {:5f}'.format(epoch+1, batch_loss))\n",
    "        \n",
    "        predictions = tf.identity(tf.argmax(Y_hat, 1), name = 'predictions')\n",
    "        \n",
    "    \n",
    "        \n",
    "        inputs_dict = {'inputs_ph': inputs, 'outputs_ph': outputs}\n",
    "        outputs_dict = {'predictions': predictions, 'Y_hat': Y_hat}\n",
    "        tf.saved_model.simple_save(sess, model+'-'+ts, inputs_dict, outputs_dict)\n",
    "    \n",
    "    return losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss after epoch  1: 0.684796\n",
      "WARNING:tensorflow:From /scratch/user/pdwarkanath/.conda/envs/myenv3/lib/python3.6/site-packages/tensorflow/python/saved_model/simple_save.py:85: calling SavedModelBuilder.add_meta_graph_and_variables (from tensorflow.python.saved_model.builder_impl) with legacy_init_op is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Pass your op to the equivalent parameter main_op instead.\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: resnet50-201811291658/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "model = 'resnet50'\n",
    "learning_rate=2e-5\n",
    "batch_size = 32\n",
    "num_epochs = 1\n",
    "\n",
    "\n",
    "losses = train_model(X_train, Y_train, model, class_weights, learning_rate, batch_size, num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ts = time.strftime('%Y%m%d%H%M')\n",
    "def plot_losses(model, losses, learning_rate):\n",
    "    losses = np.squeeze(losses)\n",
    "    num_epochs = len(losses)\n",
    "    plot_title = \"Learning rate =\" + str(learning_rate)\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(8,6))\n",
    "    ax.plot(losses)\n",
    "    ax.set(title = plot_title, xlabel = 'Epochs', ylabel ='Loss', xlim = (-1,num_epochs+1))\n",
    "    \n",
    "    fig.savefig(model+'-'+ts+'TrainingLoss.png')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_losses(model, losses, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss after epoch  1: 0.611496\n",
      "Loss after epoch  2: 0.592710\n",
      "Loss after epoch  3: 0.592802\n",
      "Loss after epoch  4: 0.592825\n",
      "Loss after epoch  5: 0.592232\n",
      "Loss after epoch  6: 0.592619\n",
      "Loss after epoch  7: 0.592703\n",
      "Loss after epoch  8: 0.592967\n",
      "Loss after epoch  9: 0.592621\n",
      "Loss after epoch 10: 0.592715\n",
      "Loss after epoch 11: 0.592791\n",
      "Loss after epoch 12: 0.592871\n"
     ]
    }
   ],
   "source": [
    "model = 'squeezenet'\n",
    "learning_rate=2e-5\n",
    "batch_size = 32\n",
    "num_epochs = 12\n",
    "\n",
    "Y_train_preds_sq, Y_dev_preds_sq, losses_sq = train_model(X_train, Y_train, X_dev, Y_dev, model, class_weights, learning_rate, batch_size, num_epochs=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAggAAAGDCAYAAABOY+jlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XucXHWd5//Xu6/VuXTl1klXCJKoCYqzeIso4ziCDhjUAR1vMOqAo7K6yzKus66w81hxcdzV38xvdBxwZhBBZhVhBh0IDorICDoKSlBwTDAkBjQxtyb3W98/+8c51alUdyedpKtOddX7+XjUo+t865xT39OQPu/6nu/5lCICMzMzs1JNWXfAzMzMao8DgpmZmY3igGBmZmajOCCYmZnZKA4IZmZmNooDgpmZmY3igGDW4CR9U9KlWffDzGqLA4JZRiQ9Len3su5HRFwQEbdk3Q8ASQ9Iel8V3ucvJa2TtE/SLyT90Unu7w8l/UrSAUl3SppT8toDknol7U8fa0/+CMwqzwHBrI5Jasm6D0W11BfgAPD7QB64FPhrSb99IjuS9ALg74F3AwuAg8Dny1a7IiJmpI/TT7zbZtXjgGBWgyS9UdJjknZL+qGkM0teu0rSL9NPv2skvbnktcsk/UDSZyTtBD6etv1b+ql5l6SnJF1Qss3Ip/YJrLtE0vfS9/6OpOslfXmcYzhH0iZJH5W0FbhZ0mxJ35DUk+7/G5IWpet/EngVcF36Sfu6tP15ku6TtFPSWklvP9nfb0RcExG/iIjhiPgR8H3g7JK+vyL9ve+W9Likc46yu3cCd0fE9yJiP/A/gT+QNPNk+2mWJQcEsxoj6SXATcB/BOaSfDpdKak9XeWXJCfSPPC/gC9LKpTs4uXABmA+8MmStrXAPOD/A74oSeN04Wjr3gr8OO3Xx0k+NR9NNzAHOA24nORvzs3p8rOAQ8B1ABHxZyQn6uKn7SskTQfuS993PnAJ8Pn0U/sokj6fntTHevxsnG06gJcBq9PlU4B/Af487ft/A74mqWucY3wB8HhxISJ+CfQDy0rW+T+SnknD2zlH+X2Z1QwHBLPa837g7yPiRxExlM4P6ANeARAR/xQRm9NPv7cD64CzSrbfHBF/ExGDEXEobftVRHwhIoaAW4ACyXD4WMZcV9KzSE6kH4uI/oj4N2DlMY5lGLgmIvoi4lBE7IiIr0XEwYjYRxJgXn2U7d8IPB0RN6fH8xPga8Bbx1o5Iv5TRMwa53HmWNsAf0dygr83XX4XcE9E3JP+ju8DVgGvH2f7GcCesrY9QHEE4aPAs4FTgBuAuyU95yjHbFYTHBDMas9pwJ+WfvoFTgUWAkj6o5LLD7uB3yL5tF+0cYx9bi0+iYiD6dMZ47z/eOsuBHaWtI33XqV6IqK3uCBpmqS/Tyf07QW+B8yS1DzO9qcBLy/7XbyTZGTipEn6C5Lf39vj8DfXnQa8rew9fwcoSHpVyWTD1en6+4HOsl13AvsA0qC3Lw1JtwA/YPywYVYzamnSkJklNgKfjIhPlr8g6TTgC8BrgYciYkjSY0Dp5YJKfUXrFmCOpGklIeHUY2xT3pc/BU4HXh4RWyW9CPgph/tfvv5G4MGIOG8iHZT0dyQjAGP5VUS8oGTd/wVcALw6IvaWvef/jYj3j7Of8mC1GnhhyX6fDbQDT46zfXDkfy+zmuQRBLNstUrKlTxaSALAByS9XInpkt6QTnqbTnKC6QGQ9B6ST8AVFxG/Ihlq/7ikNklnk9wJcDxmksw72K3kVsBryl7fRjIcX/QNYJmkd0tqTR8vk/T8cfr4gZK7BcofpeHgauAPgfMiYkfZbr4M/L6k10lqTv+7nFOcTDmGr6TrvyqdM3Et8PWI2CdpVrqfnKQWSe8EfpfDlzPMapYDglm27iE5YRYfH4+IVSTzEK4DdgHrgcsAImIN8P8DD5GcTP8DyZB1tbyTZLb/DpJJfLeTzI+YqM8CHcAzwMPAt8pe/2vgrekdDp9L5ymcD1wMbCa5/PFpkk/oJ+N/k0ySXFdyyeB/AETERuAi4H+QBLGNwEcY5+9lRKwGPkASFLaThKD/lL7cSvJ76kmP+b8Ab4oI10KwmqfDl93MzI6PpNuBX0RE+UiAmU1xHkEwswlLh/efI6lJ0gqST9p3Zt0vM5t8nqRoZsejG/g6SR2ETcAHI+Kn2XbJzCrBlxjMzMxsFF9iMDMzs1EcEMzMzGyUhp6DMG/evFi8eHHW3TAzM6uaRx999JmIGO+7RUY0dEBYvHgxq1atyrobZmZmVSPpVxNZz5cYzMzMbBQHBDMzMxvFAcHMzMxGcUAwMzOzURwQzMzMbBQHBDMzMxvFAcHMzMxGcUAwMzOzURwQzMzMbBQHBDMzMxvFAcHMzMxGcUDIyBNb9vLQL3dk3Q0zM7MxOSBk5G/+dR1/due/Z90NMzOzMTkgZKS7s4Ote3qJiKy7YmZmNooDQkYWzspxsH+Ivb2DWXfFzMxsFAeEjHTncwBs2XMo456YmZmN5oCQkcJIQOjNuCdmZmajOSBkpDvfAcBWBwQzM6tBDggZmT+znSZ5BMHMzGqTA0JGWpub6JrZzpbdnoNgZma1xwEhQ935Drbu9QiCmZnVHgeEDBU6c77EYGZmNckBIUPd+ZwnKZqZWU1yQMjQwlk59vcNsrd3IOuumJmZHcEBIUO+1dHMzGqVA0KGXCzJzMxqlQNChro7k4Cw1eWWzcysxjggZGhBZw65WJKZmdUgB4QMtbU0MW9GO1t2OyCYmVltcUDIWCGfY4uLJZmZWY1xQMhYd2fOcxDMzKzmOCBkbOGsDs9BMDOzmlPRgCBphaS1ktZLumqcdd4uaY2k1ZJuLWn/lqTdkr5Rtv4SST+StE7S7ZLa0vb2dHl9+vriSh7bZOnO59jXO8j+vsGsu2JmZjaiYgFBUjNwPXABcAZwiaQzytZZClwNvDIiXgB8qOTlvwDePcauPw18JiKWAruA96bt7wV2RcRzgc+k69W8Yi0EX2YwM7NaUskRhLOA9RGxISL6gduAi8rWeT9wfUTsAoiI7cUXIuJ+YF/pypIEvAa4I226BXhT+vyidJn09dem69e0Yi0EX2YwM7NaUsmAcAqwsWR5U9pWahmwTNIPJD0sacUx9jkX2B0RxfH40n2OvF/6+p50/SNIulzSKkmrenp6juuAKqGQllt2QDAzs1pSyYAw1qf3KFtuAZYC5wCXADdKmnWC+5zI+xERN0TE8ohY3tXVdZS3qo4F+XbA38dgZma1pZIBYRNwasnyImDzGOvcFREDEfEUsJYkMIznGWCWpJYx9jnyfunreWDnSR1BFbS3NDNvRhtbPAfBzMxqSCUDwiPA0vSugzbgYmBl2Tp3AucCSJpHcslhw3g7jIgAvgu8NW26FLgrfb4yXSZ9/V/T9Wtedz7nSwxmZlZTKhYQ0nkAVwD3Ak8A/xgRqyVdK+nCdLV7gR2S1pCc+D8SETsAJH0f+CeSyYabJL0u3eajwIclrSeZY/DFtP2LwNy0/cPAmLdV1qLuzg5fYjAzs5rScuxVTlxE3APcU9b2sZLnQXIy//AY275qnH1uILlDory9F3jbSXY5Ewtn5Xjk6Zq/GmJmZg3ElRRrQHc+x55DAxzsd7EkMzOrDQ4INaBYLMnzEMzMrFY4INSA7s6kFoLnIZiZWa1wQKgBC2d5BMHMzGqLA0INWFAst7zbtRDMzKw2OCDUgFxrM3Omt7Flr0cQzMysNjgg1IjuzpznIJiZWc1wQKgRBVdTNDOzGuKAUCMKs3Js9fcxmJlZjXBAqBGFfAe7Dg5wqH8o666YmZk5INSK7vROhq2eqGhmZjXAAaFGHK6m6MsMZmaWPQeEGlGY5WqKZmZWOxwQakTxEoPvZDAzs1rggFAjOtqamTWt1ZcYzMysJjgg1BAXSzIzs1rhgFBDFs7q8CUGMzOrCQ4INaTb1RTNzKxGOCDUkEJnjp0H+ukdcLEkMzPLlgNCDelOayFsc7EkMzPLmANCDSnkk1oIvsxgZmZZc0CoIYVZabllBwQzM8uYA0INKRZL2uxaCGZmljEHhBoyvb2FzlyLRxDMzCxzDgg1ppB3LQQzM8ueA0KNKcxyNUUzM8ueA0KNKeRz/j4GMzPLnANCjenu7OCZ/f30DbpYkpmZZccBocYU0mJJ2/f2ZdwTMzNrZA4INaZYTdETFc3MLEsVDQiSVkhaK2m9pKvGWeftktZIWi3p1pL2SyWtSx+Xpm0zJT1W8nhG0mfT1y6T1FPy2vsqeWyVsnBWMSB4HoKZmWWnpVI7ltQMXA+cB2wCHpG0MiLWlKyzFLgaeGVE7JI0P22fA1wDLAcCeDTddhfwopLtHwW+XvK2t0fEFZU6pmrodrllMzOrAZUcQTgLWB8RGyKiH7gNuKhsnfcD16cnfiJie9r+OuC+iNiZvnYfsKJ0wzRczAe+X8FjqLoZ7S3MbHexJDMzy1YlA8IpwMaS5U1pW6llwDJJP5D0sKQVx7HtJSQjBlHS9hZJP5N0h6RTx+qUpMslrZK0qqen53iPqSq6faujmZllrJIBQWO0RdlyC7AUOIfkhH+jpFkT3PZi4Ksly3cDiyPiTOA7wC1jdSoiboiI5RGxvKur65gHkYXCrA6PIJiZWaYqGRA2AaWf4hcBm8dY566IGIiIp4C1JIHhqNtKeiHQEhGPFtsiYkdEFO8N/ALw0sk6kGordObY7IBgZmYZqmRAeARYKmmJpDaST/wry9a5EzgXQNI8kksOG4B7gfMlzZY0Gzg/bSu6hCNHD5BUKFm8EHhiEo+lqrrzOZ7Z30f/4HDWXTEzswZVsbsYImJQ0hUkJ/Zm4KaIWC3pWmBVRKzkcBBYAwwBH4mIHQCSPkESMgCujYidJbt/O/D6sre8UtKFwCCwE7isQodWcYV8jgjYvq+XRbOnZd0dMzNrQDpyjl9jWb58eaxatSrrbozy4JM9XHrTj7njA2ezfPGcrLtjZmZ1RNKjEbH8WOu5kmINKpZb9jwEMzPLigNCDSqWW97qWx3NzCwjDgg1aGZ7C9Pbml1N0czMMuOAUIMk0Z3PuRaCmZllxgGhRi2c1eE5CGZmlhkHhBrV3ZnzHAQzM8uMA0KNKuRzbN/Xx8CQiyWZmVn1OSDUqO58BxHQs6/v2CubmZlNMgeEGlWYldzq6DsZzMwsCw4INapYLMlf+2xmZllwQKhRhc4OAN/qaGZmmXBAqFGdHS10tLpYkpmZZcMBoUZJojDLxZLMzCwbDgg1rJDPsdlzEMzMLAMOCDWsu7PDIwhmZpYJB4QaViyWNOhiSWZmVmUOCDWsO59jaDh4Zn9/1l0xM7MG44BQwxamxZI8D8HMzKrNAaGGdbsWgpmZZcQBoYYdrqbogGBmZtXlgFDDZk1rpb2lyV/7bGZmVeeAUMMksXBWh0cQzMys6hwQalx3Z84BwczMqs4BocYV8i63bGZm1eeAUOO68zm27e1laDiy7oqZmTUQB4QaV5jVweBwsGN/X9ZdMTOzBuKAUOMKncViSb7MYGZm1eOAUOO601oIvtXRzMyqyQGhxrlYkpmZZaGiAUHSCklrJa2XdNU467xd0hpJqyXdWtJ+qaR16ePSkvYH0n0+lj7mp+3tkm5P3+tHkhZX8tiqZc70Ntpamnwng5mZVVVLpXYsqRm4HjgP2AQ8ImllRKwpWWcpcDXwyojYVXKynwNcAywHAng03XZXuuk7I2JV2Vu+F9gVEc+VdDHwaeAdlTq+apFEIe9aCGZmVl2VHEE4C1gfERsioh+4DbiobJ33A9cXT/wRsT1tfx1wX0TsTF+7D1hxjPe7CLglfX4H8FpJmoTjyFxSLMlzEMzMrHoqGRBOATaWLG9K20otA5ZJ+oGkhyWtmOC2N6eXF/5nSQgY2SYiBoE9wNzyTkm6XNIqSat6enpO9NiqyiMIZmZWbZUMCGN9ei+v9tMCLAXOAS4BbpQ06xjbvjMi/gPwqvTx7uN4PyLihohYHhHLu7q6jnkQtaA738G2vb0Mu1iSmZlVSSUDwibg1JLlRcDmMda5KyIGIuIpYC1JYBh324j4TfpzH3AryaWMI95PUguQB3ZO4vFkZuGsHANDwY4D/Vl3xczMGkQlA8IjwFJJSyS1ARcDK8vWuRM4F0DSPJJLDhuAe4HzJc2WNBs4H7hXUku6HpJagTcCP0/3tRIo3u3wVuBfI6IuPnJ3dxZvdfQ8BDMzq46K3cUQEYOSriA52TcDN0XEaknXAqsiYiWHg8AaYAj4SETsAJD0CZKQAXBtROyUNJ0kKLSm+/wO8IV0nS8C/1fSepKRg4srdWzVVsh3AEkthDMXZdwZMzNrCBULCAARcQ9wT1nbx0qeB/Dh9FG+7U3ATWVtB4CXjvNevcDbTr7XtedwNUVPVDQzs+pwJcUpYO70Ntqam3wng5mZVY0DwhTQ1CQW5Ns9B8HMzKrGAWGKKHR2eATBzMyqxgFhiujO5zwHwczMqsYBYYoopAGhTu7cNDOzGueAMEUU8jn6h4bZ6WJJZmZWBQ4IU0R3SS0EMzOzSnNAmCIK+WI1RQcEMzOrPAeEKaIwUizJtzqamVnlOSBMEfNmtNPSJI8gmJlZVTggTBFNTWJBZ84BwczMqsIBYQop5HOupmhmZlXhgDCFuFiSmZlViwPCFLJwVlJu2cWSzMys0hwQppDuzhx9g8PsOjiQdVfMzKzOOSBMIYdrIXgegpmZVZYDwhTSPVILwfMQzMysshwQppCCyy2bmVmVOCBMIV0z22lukkcQzMys4hwQppDmJrFgZjubPQfBzMwqzAFhinEtBDMzqwYHhCmmkO9wQDAzs4pzQJhiknLLLpZkZmaV5YAwxXTncxwaGGLPIRdLMjOzynFAmGJ8q6OZmVWDA8IU42JJZmZWDQ4IU8zCWcVyyw4IZmZWOQ4IU0zXjHaa5O9jMDOzynJAmGJampuYPzPnEQQzM6uoigYESSskrZW0XtJV46zzdklrJK2WdGtJ+6WS1qWPS9O2aZL+RdIv0vU/VbL+ZZJ6JD2WPt5XyWPLkoslmZlZpbVUaseSmoHrgfOATcAjklZGxJqSdZYCVwOvjIhdkuan7XOAa4DlQACPSloJ9AF/GRHfldQG3C/pgoj4ZrrL2yPiikodU60o5HM8uW1f1t0wM7M6VskRhLOA9RGxISL6gduAi8rWeT9wfUTsAoiI7Wn764D7ImJn+tp9wIqIOBgR303X7Qd+Aiyq4DHUpEK+w8WSzMysoioZEE4BNpYsb0rbSi0Dlkn6gaSHJa2Y6LaSZgG/D9xf0vwWST+TdIekUyfjIGpRIZ/jYP8Qe3sHs+6KmZnVqUoGBI3RVv6RtwVYCpwDXALcmJ74j7qtpBbgq8DnImJD2nw3sDgizgS+A9wyZqekyyWtkrSqp6fnOA6ndrgWgpmZVdqEAoKk50hqT5+fI+nK9ER+NJuA0k/xi4DNY6xzV0QMRMRTwFqSwHCsbW8A1kXEZ4sNEbEjIvrSxS8ALx2rUxFxQ0Qsj4jlXV1dxziE2lTIF2sh+FZHMzOrjImOIHwNGJL0XOCLwBLg1qNvwiPAUklL0gmFFwMry9a5EzgXQNI8kksOG4B7gfMlzZY0Gzg/bUPSnwN54EOlO5JUKFm8EHhigsc25RRmJeWWPYJgZmaVMtG7GIYjYlDSm4HPRsTfSPrp0TZI17+C5MTeDNwUEaslXQusioiVHA4Ca4Ah4CMRsQNA0idIQgbAtRGxU9Ii4M+AXwA/kQRwXUTcCFwp6UJgENgJXDbRX8JUM39mOxJsdkAwM7MKmWhAGJB0CXApycRAgNZjbRQR9wD3lLV9rOR5AB9OH+Xb3gTcVNa2ibHnJxARV5PcMln3Wpub6JrRzlZfYjAzswqZ6CWG9wBnA5+MiKckLQG+XLlu2bEU8q6maGZmlTOhEYS0uNGVAOmcgJkR8amjb2WVVMh38Mue/Vl3w8zM6tRE72J4QFJnWuHwceBmSX9V2a7Z0XR7BMHMzCpoopcY8hGxF/gD4OaIeCnwe5Xrlh1LIZ9jf98g+3oHsu6KmZnVoYkGhJb0NsK3A9+oYH9sglwsyczMKmmiAeFaklsSfxkRj0h6NrCuct2yYynkk1oIvsxgZmaVMNFJiv8E/FPJ8gbgLZXqlB1bwSMIZmZWQROdpLhI0j9L2i5pm6SvpUWLLCMLOpOAsNm1EMzMrAImeonhZpIyyQtJvlXx7rTNMtLW0sS8Ge0eQTAzs4qYaEDoioibI2IwfXwJmJrfdFRHXCzJzMwqZaIB4RlJ75LUnD7eBeyoZMfs2Ar5nEcQzMysIiYaEP6Y5BbHrcAW4K0k5ZctQ4V8znMQzMysIiYUECLi1xFxYUR0RcT8iHgTSdEky1B3voN9vYPs7xvMuitmZlZnJjqCMJZR38Bo1eVbHc3MrFJOJiCM+bXLVj0OCGZmViknExBi0nphJ6RYTdHzEMzMbLIdtZKipH2MHQQEdFSkRzZh8zvbAY8gmJnZ5DtqQIiImdXqiB2/XGszc6e3uRaCmZlNupO5xGA1oDufY6svMZiZ2SRzQJjiCvkOjyCYmdmkc0CY4lxu2czMKsEBYYrrzufYc2iAg/0ulmRmZpPHAWGKcy0EMzOrBAeEKa5YC8EBwczMJpMDwhRXHEHY7IBgZmaTyAFhiuseucTgWx3NzGzyOCBMcbnWZmZPa/WdDGZmNqkcEOpAId/hOQhmZjapHBDqgGshmJnZZHNAqAPd+RxbPAfBzMwmUUUDgqQVktZKWi/pqnHWebukNZJWS7q1pP1SSevSx6Ul7S+V9O/pPj8nSWn7HEn3pevfJ2l2JY+tlhTyOXYdHKB3YCjrrpiZWZ2oWECQ1AxcD1wAnAFcIumMsnWWAlcDr4yIFwAfStvnANcALwfOAq4pOeH/LXA5sDR9rEjbrwLuj4ilwP3pckPodi0EMzObZJUcQTgLWB8RGyKiH7gNuKhsnfcD10fELoCI2J62vw64LyJ2pq/dB6yQVAA6I+KhiAjgH4A3pdtcBNySPr+lpL3uLUxvdfQ8BDMzmyyVDAinABtLljelbaWWAcsk/UDSw5JWHGPbU9LnY+1zQURsAUh/zh+rU5Iul7RK0qqenp4TOKza0z0SEDwPwczMJkclA4LGaIuy5RaSywTnAJcAN0qadZRtJ7LPo4qIGyJieUQs7+rqOp5Na1a3RxDMzGySVTIgbAJOLVleBGweY527ImIgIp4C1pIEhvG23ZQ+H2uf29JLEKQ/t9MgprW1kO9o9RwEMzObNJUMCI8ASyUtkdQGXAysLFvnTuBcAEnzSC45bADuBc6XNDudnHg+cG966WCfpFekdy/8EXBXuq+VQPFuh0tL2huCayGYmdlkaqnUjiNiUNIVJCf7ZuCmiFgt6VpgVUSs5HAQWAMMAR+JiB0Akj5BEjIAro2InenzDwJfAjqAb6YPgE8B/yjpvcCvgbdV6thqUcG1EMzMbBJVLCAARMQ9wD1lbR8reR7Ah9NH+bY3ATeN0b4K+K0x2ncArz35Xk9N3fkOfrZpT9bdMDOzOuFKinWikM+x40C/iyWZmdmkcECoE4X0Tobte/sy7omZmdUDB4Q6UUirKXoegpmZTQYHhDrhWghmZjaZHBDqhAOCmZlNJgeEOjGjvYWZuRa2+hKDmZlNAgeEOrIw3+ERBDMzmxQOCHWk29UUzcxskjgg1BGXWzYzs8nigFBHuvM5ntnfR//gcNZdMTOzKc4BoY4sTGshbNvrUQQzMzs5Dgh1xLc6mpnZZHFAqCOFkYDgWx3NzOzkOCDUkeIIwlaPIJiZ2UlyQKgjM3OtzGxv8SUGMzM7aQ4IdaY7n/MIgpmZnTQHhDqTFEvyHAQzMzs5Dgh1xsWSzMxsMjgg1JnufAc9+/sYGHKxJDMzO3EOCHVmYT5HBGzf15d1V8zMbApzQKgzI8WSdnsegpmZnTgHhDpTSMstex6CmZmdDAeEOuNiSWZmNhkcEOpMZ66F6W3NHkEwM7OT4oBQZyS5FoKZmZ00B4Q6VMh3eATBzMxOigNCHXK5ZTMzO1kOCHVoYT7H9n29DLpYkpmZnSAHhDrUne9gOKBnv4slmZnZialoQJC0QtJaSeslXTXG65dJ6pH0WPp4X8lrn5b08/TxjpL275esv1nSnWn7OZL2lLz2sUoeWy0rpLc6bt7tywxmZnZiWiq1Y0nNwPXAecAm4BFJKyNiTdmqt0fEFWXbvgF4CfAioB14UNI3I2JvRLyqZL2vAXeVbPr9iHhjBQ5nSnEtBDMzO1mVHEE4C1gfERsioh+4DbhogtueATwYEYMRcQB4HFhRuoKkmcBrgDsnsc91oTiC4FsdzczsRFUyIJwCbCxZ3pS2lXuLpJ9JukPSqWnb48AFkqZJmgecC5xatt2bgfsjYm9J29mSHpf0TUkvmKTjmHLyHa10tDZ7BMHMzE5YJQOCxmiLsuW7gcURcSbwHeAWgIj4NnAP8EPgq8BDwGDZtpekrxX9BDgtIl4I/A3jjCxIulzSKkmrenp6ju+IpghJFPI510IwM7MTVsmAsIkjP/UvAjaXrhAROyKiONX+C8BLS177ZES8KCLOIwkb64qvSZpLcgnjX0rW3xsR+9Pn9wCt6ejDESLihohYHhHLu7q6TvYYa5arKZqZ2cmoZEB4BFgqaYmkNuBiYGXpCpIKJYsXAk+k7c1pCEDSmcCZwLdL1n0b8I2I6C3ZV7ckpc/PIjm2HZN+VFOEiyWZmdnJqNhdDBExKOkK4F6gGbgpIlZLuhZYFRErgSslXUhy+WAncFm6eSvw/fR8vxd4V0SUXmK4GPhU2Vu+FfigpEHgEHBxRJRf0mgYC/MdbNvXx9Bw0Nw01tUeMzOz8VUsIMDIUP89ZW0fK3l+NXD1GNv1ktzJMN5+zxmj7TrgupPobl3pzucYGg569vWN3PZoZmY2Ua6kWKd8q6OZmZ0MB4Q65WJJZmZ2MhwQ6tTCfAeAb3U0M7MT4oBQp2ZNa6W9pYmtex0QzMzs+Dkg1KlisaTNuz0HwczMjp8DQh1zLQQzMztRDgh1rJDv8BwEMzM7IQ4IdayQz7Ftby/Dww1bL8rMzE6QA0IdK+RzDA4Hz+zvO/bKZmZmJRwQ6li3b3W4tj5QAAAUZklEQVQ0M7MT5IBQxw5XU3RAMDOz4+OAUMcKI9UUfaujmZkdHweEOjZnehttzU1scbEkMzM7Tg4IdUwS3fkcW3Y7IJiZ2fFxQKhzLpZkZmYnwgGhzi3M59iy13MQzMzs+Dgg1LnufAfb9vS5WJKZmR0XB4Q6V8jn6B8aZseB/qy7YmZmU4gDQp3rHrnV0fMQzMxs4hwQ6tzhYkmeh2BmZhPngFDnCmm55a2uhWBmZsfBAaHOzZ3eRmuz2OxaCGZmdhwcEOpcU5NY0JlzuWUzMzsuDggNoJDP+QubzMzsuDggNIBCvsNzEMzM7Lg4IDSA4ghChIslmZnZxDggNIDufI7+wWF2uliSmZlNkANCAzhcC8GXGczMbGIcEBrASC0EBwQzM5sgB4QGMDKC4ImKZmY2QRUNCJJWSForab2kq8Z4/TJJPZIeSx/vK3nt05J+nj7eUdL+JUlPlWzzorRdkj6XvtfPJL2kksc2lcyd0U5Lk9iy27UQzMxsYloqtWNJzcD1wHnAJuARSSsjYk3ZqrdHxBVl274BeAnwIqAdeFDSNyNib7rKRyLijrL9XAAsTR8vB/42/dnwmkeKJXkEwczMJqaSIwhnAesjYkNE9AO3ARdNcNszgAcjYjAiDgCPAyuOsc1FwD9E4mFglqTCiXa+3nS7WJKZmR2HSgaEU4CNJcub0rZyb0kvCdwh6dS07XHgAknTJM0DzgVOLdnmk+k2n5HUfjzvJ+lySaskrerp6TnBQ5t6CvmciyWZmdmEVTIgaIy28ko9dwOLI+JM4DvALQAR8W3gHuCHwFeBh4DBdJurgecBLwPmAB89jvcjIm6IiOURsbyrq+u4DmgqK+RzbN59yMWSzMxsQioZEDZx5Kf+RcDm0hUiYkdE9KWLXwBeWvLaJyPiRRFxHsnJf13aviW9jNAH3ExyKWNC79fIuvMd9A0Os/vgQNZdMTOzKaCSAeERYKmkJZLagIuBlaUrlM0RuBB4Im1vljQ3fX4mcCbw7dJtJAl4E/DzdPuVwB+ldzO8AtgTEVsqdXBTjYslmZnZ8ajYXQwRMSjpCuBeoBm4KSJWS7oWWBURK4ErJV1IcvlgJ3BZunkr8P0kA7AXeFdEFC8xfEVSF8mowmPAB9L2e4DXA+uBg8B7KnVsU1ExIGzde4gzFnZm3BszM6t1FQsIABFxD8mJu7TtYyXPryaZU1C+XS/JnQxj7fM147QH8J9Ppr/1rFhN0SMIZmY2Ea6k2CC6ZrbT3CS27HZAMDOzY3NAaBDNTWL+zHaPIJiZ2YQ4IDSQpBaCyy2bmdmxOSA0kEK+wyMIZmY2IQ4IDaQ7n2PL7l4XSzIzs2NyQGgghXyOQwND7D00eOyVzcysoTkgNJDuYrEkz0MwM7NjcEBoIK6FYGZmE+WA0EBGyi27FoKZmR2DA0ID6ZrZTpNg6x5fYjAzs6NzQGggrc1NdLlYkpmZTYADQoMp5DvYutcBwczMjs4BocEU8jmPIJiZ2TE5IDSYpFjSIRdLMjOzo3JAaDCFfI4D/UPs63OxJDMzG58DQoMp1kLY6ssMZmZ2FA4IDWakFoIDgpmZHYUDQoMZKbe827UQzMxsfA4IDWb+zBySRxDMzOzoHBAaTFtLE/NmtHsOgpmZHZUDQgNamM+xxcWSzMzsKBwQGlCxFoKZmdl4HBAaUCHf4UsMZmZ2VA4IDag7n2Nf3yD7egey7oqZmdUoB4QGVKyFsM3zEKwBDQ4Ns9+VRM2OqSXrDlj1FaspbtnTy3Pnz8y4N2aTa3g42L6vj427DrJp10E27jw08nPjroNs2dPL0HAwf2Y7yxbM5LnzZ7BswUyWLZjB0gUzyXe0Zn0IZjXBAaEBjVRT3O0RBJt6IoIdB/rZuPMgm3YlJ/1iCNi06xC/2XWI/qHhI7bpmtnOqbM7eOlps1k0u4NpbS38smc/67fv5/ZHNnJoYGhk3WJwWLogCQ5L5zs41JKIoH9omN7+YQ4NDCWP/uRnX3E5besdeZ6s25u2v+I5c3jzixdlfSg1zwGhAc3vbAcmt1jS0HBwaGCIg32DHOwf4kD/IIf6hzjQP8Sh/mJb8vxAX/KPtn9wmOEIhiMYGk4++Q2ly8nztG04StY73F5cLm2P4vOS9ggYKq47HAwHBMHzujs55/Quzjl9PkvmTZ+034WdvD0HB44YAdiYnvyLoaD0hA4wZ3obi2Z3cEahk/PPWMCiOdM4dXYHi2ZPY9HsDnKtzeO+1/Bw8Jvdh1i3fR9PbtvPk9v2sW7bfm778ZHBYUFnGhzmF8PDDJ4738GhaHg46BscpndgiN7B4gl6mN7BIXr7i23lr499Ai+e5EuXS58Pn8CX0ba1NNHR2kyutYkF6d9AOzo18tf+Ll++PFatWpV1NzKx/M+/w8uXzOHy3302B/uHODSQnrjTk/vB/sPPD5Wd3A8OHHmiP9A3SN/g8LHftER7SxNtLU00N4kmJY/mJmiWkERzk9LXKFtHNBXblTxPfjLyerG9dNvy9xkO+MmvdrHhmQMAnDZ3GueePp9Xn97F2c+ee9QTip2ciOBA/1DyiX/nkSMAG3clP/f1HjlHYGZ7y8hJ/9Q5yUn/1NnTWDQnCQEz2if/s04xODy5LQkO67YnwWH99v1HBIfuzhxLF8xg6fzDlymWLphBZ642gsPQcHCw5N/xgb7BkRNz78DwyPO+Mdp6B4ZL2o98vXfgcAA41D903H8DipqbxLTWZnJtzSMn8ORnMx1pW0fJ6x1pe27kecn6Jdvkyp43N2mSf7NTl6RHI2L5MderZECQtAL4a6AZuDEiPlX2+mXAXwC/SZuui4gb09c+Dbwhbf9ERNyetn8FWA4MAD8G/mNEDEg6B7gLeCrd5usRce3R+tfIAeHNn/8BP/317qOu0ySY3tZCR1sz09tb6GhtZlpbM9PaW5jW2sy09nS5rSX9Wfp8jLb25HlHDf1j/fWOgzzw5Ha++4vtPLRhB70Dw7S3NPGKZ8/l3HR0YXEDjS5EBANDccSw7aE0QBY/5R3sHxz5NHdwIPl0mLQfPpEUA2bvWO0DQ5T/2elobebU9GRf/OR/eHka+Wm1cbKFJDhs2nV4xGHdtn2s254EiN6BwyfJYnAovUxxtOAw1om8GMAP9g+lj/R53yAHStqSsJ4G+P4j1y/t00S1NIlcemItPWHnWptK2pvpSJc7WptpH2PdYvsR27ckfxeKJ/zWZs+Vr7bMA4KkZuBJ4DxgE/AIcElErClZ5zJgeURcUbbtG4APARcA7cCDwGsiYq+k1wPfTFe9FfheRPxtGhD+W0S8caJ9bOSA8Mue/fz8N3uY3tYycuKe3pYk7uIJvb2lCak2TuTV0DswxI+e2skDa7fzwNoenkpHFxbPncY5p8/nnNO7eMUUHV3oGxxi3bb9rNmylzWb97J++372p58kR07g6Ql/6DjHbyWO+GRX+glvWtmnvuLP6e0tnDLr8GjA3OltU/7/tWJweLIYGLbt48nt+1i/ff8RJ+lCPsf8ztzIpbeD/Sc2Cpf8e21henvye53efjiUl/67Li4nv/dmOlqTtlzr4U/subIA4JN2fZtoQKjkHISzgPURsSHt0G3ARcCao26VOAN4MCIGgUFJjwMrgH+MiHuKK0n6MeCZJifgOV0zeE7XjKy7UVNyrc28elkXr17WxTW/D7/acYAH1vbw3bXb+eqPf82Xfvg07S1NnP2cuZyzrHZHF3Yd6OeJLXuTMFASCAbTE39HazPLFswgP62NBZ3t6Um7ZWS4dlpby8jJY+RE0nb4ZH/Eib8Bg+R4mprEs+ZO41lzp/F7ZywYaR8aDn6TBocnt+9j/bb99Ozvo7uz/Zgn8ullrxVH7XItzTTVyCic1a9KBoRTgI0ly5uAl4+x3lsk/S7JaMN/jYiNwOPANZL+CpgGnEtZsJDUCrwb+JOS5rPTMLGZZDRh9WQdjDWe0+ZO59Lfns6lv72Y3oEhHt6wgwfW9vDgkz18/O41cPcalsybzquXdWUyulD8xLpmyx7WbD4cBjaXTD5d0NnO8wudvOZ58zljYSdnFDo5be70mrnE0wiaxwkOZrWukgFhrL9A5WOXdwNfjYg+SR8AbiG5lPBtSS8Dfgj0AA8B5ZVNPk9yeeH76fJPgNMiYn96GeJOYOmoTkmXA5cDPOtZzzqxI7OGk2ttTi8zzAfg6WcOJJcinuwZGV3ItTZx9rPnjlyOOG3u5I0u9A4ULxEcDgNPbNk3UvCnScmo0MuWzOGMQidnLOzk+YVO5s3wbG0zOzGVnINwNvDxiHhdunw1QET8n3HWbwZ2RkR+jNduBb5cvLwg6RrgxcAfRMSYF+4kPU0yv+GZ8frYyHMQbPL0Dgzx0IYdPLi2hwfWbufpHQcBePa86bw6nej48iVzJjy6sPNAfxoC9vDEln3JJYKe/SNzA6a3NfP8NAQUw8CyBTOn5NwIM6u+WpiD8AiwVNISkrsULgb+sHQFSYWI2JIuXgg8kbY3A7MiYoekM4EzgW+nr70PeB3w2tJwIKkb2BYRIekskjLSOyp4fGZAMrpw7unzOff0+cALeKo4urC2h1t/9Gtu/sHh0YVznzefc5bN51lzpzE8HPx658GRSwPFn1tLSmAX8jmeX+jkvDMWjASCZ82Z5uvPZlZxFQsIETEo6QrgXpLbHG+KiNWSrgVWRcRK4EpJF5JcPtgJXJZu3gp8P534tBd4VzphEeDvgF8BD6WvF29nfCvwQUmDwCHg4mjkIg+WmSXzprNk3hLe88olHOof4uGndvDAL5LLEd+9azWwmlNmdbD7YD8H+pP76ZubxHO7ZnD2c+YecYlgzvS2bA/GzBqWCyX5EoNV0VPPHOC7v9jOI0/vZEFnjucXZnJGIc/SBTN8icDMqqIWLjGYWZkl86az5HeW8Me/syTrrpiZHZWrYZiZmdkoDghmZmY2igOCmZmZjeKAYGZmZqM4IJiZmdkoDghmZmY2igOCmZmZjeKAYGZmZqM4IJiZmdkoDghmZmY2igOCmZmZjeKAYGZmZqM4IJiZmdkoDf11z5J6gF9l2IV5wDMZvn/WfPyNe/yNfOzg4/fxZ3v8p0VE17FWauiAkDVJqybyndz1ysffuMffyMcOPn4f/9Q4fl9iMDMzs1EcEMzMzGwUB4Rs3ZB1BzLm429cjXzs4OP38U8BnoNgZmZmo3gEwczMzEZxQMiIpBWS1kpaL+mqrPtTLZJOlfRdSU9IWi3pT7LuUxYkNUv6qaRvZN2XapM0S9Idkn6R/n9wdtZ9qiZJ/zX9f//nkr4qKZd1nypJ0k2Stkv6eUnbHEn3SVqX/pydZR8rZZxj/4v0//2fSfpnSbOy7OPROCBkQFIzcD1wAXAGcImkM7LtVdUMAn8aEc8HXgH85wY69lJ/AjyRdScy8tfAtyLiecALaaDfg6RTgCuB5RHxW0AzcHG2vaq4LwErytquAu6PiKXA/elyPfoSo4/9PuC3IuJM4Eng6mp3aqIcELJxFrA+IjZERD9wG3BRxn2qiojYEhE/SZ/vIzk5nJJtr6pL0iLgDcCNWfel2iR1Ar8LfBEgIvojYne2vaq6FqBDUgswDdiccX8qKiK+B+wsa74IuCV9fgvwpqp2qkrGOvaI+HZEDKaLDwOLqt6xCXJAyMYpwMaS5U002EkSQNJi4MXAj7LtSdV9FvjvwHDWHcnAs4Ee4Ob0EsuNkqZn3alqiYjfAH8J/BrYAuyJiG9n26tMLIiILZB8aADmZ9yfrPwx8M2sOzEeB4RsaIy2hrqdRNIM4GvAhyJib9b9qRZJbwS2R8SjWfclIy3AS4C/jYgXAweo3+HlUdJr7RcBS4CFwHRJ78q2V5YFSX9Gcsn1K1n3ZTwOCNnYBJxasryIOh9mLCWplSQcfCUivp51f6rslcCFkp4mubT0GklfzrZLVbUJ2BQRxVGjO0gCQ6P4PeCpiOiJiAHg68BvZ9ynLGyTVABIf27PuD9VJelS4I3AO6OGaw04IGTjEWCppCWS2kgmKa3MuE9VIUkk15+fiIi/yro/1RYRV0fEoohYTPLf/V8jomE+QUbEVmCjpNPTptcCazLsUrX9GniFpGnpv4XX0kCTNEusBC5Nn18K3JVhX6pK0grgo8CFEXEw6/4cjQNCBtIJKlcA95L8cfjHiFidba+q5pXAu0k+OT+WPl6fdaesqv4L8BVJPwNeBPzvjPtTNenIyR3AT4B/J/kbPCWq6p0oSV8FHgJOl7RJ0nuBTwHnSVoHnJcu151xjv06YCZwX/r37+8y7eRRuJKimZmZjeIRBDMzMxvFAcHMzMxGcUAwMzOzURwQzMzMbBQHBDMzMxvFAcHMjpukoZLbVB+bzG8klbS49NvvzCwbLVl3wMympEMR8aKsO2FmleMRBDObNJKelvRpST9OH89N20+TdL+kn6U/n5W2L5D0z5IeTx/FssPNkr4gabWkb0vqSNe/UtKadD+3ZXSYZg3BAcHMTkRH2SWGd5S8tjciziKpGPfZtO064B8i4kySL6f5XNr+OeDBiHghyXcyFCuKLgWuj4gXALuBt6TtVwEvTvfzgUodnJm5kqKZnQBJ+yNixhjtTwOviYgN6ZdybY2IuZKeAQoRMZC2b4mIeZJ6gEUR0Veyj8XAfRGxNF3+KNAaEX8u6VvAfuBO4M6I2F/hQzVrWB5BMLPJFuM8H2+dsfSVPB/i8HypNwDXAy8FHpXkeVRmFeKAYGaT7R0lPx9Kn/+Q5NsrAd4J/Fv6/H7ggwCSmiV1jrdTSU3AqRHxXeC/A7OAUaMYZjY5nL7N7ER0SHqsZPlbEVG81bFd0o9IPoBckrZdCdwk6SNAD/CetP1PgBvSb7kbIgkLW8Z5z2bgy5LygIDPRMTuSTsiMzuC5yCY2aRJ5yAsj4hnsu6LmZ0cX2IwMzOzUTyCYGZmZqN4BMHMzMxGcUAwMzOzURwQzMzMbBQHBDMzMxvFAcHMzMxGcUAwMzOzUf4fZ45MGcRiwg8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_losses(model, losses_sq, learning_rate)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}