koncept512_train_test_py3_with_kuti.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "colab": {
      "name": "KonCept512 (kuti version)",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/subpic/koniq/blob/master/koncept512_train_test_py3_with_kuti.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XDdbdkexFk0t"
      },
      "source": [
        "### Install and load libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sVxpvhKMLG2W"
      },
      "source": [
        "%%capture\n",
        "\n",
        "# Setup paths\n",
        "drive_mount = '/content/drive/'\n",
        "drive_root = drive_mount + 'My Drive/research/data/'  # persistent storage for dataset images\n",
        "data_root = '/content/koniq/'                         # library install path\n",
        "\n",
        "!pip install kuti\n",
        "!git clone https://github.com/subpic/koniq.git\n",
        "!pip install munch"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kRMraeLxftuh",
        "outputId": "89599c9b-cb92-417d-8e6e-9a9e8ccb4ca1"
      },
      "source": [
        "from kuti import model_helper as mh\n",
        "from kuti import applications as apps\n",
        "from kuti import tensor_ops as ops\n",
        "from kuti import generic as gen\n",
        "from kuti import image_utils as iu\n",
        "\n",
        "import pandas as pd, numpy as np, os\n",
        "from matplotlib import pyplot as plt\n",
        "from munch import Munch\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount(drive_mount)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loaded Kuti\n",
            "Mounted at /content/drive/\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fQntwgm6FBn0"
      },
      "source": [
        "### Setup KonIQ-10k dataset images\n",
        "Download images resized to 512x384 pixels (originals are 1024x768)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KuZXHjjyXK2M"
      },
      "source": [
        "%%capture\n",
        "\n",
        "gen.make_dirs(drive_root+'koniq/')\n",
        "drive_image_path = drive_root+'koniq/koniq10k_512x384.zip'\n",
        "if not os.path.exists(drive_image_path):\n",
        "  !wget -O \"$drive_image_path\" \"http://datasets.vqa.mmsp-kn.de/archives/koniq10k_512x384.zip\"\n",
        "\n",
        "gen.make_dirs(data_root+'images/')\n",
        "data_images_path = data_root+'images/'\n",
        "!cp  \"$drive_image_path\" \"$data_images_path\"\n",
        "%cd $data_images_path\n",
        "!unzip -o koniq10k_512x384.zip\n",
        "%cd $data_root"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xwy8XM54K_J8"
      },
      "source": [
        "### Define the KonCept512 model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N2k21MsYK_KF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d892021c-0b5a-4305-c634-47a1537abf15"
      },
      "source": [
        "from tensorflow.keras.models import Model\n",
        "\n",
        "ids = pd.read_csv(data_root + 'metadata/koniq10k_distributions_sets.csv')\n",
        "\n",
        "# Build scoring model\n",
        "base_model, preprocess_fn = apps.get_model_imagenet(apps.InceptionResNetV2)\n",
        "head = apps.fc_layers(base_model.output, name='fc', \n",
        "                      fc_sizes      = [2048, 1024, 256, 1], \n",
        "                      dropout_rates = [0.25, 0.25, 0.5, 0], \n",
        "                      batch_norm    = 2)    \n",
        "\n",
        "model = Model(inputs = base_model.input, outputs = head)\n",
        "\n",
        "# Parameters of the generator\n",
        "pre = lambda im: preprocess_fn(\n",
        "         iu.ImageAugmenter(im, remap=False).fliplr().result)\n",
        "gen_params = dict(batch_size  = 16,\n",
        "                  data_path   = data_root+'images/512x384/',\n",
        "                  process_fn  = pre, \n",
        "                  input_shape = (384,512,3),\n",
        "                  inputs      = ['image_name'],\n",
        "                  outputs     = ['MOS'])\n",
        "\n",
        "# Wrapper for the model, helps with training and testing\n",
        "helper = mh.ModelHelper(model, 'KonCept512', ids, \n",
        "                     loss='MSE', metrics=[\"MAE\", ops.plcc_tf],\n",
        "                     monitor_metric = 'val_loss', \n",
        "                     monitor_mode   = 'min', \n",
        "                     multiproc   = True, workers = 5,\n",
        "                     logs_root   = drive_root + 'logs/koniq',\n",
        "                     models_root = drive_root + 'models/koniq',\n",
        "                     gen_params  = gen_params)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading model InceptionResNetV2\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/inception_resnet_v2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5\n",
            "219062272/219055592 [==============================] - 2s 0us/step\n",
            "219070464/219055592 [==============================] - 2s 0us/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tP_AKK539udA"
      },
      "source": [
        "### Train it"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zDburTNbH2qT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0dd80dbd-a6c5-4317-d9e1-7dbd417660f2"
      },
      "source": [
        "# do validation in memory\n",
        "valid_gen = helper.make_generator(ids[ids.set=='validation'], \n",
        "                                  batch_size = 16)\n",
        "\n",
        "helper.train(valid_gen=valid_gen, lr=1e-4, epochs=40)\n",
        "helper.load_model()\n",
        "helper.train(valid_gen=valid_gen, lr=1e-4/5, epochs=20)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training model: KonCept512/bsz:16 i:1[384,512,3] l:MSE o:1[1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.\n",
            "  warnings.warn('`Model.fit_generator` is deprecated and '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/40\n",
            "442/442 [==============================] - 350s 701ms/step - loss: 3534.6064 - MAE: 58.2835 - plcc_tf: 0.6617 - val_loss: 3366.3882 - val_MAE: 57.3105 - val_plcc_tf: 0.8100\n",
            "Epoch 2/40\n",
            "442/442 [==============================] - 303s 684ms/step - loss: 3352.7605 - MAE: 57.2313 - plcc_tf: 0.8561 - val_loss: 3582.5657 - val_MAE: 59.3504 - val_plcc_tf: 0.8776\n",
            "Epoch 3/40\n",
            "442/442 [==============================] - 303s 684ms/step - loss: 3176.0950 - MAE: 55.7074 - plcc_tf: 0.8693 - val_loss: 3027.1306 - val_MAE: 54.4122 - val_plcc_tf: 0.8137\n",
            "Epoch 4/40\n",
            "442/442 [==============================] - 303s 685ms/step - loss: 2937.9785 - MAE: 53.4448 - plcc_tf: 0.8509 - val_loss: 2938.4065 - val_MAE: 50.3516 - val_plcc_tf: 0.8170\n",
            "Epoch 5/40\n",
            "442/442 [==============================] - 303s 684ms/step - loss: 2598.4607 - MAE: 50.2217 - plcc_tf: 0.8653 - val_loss: 2351.5916 - val_MAE: 47.8475 - val_plcc_tf: 0.8517\n",
            "Epoch 6/40\n",
            "442/442 [==============================] - 304s 686ms/step - loss: 2197.0063 - MAE: 46.0448 - plcc_tf: 0.8659 - val_loss: 1936.2799 - val_MAE: 43.3444 - val_plcc_tf: 0.8595\n",
            "Epoch 7/40\n",
            "442/442 [==============================] - 304s 685ms/step - loss: 1764.1334 - MAE: 40.8659 - plcc_tf: 0.8243 - val_loss: 1116.6843 - val_MAE: 31.3545 - val_plcc_tf: 0.6116\n",
            "Epoch 8/40\n",
            "442/442 [==============================] - 308s 694ms/step - loss: 1316.6079 - MAE: 34.9612 - plcc_tf: 0.8214 - val_loss: 1226.0243 - val_MAE: 34.3067 - val_plcc_tf: 0.8874\n",
            "Epoch 9/40\n",
            "442/442 [==============================] - 304s 686ms/step - loss: 919.2328 - MAE: 28.7721 - plcc_tf: 0.8336 - val_loss: 808.0392 - val_MAE: 27.2608 - val_plcc_tf: 0.8788\n",
            "Epoch 10/40\n",
            "442/442 [==============================] - 304s 685ms/step - loss: 594.6525 - MAE: 22.4163 - plcc_tf: 0.8220 - val_loss: 454.2777 - val_MAE: 19.6440 - val_plcc_tf: 0.8507\n",
            "Epoch 11/40\n",
            "442/442 [==============================] - 303s 683ms/step - loss: 361.8977 - MAE: 16.7858 - plcc_tf: 0.8290 - val_loss: 386.0835 - val_MAE: 15.2551 - val_plcc_tf: 0.7308\n",
            "Epoch 12/40\n",
            "442/442 [==============================] - 303s 683ms/step - loss: 226.1030 - MAE: 12.5719 - plcc_tf: 0.8120 - val_loss: 502.3305 - val_MAE: 11.8166 - val_plcc_tf: 0.7513\n",
            "Epoch 13/40\n",
            "442/442 [==============================] - 307s 694ms/step - loss: 155.5928 - MAE: 10.0457 - plcc_tf: 0.8015 - val_loss: 82.0540 - val_MAE: 7.2272 - val_plcc_tf: 0.8664\n",
            "Epoch 14/40\n",
            "442/442 [==============================] - 303s 683ms/step - loss: 126.8206 - MAE: 8.9500 - plcc_tf: 0.7964 - val_loss: 57.9273 - val_MAE: 5.8952 - val_plcc_tf: 0.8685\n",
            "Epoch 15/40\n",
            "442/442 [==============================] - 303s 684ms/step - loss: 113.0472 - MAE: 8.3998 - plcc_tf: 0.8099 - val_loss: 47.2069 - val_MAE: 5.3267 - val_plcc_tf: 0.8887\n",
            "Epoch 16/40\n",
            "442/442 [==============================] - 303s 683ms/step - loss: 105.2997 - MAE: 8.0528 - plcc_tf: 0.8185 - val_loss: 93.3694 - val_MAE: 7.4281 - val_plcc_tf: 0.8657\n",
            "Epoch 17/40\n",
            "442/442 [==============================] - 303s 683ms/step - loss: 116.3870 - MAE: 8.4294 - plcc_tf: 0.7888 - val_loss: 54.2247 - val_MAE: 5.5036 - val_plcc_tf: 0.8686\n",
            "Epoch 18/40\n",
            "442/442 [==============================] - 307s 694ms/step - loss: 105.0796 - MAE: 8.0561 - plcc_tf: 0.8178 - val_loss: 227.9040 - val_MAE: 8.1863 - val_plcc_tf: 0.6805\n",
            "Epoch 19/40\n",
            "442/442 [==============================] - 303s 684ms/step - loss: 106.0295 - MAE: 8.0247 - plcc_tf: 0.8155 - val_loss: 47.1301 - val_MAE: 5.1625 - val_plcc_tf: 0.8808\n",
            "Epoch 20/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 103.6812 - MAE: 7.9163 - plcc_tf: 0.8219 - val_loss: 48.2667 - val_MAE: 5.5830 - val_plcc_tf: 0.8923\n",
            "Epoch 21/40\n",
            "442/442 [==============================] - 307s 694ms/step - loss: 98.3388 - MAE: 7.7519 - plcc_tf: 0.8300 - val_loss: 43.4383 - val_MAE: 4.9213 - val_plcc_tf: 0.9021\n",
            "Epoch 22/40\n",
            "442/442 [==============================] - 307s 694ms/step - loss: 97.7806 - MAE: 7.7299 - plcc_tf: 0.8283 - val_loss: 53.1224 - val_MAE: 5.4210 - val_plcc_tf: 0.8808\n",
            "Epoch 23/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 100.8587 - MAE: 7.8056 - plcc_tf: 0.8202 - val_loss: 57.3242 - val_MAE: 5.8441 - val_plcc_tf: 0.8762\n",
            "Epoch 24/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 106.0749 - MAE: 7.9166 - plcc_tf: 0.8076 - val_loss: 51.8377 - val_MAE: 5.2771 - val_plcc_tf: 0.8855\n",
            "Epoch 25/40\n",
            "442/442 [==============================] - 302s 681ms/step - loss: 101.9749 - MAE: 7.8166 - plcc_tf: 0.8220 - val_loss: 48.6679 - val_MAE: 5.6500 - val_plcc_tf: 0.8946\n",
            "Epoch 26/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 101.8955 - MAE: 7.8987 - plcc_tf: 0.8207 - val_loss: 77.6393 - val_MAE: 6.2299 - val_plcc_tf: 0.8107\n",
            "Epoch 27/40\n",
            "442/442 [==============================] - 307s 693ms/step - loss: 96.0555 - MAE: 7.7146 - plcc_tf: 0.8323 - val_loss: 78.0619 - val_MAE: 6.6491 - val_plcc_tf: 0.8731\n",
            "Epoch 28/40\n",
            "442/442 [==============================] - 302s 681ms/step - loss: 96.1016 - MAE: 7.7427 - plcc_tf: 0.8365 - val_loss: 45.2968 - val_MAE: 5.2464 - val_plcc_tf: 0.9016\n",
            "Epoch 29/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 96.8332 - MAE: 7.7539 - plcc_tf: 0.8355 - val_loss: 41.0464 - val_MAE: 5.0345 - val_plcc_tf: 0.8992\n",
            "Epoch 30/40\n",
            "442/442 [==============================] - 302s 683ms/step - loss: 92.9426 - MAE: 7.5781 - plcc_tf: 0.8407 - val_loss: 41.2895 - val_MAE: 4.9405 - val_plcc_tf: 0.9050\n",
            "Epoch 31/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 91.1237 - MAE: 7.4944 - plcc_tf: 0.8407 - val_loss: 45.1642 - val_MAE: 5.1888 - val_plcc_tf: 0.9079\n",
            "Epoch 32/40\n",
            "442/442 [==============================] - 302s 681ms/step - loss: 91.4696 - MAE: 7.5272 - plcc_tf: 0.8462 - val_loss: 58.4555 - val_MAE: 5.8157 - val_plcc_tf: 0.8763\n",
            "Epoch 33/40\n",
            "442/442 [==============================] - 307s 693ms/step - loss: 93.5635 - MAE: 7.5887 - plcc_tf: 0.8390 - val_loss: 2389.9321 - val_MAE: 21.1706 - val_plcc_tf: 0.4190\n",
            "Epoch 34/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 95.3163 - MAE: 7.6831 - plcc_tf: 0.8334 - val_loss: 49.5750 - val_MAE: 5.2837 - val_plcc_tf: 0.8837\n",
            "Epoch 35/40\n",
            "442/442 [==============================] - 307s 693ms/step - loss: 93.0132 - MAE: 7.5605 - plcc_tf: 0.8399 - val_loss: 279.9386 - val_MAE: 8.0550 - val_plcc_tf: 0.7087\n",
            "Epoch 36/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 84.4158 - MAE: 7.1777 - plcc_tf: 0.8561 - val_loss: 118.8637 - val_MAE: 7.4578 - val_plcc_tf: 0.8052\n",
            "Epoch 37/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 85.1706 - MAE: 7.2612 - plcc_tf: 0.8558 - val_loss: 50.4869 - val_MAE: 5.2732 - val_plcc_tf: 0.8855\n",
            "Epoch 38/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 84.2557 - MAE: 7.1099 - plcc_tf: 0.8526 - val_loss: 69.8268 - val_MAE: 6.4766 - val_plcc_tf: 0.8631\n",
            "Epoch 39/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 90.2274 - MAE: 7.4365 - plcc_tf: 0.8373 - val_loss: 68.2207 - val_MAE: 6.1550 - val_plcc_tf: 0.8627\n",
            "Epoch 40/40\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 82.5312 - MAE: 7.1010 - plcc_tf: 0.8594 - val_loss: 79.9693 - val_MAE: 6.1783 - val_plcc_tf: 0.8354\n",
            "Model weights loaded: KonCept512/bsz:16 i:1[384,512,3] l:MSE o:1[1]_best_weights.h5\n",
            "Training model: KonCept512/bsz:16 i:1[384,512,3] l:MSE o:1[1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.\n",
            "  warnings.warn('`Model.fit_generator` is deprecated and '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/20\n",
            "442/442 [==============================] - 338s 712ms/step - loss: 84.6875 - MAE: 7.2830 - plcc_tf: 0.8636 - val_loss: 38.9870 - val_MAE: 4.8753 - val_plcc_tf: 0.9154\n",
            "Epoch 2/20\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 87.3420 - MAE: 7.3639 - plcc_tf: 0.8595 - val_loss: 36.2769 - val_MAE: 4.5706 - val_plcc_tf: 0.9104\n",
            "Epoch 3/20\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 82.1399 - MAE: 7.1090 - plcc_tf: 0.8715 - val_loss: 68.3541 - val_MAE: 6.5103 - val_plcc_tf: 0.9046\n",
            "Epoch 4/20\n",
            "442/442 [==============================] - 307s 693ms/step - loss: 82.6657 - MAE: 7.1615 - plcc_tf: 0.8642 - val_loss: 39.1464 - val_MAE: 4.9120 - val_plcc_tf: 0.9060\n",
            "Epoch 5/20\n",
            "442/442 [==============================] - 302s 683ms/step - loss: 79.3572 - MAE: 6.9805 - plcc_tf: 0.8686 - val_loss: 39.0401 - val_MAE: 4.8862 - val_plcc_tf: 0.9132\n",
            "Epoch 6/20\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 81.6595 - MAE: 7.0842 - plcc_tf: 0.8728 - val_loss: 39.2878 - val_MAE: 4.8799 - val_plcc_tf: 0.9124\n",
            "Epoch 7/20\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 76.2362 - MAE: 6.8723 - plcc_tf: 0.8716 - val_loss: 37.4677 - val_MAE: 4.7564 - val_plcc_tf: 0.9140\n",
            "Epoch 8/20\n",
            "442/442 [==============================] - 302s 683ms/step - loss: 78.4571 - MAE: 6.9721 - plcc_tf: 0.8710 - val_loss: 37.1484 - val_MAE: 4.7623 - val_plcc_tf: 0.9153\n",
            "Epoch 9/20\n",
            "442/442 [==============================] - 302s 681ms/step - loss: 79.5485 - MAE: 7.0368 - plcc_tf: 0.8698 - val_loss: 38.9380 - val_MAE: 4.9058 - val_plcc_tf: 0.9090\n",
            "Epoch 10/20\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 78.0987 - MAE: 6.9669 - plcc_tf: 0.8731 - val_loss: 36.7527 - val_MAE: 4.7492 - val_plcc_tf: 0.9184\n",
            "Epoch 11/20\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 77.0068 - MAE: 6.8853 - plcc_tf: 0.8750 - val_loss: 37.9131 - val_MAE: 4.7014 - val_plcc_tf: 0.9114\n",
            "Epoch 12/20\n",
            "442/442 [==============================] - 307s 693ms/step - loss: 75.0249 - MAE: 6.8259 - plcc_tf: 0.8817 - val_loss: 35.8786 - val_MAE: 4.6638 - val_plcc_tf: 0.9164\n",
            "Epoch 13/20\n",
            "442/442 [==============================] - 307s 693ms/step - loss: 77.8993 - MAE: 7.0067 - plcc_tf: 0.8740 - val_loss: 35.5845 - val_MAE: 4.6072 - val_plcc_tf: 0.9100\n",
            "Epoch 14/20\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 77.1331 - MAE: 6.8899 - plcc_tf: 0.8785 - val_loss: 35.2464 - val_MAE: 4.5505 - val_plcc_tf: 0.9222\n",
            "Epoch 15/20\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 73.2470 - MAE: 6.7065 - plcc_tf: 0.8795 - val_loss: 33.9364 - val_MAE: 4.4451 - val_plcc_tf: 0.9207\n",
            "Epoch 16/20\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 72.8443 - MAE: 6.7066 - plcc_tf: 0.8853 - val_loss: 35.2603 - val_MAE: 4.5746 - val_plcc_tf: 0.9141\n",
            "Epoch 17/20\n",
            "442/442 [==============================] - 307s 694ms/step - loss: 73.1715 - MAE: 6.6971 - plcc_tf: 0.8801 - val_loss: 38.7895 - val_MAE: 4.7217 - val_plcc_tf: 0.9132\n",
            "Epoch 18/20\n",
            "442/442 [==============================] - 307s 694ms/step - loss: 72.8207 - MAE: 6.7283 - plcc_tf: 0.8820 - val_loss: 36.4102 - val_MAE: 4.6752 - val_plcc_tf: 0.9187\n",
            "Epoch 19/20\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 74.0131 - MAE: 6.7333 - plcc_tf: 0.8805 - val_loss: 40.6128 - val_MAE: 4.7294 - val_plcc_tf: 0.9093\n",
            "Epoch 20/20\n",
            "442/442 [==============================] - 302s 682ms/step - loss: 73.3862 - MAE: 6.7139 - plcc_tf: 0.8831 - val_loss: 35.0035 - val_MAE: 4.6079 - val_plcc_tf: 0.9222\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f4c502b3690>"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yckb5mXFB7H4"
      },
      "source": [
        "### Load trained model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sut6co1bDoGG"
      },
      "source": [
        "%%capture\n",
        "# download the pre-trained KonCept512 model\n",
        "model_root = data_root + 'models/'\n",
        "gen.make_dirs(model_root)\n",
        "!wget -O \"{model_root}koncep512-model.h5\" \"http://datasets.vqa.mmsp-kn.de/archives/koncep512-trained-model.h5\"\n",
        "helper.model.load_weights(model_root + 'koncep512-model.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tBi5Th28CBE7"
      },
      "source": [
        "### Test model on the default test set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7nFdL8VDB0g4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "outputId": "c8426757-d571-4083-cdc2-d25c2cfb04b6"
      },
      "source": [
        "y_pred = helper.predict()\n",
        "y_true = ids[ids.set=='test'].MOS.values\n",
        "apps.rating_metrics(y_true, y_pred);"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SRCC: 0.919 | PLCC: 0.932 | MAE: 9.132 | RMSE: 10.233\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOy9eVxUZ57v/35OgQsCbiCbKyibS1pxyeLSPZ24geme6Z6eTlwTNdHMvXNvEk1Pm0y2iXZ3NOmZ+7sdTXdMxCXp9PT0jRGMaCedaBI14sYOIipLgYKyukKd5/fH4RyLogoRQVGed171gqo6y3MO8fN8z/f5LkJKiUKhUCi6DtqdHoBCoVAobi9K+BUKhaKLoYRfoVAouhhK+BUKhaKLoYRfoVAouhhed3oArSEgIEAOHTr0Tg9DoVAo7ioOHz5cIaUMdP38rhD+oUOHkpqaeqeHoVAoFHcVQogz7j5Xrh6FQqHoYijhVygUii6GEn6FQqHoYijhVygUii6GEn6FQqHoYijhVygUii6GEn6FQqHoYijhVygU9xRSSjLt1aiS855Rwq9QKO4pskprWL71CFmlNXd6KJ0WJfwKheKeIjbEn/XzxhEb4t/sO/U0YKCEX6FQ3FMIIRgZ2hshRLPv1NOAgRJ+hULRZWjpaaArcVcUaVMoFIr2wHwa6Oooi1+hUCi6GEr4FQqFoouhhF+hUCi6GEr4FQqFoouhhF+hUCi6GEr4FQqFoouhhF+hUCi6GEr4FQqFoouhhF+hUCi6GEr4FQqFoouhhF+hUCi6GEr4FQqFoouhhF+hUCi6GEr4FQqFoouhhF+hUCi6GEr4FQqFoouhhF+hUCi6GEr4FQqFwgP3anN2JfwKhULhgXu1ObsSfoVCofDAvdqcvUOFXwjxrBAiUwiRIYT4SAjRQwgxTAhxUAiRL4T4WAjRrSPHoFAoFG3FbM4uhLjTQ2lXOkz4hRBhwL8A46WUowAb8HPgN8BvpZTDgUpgcUeNQaFQKBTN6WhXjxfQUwjhBfgApcDfAX9u/D4R+HEHj0GhUCgUTnSY8EspS4B1QCGG4FcDh4EqKWVD42bFQJi7/YUQTwkhUoUQqeXl5R01TIVCoehydKSrpy/wI2AYEAr0Ama2dn8p5e+llOOllOMDAwM7aJQKhULRnHs1jNOkI109DwOnpJTlUsp64C/AQ0CfRtcPwECgpAPHoFAoFDfNvRrGadKRwl8I3C+E8BHGkvgPgSzgb8BPG7dZCGzvwDEoFArFTeMaxnmvPQF0pI//IMYi7hEgvfFcvwd+ATwnhMgH+gMbO2oMCoVC0RZcwzjvtScAcTfMYOPHj5epqal3ehgKhaKLIqUkq7SG2BD/uyqmXwhxWEo53vVzL3cbKxQKheI65hPAvYIq2aBQKDoVndGf3hnHdCso4VcoFJ2KzuhP74xjuhWU8CsUik5FZyyM5jyme8H6V8KvUCg6FZ2xMJrzmO4F618Jv0KhUNwEnfGJ5GZRUT0KhUJxE9wLET7K4lcoFF2C2+Gbv1v8/0r4FQpFl6A1vvlbFW7zHJn26k49ASjhVygUXYKWfPOm4GfZb23h1jyHQHTqBWAl/AqFokvQUrSQaalL5C0t3JrniA3t3AvAanFXoVB0eZyfBtojjLSzLwAri1+hUNx1OPvi22NBtTPmDnQkSvgVCsVdR5a9hsWbUsmy17S4aNuaSaE9JxEV1aNQKBTthKugSq7/bGnRtjWRPM7b3GpWrrW/veaWJ4COnESU8CsUik6PqyCPDO3NxkXjLfeMJzfNjbJspZQg4Z25Y4kJ9rN+b2lR1pMgm08M6+eOQyJvOaqnI0tDKOFXKBSdHlPAY4L9yLRXA1iLpy1ZxTfy3WeV1rB82xGEEGSX1Vq/O29vCr2u61Z8vjtBzrLXsCTxsDW2W43q6cjSEEr4FQpFp8TZsjYFPLustono3qpV7CyunvrsZhRXMf+9g/zuixPMf+8gUncf8unsfmqPxeKOXHBWwq9QKDqctvirM+3VLN6Ualn40NwKdmcVt+SKcf3cOezStbWiOansy6+g6lI97+49RdXlevblV1jncz6es/ups6OEX6FQdDhtscwFoslPaGoFe+qD6xzxYyKlZEeanWVbDltjcHbh7Eiz8/TmVHak2S1fvZSSdx4fy5ThAbz9T2P4cMlEVkyP5MODhWTaq5sc727ryauEX6FQtImbseI9uVFa2jc21J+Ni8YTG+rex+0pgsbZ5WJta69hTXIOK2dEEh3ky6fHSvjkaBFPfnCIHWl23tyVww9jB/DmrhwruueZbUcpOH+Rf/7oGCOC/BkzuB/LpkXwwswoCs7VsXZXLitnRhEb4n/X1ehXwq9QKNrEzYidq7/add+W3DDmPlJKMkuqSS+uJLOkmuggX1bMiETXdcslJKVE6pJ59w9E6rLJZGD+npRm53//8Rgvb8+i3qFTdP4SfxcVyOdZ53hs4iCkLokJ9mP9vHEkjA4xonSkRNd1ktJLWZ2cw+qdOaycGcWcMaEIIdo0sd1JRGcdmDPjx4+Xqampd3oYCoXCiVtxb5jCKBDEhl63mNfPG+fRR276/OsdOl6aYOGDQ9j0zWkWPjiULQfOsHHhBAAef+8gNZfr6derG4unDGXZ1AiEEOw4bmfNzmx+OTOa15Oz+bf4aOxVV1i7Ow+A2aMH8G3+Bbp52Xj/iQlEB/ny7r4CpkQE8M8fHWPFjEjWpuSycnokEYF+xIZ6vm4z8qel67kdCCEOSynHu36uavUoFIo20ZZ6NK6ThSmOrQldjA3xZ+PC8ehS51TFJVbvzObSNQeb95/mxYRYYkP9ySypxqbBvEmDCPLvwboUQ9SXTY1AlzrXGnSG9O/B4ilDiR8dzO/3naKXt0AKjb0nztPNZuPF2dEgYcPek6xNOYGcAevnjkOXOhvmxlmCb05e7ia+mw3FvN1rBMrVo1AoOhxTJJ1j4GND/A1BdejsOF5CTLBfU7eOU+y888+CiovEjw7mxdnR9PTWWPjgUBJGhxj7CmPf7cdLmTK8P49PDGP9306y/UgRL3+SiUOX/OWonbd2n+CNz3J4a88J/mH8INb8eCQ9bBpPTh7KsIBeLE5MZcrwAF6YGcmyqeEg4JltRzlZUYeu63x6vISMkiqPrq6bDcW83WsEyuJXKBQdgrMVa7ly5o5rYgmfrKjj1e2ZVF9uACGYMybUED8Jy7cdYcX0EaxOzmH+A4N5/+vTNOiSi1cbQIIQ8FJ8DL/alce0yAGGJY5g8eRh/HbPCfadPM//O1rGxWsONh8spOaqgx5eGp+ll/LsDyOQUjJv0mBSMs/ys7hBvDQnljU7cwjt3Z1rDQ4KyuuYOiLQsu5XzohibUouxZWXeCslj+cfieT56SOMjN9b5Hb38VXCr1AoOgRnv71p3UskI0MM99CONDtrd+Xy6pxYEJAwOsTa5525Y1k/bxxSlzTokg++OY0QgqemDEU0WvXP/Vc6j08YyKVrDei6boRxJqbyi5kjeH56JA+F9+O9vQVMGd6Pr/Mv0E2Dqw06//PvIjhbe5XN+wvp5a3Ro5sXCBjWz4eLVx2Nk4rg35Nz8bZprIqPZl1KHu/MHcuGeXFEB/kCsPnbM4AgItAXIQQxwX5kl9W2yV1zu8s4K+FXKBS3hCf/dLMa9wKe2XqU9XPHcbKijjc/y+GFWdFWZIzzPtFBviRnlBE/KpjEJyeAhIKKOlYn5yAE/HJmFHMnDuS/j5RwuV7n1PmLhAf6Uu/QeW17FvUSyqqv0KObFy/MiOYfx19icN/ufHuykpA+3dl24AxTRvTjuR8Op6jqqnHer05Sd7WBkuorJD4xgfxztQghiB8VTESgL7Eh/kgpSUov5ekp4UwbMcDI0m3strViRiTrUvLu+IJua1DCr1Ao3OJO0M3PnK1bTxE5rlasKepSStam5PLCrGjDyrfXoEsdTWjEhPghdclrSVlsO1hE0YVLTBsRCAJmjwqmqPISIX7dWfVJJrrDwWUHjAnrxeC+PdB1nVcSYjh06jxbvitm84FC5owJJjrYl4LzFzlz/grvfX0KL00wMbwvSWlnGRZQxt9yyokY4MeyqeEICSF9ugPw613GwvDwAX6YOWRJaaU8+/Fx+JkkIsjPenox6wiZE0RnR4VzKhQKt7gLSTQ/c7ZuTfF3N0G4c3s4Tx470uy8viMLXZdomsYrCTG8tD2TmisNTBnej7SiKtBsdNNgUkR/ktPKmD06iOT0swzu24OiyitIoIeXhk1Ar+5eTB85gK0Hi9EAHZhzXzA7jpfR01tD13X++fvhJB4o5sq1Bnp08+Lf4qMRwnDZnCyv5bk/pbPupyOx11xl6vBAhCZ4ZttR3nl8LF/mnWPjvlO8+mgsb+3J7/TWvadwThXVo1Ao3OJuwTE2xJ935o4lvH8v1s+97sZxrZTZUpSKc8G1Nck5gCBhTIiRJCV1bAJ8u3uRWVqLFBo2DWaNDmFnehndvTSig3zpZhMUVl5h9sggHoroQ3cvjcsNOjNHBRM3qC8zYwLQgVmxgXyTf55ZIwO5XK9z1QG5Z+u45nCwbNowvGyCkqrLPPvxcRZ+cAiBoI+PN6XV13h7Tz578yusUs0FFXX89q/5LJ0Wzpz7wjp1T90boVw9CkUnpaNiu1t7XHcLjmbJ4uXbjAgd5+M4u3xigv1Y0VgewTXW3Tx/dJAvq+Kj0Rt0Xvo0C4EkPMCXVx6NNRZYgSH9e/JtQSVBft5sPgANDTof7C/k1/8wkr9mn6N3dxvJmVXMmxhGSB8f/rC3gM0HCpkc0Q8Am02j6lI9sSH+7Mu/wPfC/Pn6RAUXr+kM7u/L+4smoDt03tt3iicfGsrs0cGgCWbFDuBs3RU2fX2K978+zStzYokfHQJCkDA6BE3TOrWlfyOUxa9QdFI6Krbb03FbW2bA8tU3Nhsx4/PNMgexIf5kl9WyNiWXd/cVsGzrYTJKqvj0eAkNDQ2s/yqfpzenkpxeyq925nK0uJqL1xxcbdDZe6KC1z7N4pVPs1jzWS7fFFxg3e487FVXGDuoNw3ApasNlFRe5rOMc3x42A7A9rQypIQHI/oDkF5SxcIHBvODEQH09LahO3TqrjrIsNfw2o9H8R//9D3m3BdqTUhLpg7jj4eK2Jl5lnUpeXyWdY4vcspZNHkoUkpWJ2eRnFHGnDGhaNrdL5vK4lcoOikdFdvt6bitKZsA158EpDTq0iNptl9siD8rZ0RZkTunKi7x7J+Os2d0EDvTzzLv/kEM7efDtQYHYwf15nzdAJIzzrFhbwEOhwObzYtfzByBveoKPt1s/O7Lk1xtAG/N8OOX114DDFfO3/LKmRLej3V7Tly/xmA/dF3yWnIOl645OFRolHZ+aER/Hr0vzHpCkbpk4QeHsAl4aU4sCaNDiAj0tRZqY4L9mBY5gILyi6zdlUtEoK917XdTNU5XlPArFJ2Ujort9nRc0z3jKSHJVezM4zgcDn4+caAV326eY86YUMIDeiEQRAf7UlR5kQ/2nWJm7AA+OVxMYK/uOHTJr1PymBEbhI+XoMGh43DA1YYGjhZWseVgMT28QOrg39OLVxNiOFJYxafH7cy/fzA/GRvKtwWVfF1wAQE8GN6Xbwoq2X+6mv2nq/HWYPaoII4XVzNlRD++K7jAjjQ7wwP9WL7tCM9PH4GXTbBqthFWKqXkZHkdMcF+1j0aFdaHkaG9iRhwPWKntZNkZ+Xuf2ZRKBS3jBmfvjYll+yyWuszZ9ePJxdRcnopb6WcIDm9tMnnQggjkzYxlZyyOqaNGEC9FHydf57qqzrvfnUSXdeZMKQPmw8UogNXGyTeNpj/wGDGDu6DAKKD/bmqw9LJwzhSVMWWg0VUXXHw8aFC9p4op7uXxo/vCwXgp+MGsuD+wfT01ujhJajXYV9+BT+fOJCMkhqu6bBmZw4S42klflQwCx8cYpV8SEqz8+wfj5OUZm92LWYUkq7rtz3Ttr3pUOEXQvQRQvxZCJEjhMgWQjwghOgnhNgjhDjR+LNvR45BoejqtMZ3n1Vaw5u7cnhs4iDL4ncVek/drqSU9O7phdSlVTK5WW38RkvaJiRLJg/Ft5vGj8eG4e1tY39BJQmjg9EACfxk/CC+yClHQ2Py8H6kldSw4IHBDOrrw7aDRXTToLuX4KoD3t17moUPDuGncQP5j3+6D82msSvDCN38Hz8YzopHRuDTzYvBfXvRzWbjqSnhbFww3rqGnRllvL37BMkZZQCEB/jSx8ebYf17kVFS1eRaktJLee7j4ySll3ZoW8TbQUdb/P8J7JJSRgP3AdnAvwKfSylHAJ83vlco7ko6e911aCrgZk37jJKq6+LcKN4vzIjmo4NFlsXvLPQeu12V1rA2JY853wthzWc5LHz/EPM3HmTH8RKklMSG+LMqPhqANTuzudYgCevTHZumkZJZxpMPDaXeofPtyfN097axckYkr84Zycrpkaz6JIN9+Rfo6W3jH+MGEj6gF77dvfDpZuOfp4bjrYFD1/ndF/ks3HQIIWDdnjxeSojh1Tkj2XKgkO9HDWDjwgkMC+zFqtnRbN5/BoQx7sWbUhnaz4dnHxnB0L49ySypJjbUn81PTqTg/EUWJ6ayODHVmvgSRofw9j/dR8LokNv8F2x/Okz4hRC9ganARgAp5TUpZRXwIyCxcbNE4McdNQaFoqO5GzovOQt4VqlRz2ZJ4vUWhGa3qfCAXqycGWVZ/M5WbVZpDcu2HrZaE4IxYSDhhZlRfJ51joUPDuXlhBgAXt2RxTtfniCzpJrVSdkUlNcx//4hXLzmoLS6nh7eGgseGIKQgu42ePLBoSyeMoynpwwjs6SaP6UWcqlep7uXYPWPYq1xbF4Ux0PDAzh38QoOCfW65FKDxKEbY3nn8bFEBPoxLLDX9WqdSJYkHqak8jJCABJOnqul3mGUetiyv5AnEg+zODHVmPQErEvJY9XsaDYuvP50oGkaj94Xdk9E9XRY5q4Q4nvA74EsDGv/MPC/gBIpZZ/GbQRQab532f8p4CmAwYMHx505c6ZDxqlQ3AqdLbqjpTILVoPwkmoKKupIaAxNNL+XUvLMtqNus3HNnrVrd+WyYX4cI0N7W1m8RnKTEfXy8wlG56sN+05Rd9XB7FGB7C+opLuXjVWzonl1RxYvJ8RQXHWJ3+89Td3VBuJHB7Mvr5yaqw6ee3g4v/vyFJfrHVan3f/42X0Im+DNz3IIH9CLvXnnAfAS0CChhwb/8nAkHx0q4rGJg9i8v5A/zB9HwfmLRAQYC84LPziElyZ4MSGG8IBeLN6USoMuSXxigpEw0FiYzWzz2Jn+preCp8zdjhT+8cAB4CEp5UEhxH8CNcD/dBZ6IUSllLJFP78q2aBQtA7XMguWYKfksmFunCVyy7c1j0gxWwvGjwomOaPM6imbMDqE7LJaq3CamcDkOqGs/zKftSl59PTW0DSNuqsNaAKefTgCm7Dx0PB+nD5/mdeTsrjWoANwrcFBN5vGlBEBfJZ5jnmTBpGcXsbgfj05VlxDNxv8jx9E8MdDRcQN7ceR05X06q5hr7zK348NYduhEp5/OJLlP4ggKc3OK9szAVgydRibvy1ECFg1O4ZhAT5oQrOEPaOkilMVl0gYE9KiBd/ZJvab5U6UbCgGiqWUBxvf/xkYB5wVQoQ0DioEONeBY1C0A3eDH/te4VbvtesCbFZpjSHgM6KshCszosU1IiW7rJZ1KXkkp5exOimbH0QF8saOLJLS7CzfesSYDFJySUovtdYFTpbXIaVECMHkiP708NLw1uCNR2NZ8chwnnt4BIP7+vLWnjwWvJ+KlJIGh8QG/P3YMLxtGhfrdf6WW05Pb42/HCmm3qGz6IEhLLh/MA06bPz6DD+MHkByWhnfG9yHE+cug6YxwL8nfXy8mRoZQHZZLeEBvnT3trF0ajh//K6YVfHRrJodw9qUXCPTNux6WYlTFZdajGAyuRtceW2hw4RfSlkGFAkhoho/+iGG2+dTYGHjZwuB7R01BkX7cK/+z98ZudV77SnsMCLA1/p9ZGhvy5XjusC7fu44hgX40KBLktLsOKQR6WI2Hl85I4q1u3LJstew/st8/vdHx9hxrITMkmq+yivncoPOo98LY3iQH1sPFrPlQCESyaxRQXTzEkgkDglzvhfCFznneOPvR/H8w5F099L4ybgwbDYbAvjVrjziBvdh3qTB9PDWCPLvgZTwcNQAnp8+HIFk0zeniB8dzFd55TyVeIiTFXVsXDiBp6eGW43Q59wXyoZ5cdb1ZZXWsCTxMGt2ZrNyZlSzuHzX+363h216oqMTuP4nsE0I0Q0oAJ7AmGz+JIRYDJwBftbBY1DcIvfq//y3m9a4DdrjXpthhwCP3hcGwOLEVN5bGMeoMMPL6uoSyrIbgviH+eM4df4imxaNN3z7yCZNRszMVl3XeXdvAVJAcdVlfrUrj9rLVwGjwcrPJgxi1exodKnz+o5sKi/Vs2LGCEqqLlN3tYHtx0p54x9GM2dMKJn2at7/9jTJGWUIIVg6ZRiD+vrwenI2uq7z2qMj0XWJXw8vIgb4YvOy0dNbIyrYny0HitAEzB4dxJrkHDYuGk/O2TrWpuQCMGdM6PU+AI339b2FcVaTd9c+AK73/XY3SLlddOjytJTymJRyvJRyjJTyx1LKSinleSnlD6WUI6SUD0spL3TkGBS3zt0es9xZyCypZsHG78gsqfa4zc3ea3cuCjPsMH5UMJn2ahpD6RFcP6ar0Jnx9l+dKOd/f3ycvfnljAzrjaZpLNt2mA17T7Js62GyS2vRdZ19eRV00wQL7h/MU1OG8ctZUfwkbiCagKWTIzh5ro7XdmQREeDLq3Ni6derG9NGDGDZ1AgW3D+Y1+bEEN6/lxUZ9NLsKGaPDMLhcLDpm1McLqpElzpCCHQpeWVHFg0Oo9/ump3ZTAzvz9f55+lug1kjB3Dw5HkadKMpilkuYu2u3Ou9fZ0awowK68PIsKb3uKv9P96ixS+EeK6l76WUb7fvcBSKe5eCijqqLtVTUFHHqIHNAtnahGt7Q1Po5owJtaJw1s8bx8ZF45tYs66W7MjQ3mxcNB7dofOHfafZ9M1pBvb1ISLAlxWPGH1vX0qIQSJZ9EEqVZfqmXf/IHZlngVgV8ZZpJSsmBHJ5Ij+zHvvO+quOig4f5GEMSEUV18mJsR4akhOLyMprZRuXjZWzYri3z7NQkpJ7ZUGenW3ET8mlK0Hi3j+kRFMjQyk4FwdtVcb0CUUV16i3iE5cLKChDHBfHvyAqlnqnkpIRbR2MjFLBdhNkW5V632W+FGrh6zaEcUMAHDPw8wB/iuowalUNyLJIwJtcr6thdWL1spybLXWNE6AGtTjKgc05I1nw7cuZqcC699uGQS+eW1rNmZg0Dwy5lRXG3QGdK3J/nnanlpVhSlNVd5MLwvfzpUTOL+QhY8MIifxQ1mZJjhNururTEtKpD4UcEkpdl5c1cemcXVRAf7YdNg5shgfho3kK/yyqm5XM/sUUF8lXsOb03wk7gw4ob2ZVh/o87P7NHBpJ6p5C+Hi5gyIoAfRAdzsryWN1NyeWVOLOGBvQB4ZttRIgb4WtfrKvaeXG13e+ROW2hR+KWUrwEIIfYC46SUtY3vXwWSO3x0CsVdjKugmAlA7YlzL1uzQXlMsB9ZpTVsmBvXxI/t6emgWZ38YF9OnK1h/qRBTI0MZN+JCuquNvDeN6dJTi/Dr7sXPbt5cbb2ClcdOpOH9yMl8xzjh/RjZFhvYkL8mDU6mM37i6hvqKfuihG6mZxxluSMs0we3o+tB4sI6t2T9785TU9vjf0FF1jzD2MIbyyEdvrCZRZvTsWhwytzYtiddY7u3bzxsnkxMqw3saH+RAzws9YenEtCm9eSaa9u4ss3m7FvXDjeivBpEu46L67LPBm01scfBFxzen+t8TOFQuEGU1CWbTl8UxE6bQnndI7WMTtbPbPtKAg8Nj83J4HMkmo+PV6CruvWZ+/uK+C5/0rj7T35fH3yPB9+V8js0cHszy/n+YdH8HJCFOOH9GHHsWLmTRpEZkkNl685ePXTLDJKqkhKL2VX+ll6eGvsyjrP1wWVaEA3mzGW7NI65t8/mFD/7nT3srF8WjgNumRogA8jQ3uTlF7Km5/lsPDBIXjZBOEBvmxcOJ7EJycQE+JnrFtguKfM0gtZpTVNfPTm5wve/87a3qobxPV76xzu2pWCF1ob1bMZ+E4I8f8a3/+Y62UXFApFI6bVjLzuarkZQblRud/WRga9M3csSHA4HCSllxIR4MvIsN5N6uWvnzeOk+dqee5PaYARAfPO42M5ea6Ot346Gk3TiB8dDBLe/TIfHYEU8NqOHGquOujppTFucF9SGhOv/rDvFF/llrPlwBlWzYqmuOoSJZV1/OVoGd6awNvLxiPD+/FITDC/+iyXXRlneTEhxmiScqWBfScqsGm2Jo3Yvx8V1ORam/UBllDv0EE2vwcvxkezZmeutahtrmO4tpLcMD+uS7l54CYyd4UQ44ApjW/3SimPdtioXFCZu4q7BecyBkKImxaUGwl7Sw3Qna15Mzv35xMH8lZKHn18urF58cRmLh5d10lKsxMe4EtsqD8b9p7krZQ8fvvz7/HofWFk2qt54oPvqLncgLdN4NPdi+mxgQgEO46XsvnJiSDgz6nFJB4opFc3DU0IJg/vy2eZFfzHP45G2DSG9fPhv4+UsPlAISumRzItMhCEIcYZxVX8/A8H6aYJNi+ZiKZpLZaccB5/RkkVSxIPNwlVbXIv7TVIZJeK2HGmPTJ3fYAaKeV/AsVCiGHtNjqF4h7B1e1ys2Jzo7BCd81S3LlwJJJ3Hh9LiF93nn14BJsaLV3XRCVN04gYYDQl2bD3JB8dLOL56ZEM6++DrutIXTJzZDDXHDrTIgOYP2kgWw8UI5F4aYJ9+RWcPFfH1oOFxI8OoptNQwKfZVYggCNF1azdlct/Hy0hKb2MHt6Cd786CRgNTsy1jx7eNjSbhia0ZovR5sSWVVrTZNE2o6QKJIbP3s3TkbX+se2oSj50oVXCL4R4BfgF8MvGj7yBrR01KIXibqUt8eDuSiV72s65WYq79QBzYogN8afg/EVW/DmdD749g2bTrN86hXAAACAASURBVGOsnzuumbtj5cwoPjxYyMqZUUwZEcDSzUfYkWZnyebDfJZRyuQR/dmZcRZN2PDv6U1yWhmzx4Tw1p4THCmsxKe7jUeig9i8eCLbFk9g9qhAenXT+CyjlPABvUjcX0i9Q+encQO5eM1BQUWddd26rvNyQjSJT0ywaunAdbeXQDRLrjIzcJdsPtxsLcMZlXzontb6+P8eGAscAZBS2oUQ7vuzKRR3gM4QktfSGFr6ziyVDLBxUVPr1Xk/ayGycd3AFMYVMyJZl5JnhXGuS8kjItCXhNEhSCmNRCld8umxEtbszOH9RRMsizrLXmNZ9hvmxlnhmAARAb68tyCOr06Us/nrkzw0vD+ThviTVdqfAycr+Mm4MIL8e/Dul/k4dPj3ndkkPjmRM5VXSMmq4PGJA6m8WM/OjDLmjAlmyeShaEJjwtD+JIwJIctew4L3vwMk3jYbGxuzhU2sp5sQv2aF1Jpk4LYg6iqG3z2tFf5rUkophJAAQoheHTgmheKmuRM9UF19z65li280PnP/mGA/Ni4cj0S22ADddSHStGZjgv0IDzDEHeCduWOtbX70vYFk2qtZvCmVeocDEOhSN54UGhuNX613cLnewW9//j1iQvz4MvcsL8wYTv65WjQh+Oi7QiYOD2TH8TIOFJyn3gELHhjM6IF9sWk2Nn59Gr3x2EgY1s+HuZMGkXS8lAuX6pk6oj/78yuICfHnw+8KeWFmtFUOwqbBwgeGMjUy0GPROLPBuTNmBq6ibbRqcVcIsQIYATwC/Ap4EvhISvl/OnZ4BmpxV3Ej7oTF77yoCrBs62FWzjCKg7XG4m/NQnBrr8sQ90NcvuZg6+KJaDbNiudvEvEijNINZqPx1xqzZpdODWfZtAg27D3Jm7vy6GaDegf09fFm8ZShTI7oz5+PlLD9SDEOND5aMhGbl804h72GvLPVHCuq5qfjBrJ0yxHqHTovx8dwuKiSnelnqXfo+HSzseDBIXx0sMiawFqaLNvyN+0MT36diVuuxy+EeASYjlHRO0VKuad9h+gZJfxdm876j7mlaJOb2d9TfXzn7cxkJLPsgXODlKzSGqKDfHk9KYvN+wtZMSOSPzY2Jdn09WkcEl5OiCEi0BeE4Sax6uunlxLeGOophMDhcPDajkz+klqM0AQ/HhvK9mOleHtpJD4x0Zo4dKmzdPMR/rBgHKfOX+LfPsmg+nIDC+4fzAC/bnzwbSEvJ8SwZmeOMQnMiWF4oB9SSk6dv0j86BByztY1q/F/q7iLeurKeBL+Vrl6hBC/kVL+Atjj5jOFokO5E24ck9ZOOm3xJTuXSWhpAdJcyARYFR9t+PPnjjM8K41ds1bMiOSLnHJWzozk6SnhDOzbkzeSspg5Koid6Wd5bUcWQgi8bIJ/nRlJadVV8stqWLsnj5UzGiunN4ZX/tOEIaRknWPCkD78v6N26q7p9NYEe/PKmRIZQEyIH+/uLUDXdU5VXOLNlBx+NCaEPx0uITm9DF3q6DoM6+/Di/HR1sSSaa9m0QepeNkExVWX+fBgIY9PGsxH3xW5dee05W+iFnNbR2t9/I9gRPU4M8vNZwpFu3Mn/zG3NOm0ZUJyJ1o3mjRigv345ewoo6Z+qD8Rgb6G4G89yvq546xkLXMyEMJIWXLokJRWxisJMQg0pNQprrrMv/53Blcdkl7dbDzzg3DW7MyhwSFpcOhsXTKRUWF9WPjgENbuysO3u43nHx5O7tk61u3O4929BSydOoy395zg+ekjiB8dTHHVJTZ9fQqfbjZebizk9quduZy+cIl1u0+wft64xjEJvDTBwgeGWKL/4YFCXpgV3ayhu3l/3dXVMd1DZs19d1U2FS1zo+qcy4FngAghRJrTV37Atx05MIXC5E7+Y3aedFxFuy0TUlsmi+yyWt5qFFBN05o8JcQE+xkhno1lB1YnZzP//sFs/PoUiycPZevBIjTNxurkLGouN+BlE1x1SMYO6o29+grTRgxgWuQAvsot563deZyquMTogX15eko4ui4Z1KcnAG//NZ9ZIwfweU45of7d6OtjlFnOOVvHRweLWDU7Bnv1FauV4Yggf2KC/YhodO9IKYkN9ef9JyYQHeTLwH4+xI8KZlrkAGstwrnnL+D2PpmRTY9NGsTaXbk39aSguM6NnGofYlTi3N7403zFSSnndvDYFIrbiru4eOe4fNfkJ9eY/dbU2fE0WXja16xXb0bqOG+LhIziKv7tk0xWPBJpdc76w9enqb7cwKB+vdi4aAIJY0KYOSqYyw06fz82jJUzIimvvcqL8TFWAbOnpgxl/oODmT0qCCklyRllbD1QxK925QEC/57eCAFXHZKjRTVsWjSer06cI2pALzbMi8NefYW39pwgOb3MGpt5X5YkHiazpNqaNHPO1rEuJY+cs3VGbaHSWhZvSkXq0kpOc51wzXtjRjYtmxphLRArbp4WhV9KWS2lPA38J3BBSnlGSnkGaBBCTLodA1Qobhc3ant4Iwvf0/7OwuX89OIs9J72NStKCsT1AmT2Gha+/x0L3v+Oj1MLqb5cz+nztWhC4/2F41g6eShv/+MYwgN7WU8nPxkbxoL7B/PTcWEsnxbBuwvGM2dMKNlltSzfeoQ/fHOabQeK2JlRxo40O29+lsOq2VG8tyAOKSQ9vDSignzp1c3GT8aF8l9HilibcoJ39xWAgA8PFvL89BEMC/BhSeJhFicahdNk439fnTjHsq2HmzVGgetF0wrOX2RdSh7ZZbUeJ1zzc/PJ56YT5VTvaKD1JRvWA3VO7+saP1Mo7hluJOwtZeW6s8xN3Im69Zm9hkx7tceywro0ShpblSWlJL+8Fl033CeaMP4Jn7/YwPJtR/im4AK//TyfoyVVVqmCTHs1Cz84xH8fKWHp1iMkpZcSHeTLjjQ70UG+vDN3LFMiAnj7Z2MID/C9XiRtTKghxrvzWPjQEP50xM6afxhFYeVVPksvw7e7FwN7+xAT7Me7C8azfNpwRoX14Q8LxrFqdjQxwX6MDO3Ni/ExfHSwyKqA6Tr5xYb4s3GRMRG5u//OPQduRbRV7+jrtHZxV0inOy6l1IUQHd2vV9GFuRMhnLeylpBVer0JimtxsZhgvybCZfawfX76CPLP1bJud16zOPZMe7XVA3dVvCGiprW6bncer/5oJMMD/YgK6kVIn55MjujPP44fCBLkw5LEb0/zUsJIYkP8ySyppt4huXjNQdyQPvzms2yKKy/x9u4TAIQH9GLpliO8tzDOalsYPyqYT4+X8NqnWbz6aCwJY0IJ692T4guX+eOhIqPjFbA2JY/hjQJvomka63bnWU1RXLthNblnLn78FmvuNPbNbevfSEX8XKe14l0ghPgXrlv5z2A0T1coOoQ7GcLZFtwtAjvH57sKV1ZpDa/vyMbLJlj44JAmRdfgen/cU+cvsW53HkjJ6p05rJoV1aTBipSSgf18+OePjrFyeiSvJWXzxINDEEJjWICPZd326uHFD6ID2ZlRxvPTG0M++/gwtF9PCs7VIZEUnKujoLyOtbvzkFLy8vYsaq80IIRGdlktL2/PpOZKAytmRFpCHjHAr5mQOheSa6/Qy/YQbRXxc53WCv8y4P8AL2HkAX4OPNVRg1Io2ss6u11PDq7uCzMj1/kanH93Dm386GAR0yIHNBGl2FDD/REdZCRdDevnQ4ND8kZyNk9MHkZ0sC9ZpTXsy6tg0/7TvBQfi5Q6lRev8f43p3n1RyPRhMayLYdZMSOS9xdOIDrYl0dijYQtoyqnr7GoKiULHxrC6p3ZXG1w8Oj3whjarydeNsGKxpDN5PRSenSz8fTUcML69DSqeXroWOVcagHcR+e43rObub+KW6fVmbt3EpW5q2gr7Z3J2ZqJxNnF45xl22ybxgJpBecvMmdMaLPMVSklO46XsDo5h/cWxqEJjS/zzvL2nnyee2QEv/+qgNqrDfh42/hw6UQA9p2oYOqIQKuZuxnz/tikQTw9JZyktFJeT8om8YkJjAzrTUZJFacqLjF7VBC/33uK3315ksv1Dp6fHsmW/YVWDaHFm1JZFR9NRKAvy7cecRtD7+4eAW5/70wZ2PcybcrcFUK8IKV8Uwjx/9Gsxw1IKf+lHceoULQ7rXlyuJmnAssF1Zgs5Vo6wXnxsqVJx/RbL9l8mAZdEhHo26zoWFZpDWt25tKgG4u4saGGG2lQXx+G9vPhD1+dZPaoYA6eOs9/HylhZ3opuhR8PyrIGlN4/178IDqQdSl5ALy39xRVl+o5WVHHqIF90DSNtSm5RhLWt6fp7qXxzA/CWTY1nO9HGTH2SWmlAAwP9CM21LifpiuqNQlpzRrGuNy7tv4tFG3nRlE92Y0/U4HDbl4KRaemNfXxbybaw5xIJLLJPlmlNSzbepgdaXYr8sS5emZGSRWZJU1DCWND/HlxdrTxxs2Dd2yIv9VrNjrYl/Vf5bNs62EiBvhhs9mw2WwcOlPJ/RH9SdxfyMRhfXHoDvLP1eJwOFj/VT5LNqeyK6OMFTMMv/6mJ8bz/CMjkLpE13WrFv9HB4t4KSGGLYsn8cz3R2Cz2YgN8bfq/7+YcD3mXwijyFtWaU273DvXv8WyLU3vo6L9Ua4exV1Je1qG7VEF0rmUgLsIncWb3Nfbd9c60J2raEeanWc/Ps7z00ewbGqEtXhcUFHHm7tyGTukD9+dquTKNQc9u3kx//7BrNudx+zRA1g2dTijBhrdrjLt1SzY+B1Vl+r57c/v49H7wtxev+FmsvNGUhYvxccQ3hih4/p0A20vTufJ4t+RZmdtSq7HNQRF62lTdU4hxA7c2iIGUspH22d4LaOEX+FKZ6zC6EnQnKtrmlazia7rJKWXNqlOaV7bihmRRsP26ZEM69+LgvKLSHQ0ofHm7lxemGk0JE9KL+U3O7N5/P4hTInoz6nzlyg8f5G3/poPwAszI1k2NYLsslqrjHJBRR0JjT561wnMLJ+weNMhrtTrvPajWN7ek9+kvWNHumKUu6f9aGt1znWNP/8BCOZ6u8XHgLPtNzyF4ubojDHZniJPpJQUVFwkYXRIE3GNGtCL15OzSMk4R3hAL8viNxOWdKnz/MMjeOmTTHp4aSyaPNRqnP7k5KGsTsomPKAXc8aEArA2JZdB/Xz41We5SKmz8P7BBPh6k/jNadAhcf8ZXoyPZs59YYwa2KeJdb1+bmP+gYRl2w6zcnoUq2ZF83pyNhEBvpbLqqX6+R19HxXtR4vCL6X8CkAI8ZbLrLFDCKFMcEWXoq2WaFJ6Kc99fBwpJcMH+Fnx/T+IDmDz/iJ8utkoOFdniV2WvYb8ciMk8rGJg7h4tYFl0wwffZh/D4oqLxHm3wPRWBtfCGElScUE+xHevxd7T5Tz0XdFrJwRSb1D8u6+AqSE1ck5RAzws3IJ1u7KZcWMSArKLxoTwLxxrJwRxZu7cnhs4mC8bTarPEKmvdp4Amls/ai4e2ltHH8vIUS4lLIAQAgxDFDtFxV3jDuR4HWjc3qaGBJGhwAQ3r9Xk/j+qAG9DKv7WCmrd+YQEWQkcS1OTEXXdWaNCWbxg4M5W3uFp6YMQ9M0Smqu8PaefHr39GLp1HB0qTerASSRvLfvFK/MMTJuJbAmOYdfzopC0wTRQb5WmYgN8+OuW/kzIhEI4kcFU1x5iW37z7CqsZAbGE8iG+bFWdU0lSvm7qW1wv8s8KUQogCjA9cQ4OkOG5VCcQNuh6vHbRlml9ILzts6x9yb9edjQ/3RNI1H7wtD13VWTI9ESmktlL726Gh+Nn4wAkOQk9JL+cP8cezNL+et3Se4cPEayWllAPzjuEF8dLCI52dEMrB3T15PziZx/xneXzShSR/ffScqqL5cj8BIpjKfBgrKL/JmSo7RBKWxDr7pJtowLw4pJcu3GWsLH31XxAuzookI8LWusTVhqu7um6Lz0aoibVLKXRg9d/8X8C9AlJQypSMHplC0RGvCNG8Vd2WYEVjFz6SUVphmRkkVr+3Ipt5hdKVyrlBpkl1Wy5qdOSzd3Ly0MwKS0kp57uM0Tp2/yPJpw1kxI5KjZ6qIHxPM1gNFnKyoY8P8OJZPiwANbAJWzY5uUthtR5qdbQfO8PyMSIYFGk8Y2WW1AKxOzuaxicbk8fj9g1mbkkuWvcYS6ZGhvVk/bxwJo0PYMC+O4YF+VtimM22tUqroPLS22boP8BwwREq5VAgxAkP8kzp6gKCiehS334o0M2sl8rrv3SXEMqu0xgrTXDU7mjU7jVLG8Y2RNhFOvWzdHdM1DHTF9Egrq9ZcfDUXgX+/7xSTI/pjs9kA0zXTNHs2017dpOE7GOGip8ovIiX8alcO7y0Yj9CMInHZpbXWWkJ7NTu/lf0U7Y+nqB7rsbWlF/Ax8AKQ0fjeBzjWmn3b4xUXFycVXZuMkio55TdfyIySqjtyPnfn13VdphdXyoziKulwOGRGSZXUdV1mFFfJSav/KrcfLZK6rhuflTTdxjpPcZWctHqP3H602PredbuMEuN4417fLSet/qt1zrSiC03O7e74248Vy/B/TZZjX0uR248VNz13SZWc/OvP5e/+licdDkeT63I9juLuBEiVbjS1tfX4I6SUbwL1jZPFJUBN5Yrbxu0M35RSGt2gpo9oshDqriOUENf9+JYVj6TeobM6OYcsew070uws23qYpPRSywViHsOoty8ID+xlHcNsjmK6SmJD/HlvQRz/lhDNewviLNfQ0s1HWJyYah03u6y2yZNEpr2a2SODeG76CBKfmGA9BTh3szKzdk13EChXTZfA3Wzg+sLor9sTONL4PgL4rjX7tsdLWfyKW+FmLVjTwp60+q9y+7FiOfnXnzexlk3rf/uxYjnlN18YFnjj8c2ngLTCC3L70SKZVnTB2L/x94ziKms8U37zhcwormqyr2m5pxVdsJ4CnM9pnsvhcLh92nC+BucxOj+5OF+Pw+GQ248VK4v/HgUPFn9rhf8R4CugHNgGnAa+35p92+OlhF9xK3hyE3kSONNdk15cKR0Oh/zkaJGcuHqPIdwlzV0rGcXXj99McBuF3Xkbh8Mhtx8tkulFlW7FOqOkSm4/Viwjfpkstx8tbnKu9KJKy93TkkCbgt7Q0NBkG13XjcnsN583Ge/tcqEpbi9tFn6MyJ+fAf2BeCABCLjRfk7724CjQFLj+2HAQSAfY+2g242OoYT/7sRVmFoU2g60MD0dv7Wid91nX3zDCcSTP9/58//7RZ4c9osk+cmRQpleXCnTiyqtSca04hsaGuT2Y8UyreiCdU5d1+X2o8Vy0uo91oTiafyu3zmP0Zh4jGO7e1pQ3DvcqsXvdudW7vsc8KGT8P8J+Hnj7xuA5Tc6hhL+u5PWLJC29Lkn2jJRuNvHk1Xsuk9Li6c3GpOrK8VwI+2RY17ZJbcfKWqyaOtsgacXVTYbW0ZxlZz4xh75uy+MxdiW7oPrdxklVXLybwwXjzmRmefsiHur6BzcqvD/GlgBDAL6ma9W7DcQo1vX3wFJGAvCFYBX4/cPACk3Oo4S/ruTjrL4W5oo3FnfTXzqTvt48oObx9l+1BDmjOKWJyTLffLrz5uNyXLZHCuWUsombh7Twjctfl3XZUNDg/zd3/LkXw6fabKflFKmF1fKca/vlhPf2NPkupzvm7ObyvVzc4zmU4XrNje6t55Q7qLOy60K/ymMHrtNXq3Y789AHPD9RuEPAPKdvh9EY4iom32fwugDkDp48ODbcpMUdwctTRTOIuT8u0eR9BRmWWJY1+Ne3y3TiytbPLezNe064TQ0NMj/+0WuTCu84HECcsYIv0ySv/s8T35ytKj5oquTYLu7vlu15pXFf2/hSfhbm8DVE6PB+mSMMs37gA1Sysst7JMAzJZSPiOE+H7jE8Mi4ICUcnjjNoOAz6SUo1o6v0rgureQ8tbqubfl2ECz5Cl3+2baq0Fer6gZEehLbKh/k6Qts94OGEXSYkL8rJLHZlLU2l25PH7/YKZEBLBoUyreNsHGRROMjlbppcSPCibnbF2zcsjRQb68u6+Ajw4WWe0NXe+Rua1rMpk5LoFo8ToVXYe2lmU2SQRqMBquAzze+NnPWtjnIeBRIcRsoAfgD/wn0EcI4SWlbMBwBZW0cgyKe4Qsew2LE1PZuHA8CM/NuG8WZ9E3Bc+57d/ixFTqHTqbn5zIyLDm58oqrWFJ4mHqHToA3jaNjYvGk11Wy7Ith1k5M4qE0SGsnzcOJDyZeIgGh7SOZ55DSsnM0UG8tfsEZZOu4KUJVs2OMQTaXsPqpGyKL1zio++KrIxZ5wJwy6cNZ2BfH9buynXbsNxdsbiYYD9WzIgkNsS/We9ehaIZ7h4DXF9AVms+a2H/73N9cfe/aLq4+8yN9lc+/nuL9OLWhSTeCHcLmO589c4hlBPe2C0/acyodXc80+eeVnhBbj9SZEW+bD9WLB/6tRHZY4V5HimUY19LkWmFF6xM2rTCC/KTo0XyypUrcv7G/XLSG7ua5ABYfvrVe6xF35aigJxj7c2F3pbi9tvqZ1fumnsTbjFz94gQ4n7zjRBiEob/vS38AnhOCJGPESK6sY3HUdyljAztbbUgbGuxNSmNGjfLthxukuHqmt1rWsfZZbXMuS+MlxJiWZeS5zYrVQjBqLA+RhNym8aaz3JZuvkI2aW1hAf04oUZ0azZmc2SxMNkl9UyfIA/3bxsnDp/kcWJqSzdfIRTFy7x1u4TrN6Vx96880wfFWrV05GNbtXEJybw/kIjk9bM0jWzbsF4OjHvExj1/NfuyiU5o6xZhq6Jc39fMzP3ZlDZul0Md7OB6wuj6bqOkbh1uvH3bCAdSGvNMW7lpSx+hSuuC6omLUUSOcev3ygE0lxINTJoi6xomPSiSmvRtcnisFPCV0ZJlTx+5rwc88oumVZ4ocmYPT2RNHtqcUr8Mq+zNTH3N1rQ9oSy+O9NuEWLfyZG4tW0xtewxs8SgDntNw0pFK0jJtiPlTOirHaGJln2Gp784BA7jtuN6IXGOvpZpTXsOG7n2Y+PUVBxsUk9Gykluq6z/qt8nt58vZSyLnX25lbw2o5sfj5hkLF2oAnWpeSRlF5qWdVmvR5rHBKEJvhw6f3Ehvrz6fESdF13+0Ti+sRjbiORLN96BF3q1nU61wPyhPM5bsaK9/Tk5XyPFPcOra3Hf6alV0cPUqFwJbvMKCfsXFwMsAqkvbYjiyy7IXimAAL08elGRIBvM1dRUlopb6Wc4LGJg0Aa7pZFH6Sybk8e5y9eY+PXp0hKLyU6yJfHJg1i7a5cqzhaRnEVr+7IYNmWw2zYe5InNx1i6eYjICA5o4znPj5OUnpp03G6TDrm5GBOVALBO3PHognN7XV6wlnA26OwnXIB3Zu0NqpHobjtSDdROiaeumGNDO3NK4/GsmZnDhLDSjUjXuJHBTM8yI+YYD8+PV7CG0lZLHpoGDHBfugOHd/uNsL69GRxYirvLYhj06LxfJVXTlifHtg0jbW7cgGsUMuE0SFEBPryVe45Er8tZM6YYBK/Oc2qWTGgGeeNDvKluPIS8aOCm0TjSClZvCmVVfHRADz/pzQAHr0vzNhu2xFLtN+ZO9YKMb2ZtZD2aFreGZvaK24dFfel6LRk2qt5ctMhPj1e0sTVYE4IEml1w3JmeKAfGxeOt0TPfDrIOWs0NM8ureX1HVlca5Bs2V9IdlktQhP07OaFAOodRh/bUxcu8cdDRdi8bMSPCWHlzCjiRwWzYX4cc8aEWq6Xp6eG88LMSJZOHoYQGmiwbrfhDsopq+OP3xVbMfuWiEq4Uu9gdXI2EQG+vPWzMYT3N3rwOm8nhNFM3V0nLFc6wi1zOzqdKW4/SvgVnRaBoMEhWZOc00T0TMtZINxH8Ww7YgkmNLdaJRKbJlg6dRjvLYiz2g5uXDSe8AG+eNs0Tp2/yNqUXKtFYXJ6GWuSc8gpq2smhJqmMXVEIJqm8d5CY1JYOSOKNz/L4WR5Le/MHWuJuLmvEMZE81J8DCPDejN8gB/LPzzCjjR7M+FurdWt3DKK1tKqzN07jcrcvbdoyYXTbDs32bYt7X+j7zLt1Uhdsje/go8OFvLugvFN3CHmNo1eInSpc/r8ZYb278lTW47y3oI4hCaaZAbrus6iD65n544M7W2tIazdlWtl4DqPP9NebTVjd23BuHJmFOtS8lqd1GZes3Mmr7LQFeA5c1dZ/IrbTkuWqbO7QgjByLDejArr0y5CZmbmLtqUypb9hbwwK9ptRy2BYMnmVPaeKGfpliOs2ZmDTbOxcdH1TOOs0hojU3dTKgXlF5tk55pEBPiyYkYka3flNntieWbbURBY1yWEYM6YUDbMj7Oyg1vrV3fOVVBuGUVrUBZ/F8edhdxai7w9z2mSaa++YQkHT9s4W83OzcOdLeKs0hrLmkcYi8FmCQnTmpe6ZOEHh7AJeDEhxogCQqIJzarLY5ZfcN7P+XqsMc4dZ4zB6alF13WS0koZ2r8nNs3WNBS0ne+nomujLH6FW9xZ3x3tK25pwdDZny2lJLOkmoySqiZ+b3c+b1dXiafsXTMzV2jCWhg2o39OVtSxfOsRJJKX58Tw/qIJPHpfGEITVn9b56JoMSF+RgZyWO9m12NGEkUH+1rHNe9ndlkta3bm8MSmwyxOTL3l+6wWYBU3i7L4uzh3wuJv7Zh0h8689w/R3UvwwaKJIPDox860V/P0llQenziYqSMCrSJsnnzfzusHppAjDcv8VMUl1qZcf2pw9clbYZlzx4GgxSeXFTMieXNXDo9PGsyyqRFommadW5c6mtCUxa/oMJTFr3CLO2vROf67I7I2nROW3GEK6978cmqv1LPwwSFWJquZNOVqJceG+PPCzGgSvz3Dks1GUpazpW+Ku3O2LQKe2XbUCOcUgmc+PMqp85dYu7vpU4NZw2dkWNPEKHNM5lic1wqccwdemBnNR98VWUlY5trF6IF9rYlE1/U232sVzaO4adzVcehsYfNtPQAAFBdJREFUL1Wr587QloqPnjpgOePalarZ/o11b8y+s83q4rTQbtDsLuVcb8essTNx9Z4mlTlb0yu3Jdy1VfTYBMapgYq7zmDuuoC15Z4rFM5wKx247vRLCf+doS2C4kn8nHEVTOfzWeWPXYqvtXVslrAeLWrStrAt3KgMdEuT3o06g6mm54qOwJPwKx+/ol2RsnXdtZy3M7/LKKli8aZU5j8wmK0HiozsWzcNU6B10T/O5zG7Y0kprWiem/WHu55TSvfx+O6uubX3RaFoT5SPX3FbcM1O9SSwzn5pU0CRxv6D+voAWNE27miWjSubliuwjtm4bXZZLTEhfhScv9hqf7jrMU2/fUywn3WtAmFE5thrrMiiZVsPk2WvabJva++LQnE7UMKv8Iir8LUnzsXHMkuMRCiAjYvGM+e+UKtRS0uC7iyerguc1nu7UY75yQ8OkZRWytqU5uGennB3TNfyEebkJDEs+rW7clk5I6rZwq9C0ZlQwq/wSGuiRW5mcnDe1rn42MmKOmODRheMpmlWFE6W3YOgu4nqcX4CcI68WbMzmwZdMizAhw1z4wgPMIqhZZRUkVnieeyuxxSIJj+haTexmGA/q2rnyNDeqqqlovPizvHf2V5qcbf1tGeER2uOdTORP54iXVrqIZteXNnku5YWhp0jeppcg1NEjXMUzaTVf5WTVv+11Yu9N7oft9r3VqFob1BRPV2D2y0+NxJDT5EuN2pD6KmFYnpxpdvryyipkhPf2CPHvb67xSbuzlE07iaKW7l2FaGj6Gx4En7l6rnHaGvjDNlGf/6NFiqdXTOu3aHeeXwsJ8/VWolc5raZ9upmUS9WFy1JkwVW5+t+MT4ab5uGQNzQTeWalOV6H1qTUOU8Xud1B7OBuvLvKzorSvjvMdoaMZJVWsOyrYfd1oO/FVwjYZzHWXD+Is/+KY0Ne08ipbQWfAvKLxqRMU7CafnspeS1T6+3VXQ+3pz7wti4aDyxof7NOnQ5X6erKDtPeub3njKEnTHHJBBNtlVdqxSdHSX8dyFttc5bOpbZvNy1hPCtkl1Wy9pduexIs1sLqeY540cF8/z0EXx4sJAsu1FiQSB4c1cOj00c1GSyMCe0goo6Ki9e48u8c80ifZzvh3NJBncTiLsibuZTxjtzxxLev5fVQMUT5phiQ10WgVW4pqKTo4T/LuRGboybmRic69mY9eDbaqlK2byaZmyIPytnRrFmZ45VidI8Z87ZOpZPG86788db4Y8SyQuzjNo2WaU1TSJvpJQIBL7dbWzef8bKATCbprta6abIxwT7Na3x7yLK7lodPvPh0SZdvJzvq6sbSAm94m5DZe7ehZguCU+Zn63Nam3NsW4GszEJ0CwO30zOig31t8Q6IsDX8rE7j8M8VkH5RVYnZwNGXfxh/X1YkniYBQ8YFTgRWJU0zTBKT5U7W7ofrvfA0z2xKm5Oj2TNzpwWM4sVis6Ap8xdrzsxGMWt4Vw90x0342O+0bFawlUgY0P82bhwvFXqGK53m3Iuc5CUXsqanTkIhDVBuI5DCMHa3bmsio9GIAxxnx5Fgy7ZvL+QQf178UZSFlfqdf79RyM9ir55P8xkMdPqd8Yqs9w4Rk/3xLyv5mJ0S5nFCkWnxl2oT2d7qXDOG9Me8fs3e4wbhWS6e59RUiUn//pz+cnRohZDKZ3j9ZuESTrF/v/fL/Jk+L8mye1Hi24YxppeXCknvrFHbneqztnW61bVMBV3C6hwznub9qjJfrPHcFcvZ/vRYh77/QEyiquabCsbXT7RQb5W83HXUErn7bJKa1iXkmfVsIfrdexHhfVB0zSWT4vgtz//HgljQm/4lCMQNOiSNTubL17frI9e+fQVdztK+O8BzFl8/dxbCyG8kXiaoiw9LGpmldbw8qeZ1Fxp4Ku8ckPA7U3DI5MzypoJujPO8frPTx+Bw+HgnS9P8FTioWahppqm8eh9YWia1qIYSymRSBKfmMDGheNVmKWiy6OE/x7A9KMjaJMVago6eC5XLKXk/2/v/oPtqMs7jr8/SUj5ETBBbHohtEER0ou0JMRqB+w4CINiQGd0ajvAYJVxCp0aW7GT1ukgzDDFhlr9ow3j8MP8kWlt0ZKILYFSaNU/MIEEAgkJFii54QZDgSRQBcN9+sfuudl7sufHPT/u7jn7ec3cuefs7tn9ZrP32T3Pd/f5bnhsD5+5c9PksvVGR07gxsvO4m1Hz2HRgmMm79JZc8UyVpw9cvh3zv312XWsuSIZ0vCv/nUnV96xmb+572kuPGshqzcevlqvPwk1mpbdP5ql3G8ZZpWTl/8p249z/M11knNuVT6h3uO7X46zr783lt6wMZ4YO3K5+nIFeWULWpVdqF/ftrFX4vHnX467t+yOQ4cONR0EpX5aOyOBtbNvzAYZzvEPpojDVSTzyghEh7djZvP5jZ6uzXp23+u89sYhrj7/tMlbMrNtqR/fNm9w81qt+kZlF7JqJRXOPnUBHztnEbNnz57ybSQvLZWd1qhUxHT3jdkwcuAvue3jB7h67SN8du3m3DICnQapbJDcsffglLx73mDop73jOBYcO5cPnHES0pG1cLInjydf2M9n7tzE9x7bM+XEUKtVj2ia529Hs0HiswOij44ceZKazr4xG0p5XwPK9lPlVE+23HCz9EknKY28dUTkD4bezm2ak2PKjr0ay268b0rJ427bWV9eeTr/JpdLtqrCY+4Ov+k8sdvMxMQE92wbZ8XZI8ya1fxLYURmTNs0vVNrC0HH49vWy3squNXytX1RS/14jFurmkZP7jrwD5FaEG4W4NpZZjrbyT71Wh9guzkRRUwdyBxIxrUl2jqR5J2QHPStajzYegW004nZqE+gFmizFwLZabXXExMTk0XRakE+r0MVWg+I3mz72b6NyQ7a9OGtdgJ4bV+4Nr7ZkfoW+CWdKulBSdslPSlpZTr9REn3S3o6/b2gX20YdM0CZaefb9RxmXdCyE7L1qmvFUX79V85fkphtZgIvnjR6cREMDExMXnFXaui2awjun48gNGRE7jtqnO7fuDKHbVmOfIS/734AUaAZenr44FdwCjw18CqdPoq4Kut1lXVzt1GnZLtdo6206lZq4lTf598dt4R9XJyOk2f2PNqvO+mf5/s1F2/dWxybNu8e+vrTUxMxPqtY3H+zQ+UohPW9/LbMKDoMXeB9cBFwE5gJA6fHHa2+mxVA3+j4NPuXSrtBK+7t+yOd666J+7esvuIea22U3+nzraxV2Lb7ql3INUXVmt3fN6i+U4gGwaNAv+M5PglLQaWAg8DCyNiPJ21F1g4E20YRI1y9u2mL1rl/CMCAuYfO5d3nTQvd152FKqoy/lnO3JrD1y9Z1Eyju2sWbM46+Tkd20krFbDGZap+JlTRDbM+h74Jc0DvgN8ISKm/MWnZ6TcBLakz0naLGnzvn37+t3M0soG25p2Ano7fQPbxw9wy327uP6y0SMGFNk+foBr1j06ZRSqvJx/O52mtSBaq9fTLJi22/Z+K9NJyKzn8r4G9OoHOArYCPxpZlppUz1lSTV08/DRxMRErN8yFu+76f7cmjqNtlM/rV8Pi7XiFItZ7zDTqR4ll0q3Azsi4muZWRuAq9LXV5Hk/kuhLDVa6gf/nk7KYfv4AW76/g5+9uZbLa+a865qszV3svMiJ7VTmx85V+l509rRSYql022ZVVU/Uz3nAVcCF0jamv5cAtwMXCTpaeDC9H0plCWvWz/4d30AbhbkRkdO4MsfXcLRR82GnCxFO59vdrvnky/sP+LzrW4FnU5g7iTFUpYTttnAyPsaULafMt3VU3Q6qJ1USLM2dppKqa3zibHWwy3WT+t3+qbo/xOzssK1enqjV/VwOtWqjk5E85IMrea3Wn66n+9km2bWG5Us2RB9yP0WnQ7asfcgqzfu5J5t47n/rlZpj25r03eSivEdMmblMtSBvx+531Zju/a7k3F05AS+dPGZrL73yEHDa/P//vKlkD6c14vtlaHfw8x6Z6gD/0wHrU5PNNM5YUji0t84mVuvPDf3waraHTfXrGu/Hc2276t1s+Ez1IF/poNWpyea6Z4w6v9dk59/4QBPvrCfJQvntRzasJvtm9lgc+duCXTb+Vn7fERw7botXHfxGdyycVfbHdDufDUbTo06d+cU0ZiqaRVYa1fw3a4bYM0Vy1iyMKm70+4VfzfbN7PBM9SpnrLoZSqlPh+fXXctgD/14mtdD2ZuZsPLgb/H8jpKe9nJ3GqUq3a3NxN3IJlZOTnw91je1X0vO5nrg3reursZgrHGJwaz4eXA32P9voW0VyeRVu2czti8ZjZYHPh7rBeBeSaCa6t2TmdsXjMbLA78PdSrgF0/8Hi/dPLglp/kNRt8Dvw91Ku8eauyDL3SydW7n+Q1G3wO/D1SK3e65vLp583r5ZVl6KZdjU423Vy9O9dvNrgc+Htk+/gBrl23BcSUq+FsgJxOoO3VlXWzk00323Cu32xwOfDTm6vXdjpDJTE6csJkeYWZ0K+cvHP9ZoPLgZ/eXL222xk601fK/crJO9dvNrgqG/g7TcFMV32ALPJK2Xl5M4MKB/68GjczcfXazba6DdzOy5sZVDjwlzlH3SjAdxu4y/xvNrOZU7nAXwuqQGlz1I0CfLeB23l5M4MKBv5+pzv6eYeQA7eZ9ULlAn+/0x39vEMI3EFrZt2rXODv91XzIJxYzKzaKhf4+21QTiz+5mBWXQ78A2YmSjmY2XBz4K8o39ppVl0O/AUqMt3iO4TMqsuBv0BOt5hZERz4C+R0i5kVYU7RDaiyWrrFzGwm+YrfzKxiHPjNzCrGgd/MrGIc+M3MKsaB38ysYhz4zcwqxoHfzKxiNAjVGSUdBHYW3Y4SOwl4qehGlJz3UXPeP60N4j76tYh4R/3EQXmAa2dELC+6EWUlabP3T3PeR815/7Q2TPvIqR4zs4px4Dczq5hBCfzfLLoBJef905r3UXPeP60NzT4aiM5dMzPrnUG54jczsx5x4Dczq5hSB35JH5a0U9JPJK0quj1lIOlUSQ9K2i7pSUkr0+knSrpf0tPp7wVFt7VIkmZL2iLpnvT9aZIeTo+lb0uaW3QbiyRpvqS7JD0laYek3/YxdJikP0n/vp6Q9A+Sjh6mY6i0gV/SbODvgI8Ao8DvSxottlWlcAj4YkSMAu8H/ijdL6uAByLi3cAD6fsqWwnsyLz/KvC3EXE68Arw2UJaVR7fAO6NiCXAb5LsKx9DgKRTgM8DyyPiPcBs4PcYomOotIEf+C3gJxHxTES8Cfwj8LGC21S4iBiPiEfT1wdJ/mBPIdk3a9PF1gIfL6aFxZO0CPgocFv6XsAFwF3pIlXfP28Dfge4HSAi3oyIV/ExlDUHOEbSHOBYYJwhOobKHPhPAXZn3o+l0ywlaTGwFHgYWBgR4+msvcDCgppVBl8H/gyYSN+/HXg1Ig6l76t+LJ0G7APuTNNht0k6Dh9DAETEHuAW4HmSgL8feIQhOobKHPitCUnzgO8AX4iIA9l5kdyjW8n7dCWtAH4aEY8U3ZYSmwMsA9ZExFLgderSOhU/hhaQfPs5DTgZOA74cKGN6rEyB/49wKmZ94vSaZUn6SiSoL8uIr6bTn5R0kg6fwT4aVHtK9h5wGWSniNJD15Aks+en35tBx9LY8BYRDycvr+L5ETgYyhxIfBsROyLiF8A3yU5robmGCpz4N8EvDvtSZ9L0rmyoeA2FS7NV98O7IiIr2VmbQCuSl9fBayf6baVQUT8eUQsiojFJMfMf0TE5cCDwCfTxSq7fwAiYi+wW9KZ6aQPAdvxMVTzPPB+Scemf2+1/TM0x1Cpn9yVdAlJvnY2cEdE3FRwkwon6XzgB8A2Duew/4Ikz/9PwK8C/wP8bkS8XEgjS0LSB4HrImKFpHeSfAM4EdgCXBERbxTZviJJOoek83su8AzwByQXgj6GAEk3AJ8iuYtuC3A1SU5/KI6hUgd+MzPrvTKneszMrA8c+M3MKsaB38ysYhz4zcwqxoHfzKxiHPjNuiTpK5Kuy5k+X9K1Ha7z05JOzrx/TtJJ3bTTrMaB3yoh88TlTJoP5Ab+NtrzaZJyAWY958BvQ0HSX6ZjN/wwrZ9+naSHJH1d0mZgpaQPpUXJtkm6Q9IvpZ+dvJqWtFzSQ+nrr6TLPSTpGUmfz2zvy5J2SfohcGZOkwBuBt4laauk1ZI+KOkHkjYA2yUtlvREZp3Xpdv8JLAcWJd+9ph0kT+W9Gja/iU93oVWIQ78NvAkvRf4BEld+Y+QBM2auRGxnGRsh28Bn4qIs0kKlV3TxuqXABeTlAm/XtJRks4lKQdxDnAJ8N4Gn10F/HdEnBMRX0qnLQNWRsQZjTYYEXcBm4HL08/+LJ31UkQsA9YAR6SWzNrlwG/D4DxgfUT8PB2j4HuZed9Of59JUnhrV/p+LUlN+la+HxFvRMRLJEXLFgIfAP4lIv4vrYw6nRpSP46IZ6exfFatIN8jwOIO12HmwG9D7/U2ljnE4b+Fo+vmZWuxvEXyTSGXkmExt6Y/f9hGe7Lbzdt2vVpbmrbDrBUHfhsGPwIuTcdFnQesyFlmJ7BY0unp+yuB/0xfPwecm77+RBvb+y/g45KOkXQ8cClAROxOUzPnRMStwEHg+CbreRH4ZUlvT/sbsu1u9Vmzjjnw28CLiE0k6ZbHgX8jqVy6v26Zn5NUoPxnSbXKprems28AvpF2Ar/VxvYeJUkhPZZub1OD5f4X+FE6YPfqnPm/AG4EfgzcDzyVmf0t4Na6zl2znnB1ThsKkuZFxGuSjiW5Iv9cbWxiM5vKeUIbFt+UNEqSJ1/roG/WmK/4zcwqxjl+M7OKceA3M6sYB34zs4px4DczqxgHfjOzivl/PEweuDJ0ijwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M_K2cXQIK_KL"
      },
      "source": [
        "### Predict on a custom image (example)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5wqSLmcvK_KN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "f90ef4fb-315e-4d12-fccb-ba12adbca628"
      },
      "source": [
        "# Load an image\n",
        "image_path = data_root + 'images/512x384/' + ids.image_name.values[0]\n",
        "im = preprocess_fn(iu.read_image(image_path))\n",
        "\n",
        "# Create a batch, of 1 image\n",
        "batch = np.expand_dims(im, 0)\n",
        "\n",
        "# Predict quality score\n",
        "y_pred = helper.model.predict(batch).squeeze()\n",
        "print(f'Predicted score: {y_pred:.{2}f}, ground-truth score: {ids.MOS.values[0]:.{2}f}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predicted score: 68.33, ground-truth score: 77.38\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}