concatenation_experiments_and_error_analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "concatenation_experiments_and_error_analysis.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "mount_file_id": "1MmA_cg4owDtDm7Y0lfWRVZ1tlNGprhn1",
      "authorship_tag": "ABX9TyPCEKL3i8zCM5kZT3zDthb6",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/arunraja-hub/Preference_Extraction/blob/master/concatenation_experiments_and_error_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w7ASSmHyk1A4",
        "colab_type": "text"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IMKhp_sBk2yq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import joblib\n",
        "import concurrent.futures\n",
        "import itertools\n",
        "import os\n",
        "import pickle\n",
        "import random\n",
        "import sys\n",
        "import time\n",
        "\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import PIL\n",
        "from PIL import Image\n",
        "import scipy\n",
        "from scipy import ndimage\n",
        "from sklearn import metrics\n",
        "from sklearn.utils import shuffle\n",
        "import io\n",
        "import collections\n",
        "\n",
        "%tensorflow_version 2.x\n",
        "import tensorflow as tf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LhVtgzu_k67U",
        "colab_type": "text"
      },
      "source": [
        "# Load Data from Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "STmpXR0ok5rn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "obs = joblib.load('/content/drive/My Drive/_data/PreferenceExtraction/obs.pkl')\n",
        "prefs = joblib.load('/content/drive/My Drive/_data/PreferenceExtraction/pref.pkl')\n",
        "conv_acts = joblib.load('/content/drive/My Drive/_data/PreferenceExtraction/convAct.pkl')\n",
        "dens_acts = joblib.load('/content/drive/My Drive/_data/PreferenceExtraction/densAct.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qtv0NDAblJm3",
        "colab_type": "text"
      },
      "source": [
        "# Modelling Functions\n",
        "\n",
        "1. Metrics\n",
        "2. CNN\n",
        "3. DNN\n",
        "4. Training code\n",
        "5. Error analysis"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZiEZGLIKlBxa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_val_auc(logs):\n",
        "      for key in logs:\n",
        "        if key.startswith('val_auc'):\n",
        "          return logs[key]\n",
        "\n",
        "class BestStats(tf.keras.callbacks.Callback):\n",
        "  \"\"\"A callback to keep track of the best val accuracy and auc seen so far.\"\"\"\n",
        "  def on_train_begin(self, logs):\n",
        "      self.bestMetric = -float('inf')\n",
        "      self.bestLogs = None\n",
        "      self.bestTrain = -float('inf')\n",
        "      self.num_epochs = 0\n",
        "\n",
        "  def on_epoch_end(self, epoch, logs):\n",
        "    self.num_epochs += 1\n",
        "    self.bestTrain = max(self.bestTrain, logs.get('accuracy'))\n",
        "\n",
        "    val_accuracy = logs.get('val_accuracy')\n",
        "    if val_accuracy == None:\n",
        "      return \n",
        "\n",
        "    val_auc = get_val_auc(logs)\n",
        "    \n",
        "    metric = (val_accuracy + val_auc) / 2.0\n",
        "\n",
        "    if metric > self.bestMetric:\n",
        "      self.bestMetric = metric\n",
        "      self.bestLogs = logs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AYrjzFNjlOYn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def cnn(reg_amount, drop_rate, xs_shape):\n",
        "\n",
        "  model = tf.keras.models.Sequential([\n",
        "    tf.keras.layers.Conv2D(32, 2, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(reg_amount), input_shape=xs_shape[1:]),\n",
        "    tf.keras.layers.Conv2D(16, 1, activation='relu', strides=1, kernel_regularizer=tf.keras.regularizers.l2(reg_amount)),\n",
        "    tf.keras.layers.AveragePooling2D(),\n",
        "    tf.keras.layers.Flatten(),\n",
        "    tf.keras.layers.Dropout(drop_rate),\n",
        "    tf.keras.layers.Dense(1, activation='sigmoid', kernel_regularizer=tf.keras.regularizers.l2(reg_amount)),\n",
        "  ])\n",
        "\n",
        "  model.compile(optimizer=tf.keras.optimizers.Adam(.01),\n",
        "                loss='binary_crossentropy',\n",
        "                metrics=['accuracy',\n",
        "                        tf.keras.metrics.AUC()\n",
        "                        ],\n",
        "                )\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K8blFY9vlvnW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def dnn(reg_amount, drop_rate, layer_sizes):\n",
        "\n",
        "  layers = []\n",
        "  for layer_size in layer_sizes:\n",
        "    layers.append(tf.keras.layers.Dense(layer_size, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(reg_amount)))\n",
        "    layers.append(tf.keras.layers.Dropout(drop_rate))\n",
        "\n",
        "  model = tf.keras.models.Sequential(layers + [\n",
        "    tf.keras.layers.Dense(1, activation='sigmoid', kernel_regularizer=tf.keras.regularizers.l2(reg_amount))\n",
        "  ])\n",
        "\n",
        "  model.compile(optimizer=tf.keras.optimizers.Adam(.01),\n",
        "                loss='binary_crossentropy',\n",
        "                metrics=['accuracy', tf.keras.metrics.AUC()],\n",
        "                )\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ev7HrTurly2w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_best_logs(xs, ys, prefs, num_train, num_val, do_summary, hparams, model, epochs):\n",
        "  \"\"\"Trains the model and retruns the logs of the best epoch. randomly splits the train and val data before training.\"\"\"\n",
        "  tf.keras.backend.clear_session()\n",
        "  model = model(**hparams)\n",
        "  xs, ys, prefs = shuffle(xs, ys, prefs)\n",
        "\n",
        "  xs_val = xs[num_train:num_train+num_val]\n",
        "  ys_val = ys[num_train:num_train+num_val]\n",
        "  pref_val = prefs[num_train:num_train+num_val]\n",
        "  early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=30, verbose=0)\n",
        "  best_stats = BestStats()\n",
        "  model.fit(xs[:num_train], ys[:num_train], epochs=epochs, batch_size=256, validation_freq=1, callbacks=[best_stats, early_stopping], validation_data=(xs_val, ys_val), verbose=0)\n",
        "  ys_val_hat = model.predict(xs_val)\n",
        "  if do_summary:\n",
        "    model.summary()\n",
        "    print(\"Best train accuracy:\", best_stats.bestTrain)\n",
        "    print(\"Number of epochs:\", best_stats.num_epochs)\n",
        "    \n",
        "  ys_val_hat = (ys_val_hat.flatten() > 0.5).astype(int)\n",
        "  errors = (ys_val_hat != ys_val)\n",
        "  pct_diff = pd.Series(pref_val[errors]).value_counts(normalize=True) - pd.Series(pref_val).value_counts(normalize=True)\n",
        "\n",
        "  return best_stats.bestLogs, ys_val_hat, ys_val, pref_val, pct_diff\n",
        "\n",
        "def multiple_train_ave(hparams, model, xs_for_train, ys, prefs, num_train, num_val, epochs, num_repeat = 10):\n",
        "  \"\"\"Trains the model multiple times with the same parameters and returns the average metrics\"\"\"\n",
        "  start = time.time()\n",
        "  all_val_auc = []\n",
        "  all_val_accuracy = []\n",
        "\n",
        "  pct_diffs = []\n",
        "  do_summary = True\n",
        "  for i in range(num_repeat):\n",
        "    print('Doing training iteration no.', i+1)\n",
        "    logs, ys_val_hat, ys_val, pref_val, pct_diff = train_best_logs(xs_for_train, ys, prefs, num_train, num_val, do_summary, hparams, model, epochs)\n",
        "    all_val_auc.append(get_val_auc(logs))\n",
        "    all_val_accuracy.append(logs.get('val_accuracy'))\n",
        "    do_summary = False \n",
        "    pct_diffs.append(pct_diff)\n",
        "\n",
        "  mean_val_auc = np.mean(all_val_auc)\n",
        "  mean_val_accuracy = np.mean(all_val_accuracy)\n",
        "  metric = (mean_val_auc + mean_val_accuracy) / 2.0\n",
        "  print_data = (\"mean_val_auc\", mean_val_auc, \"mean_val_accuracy\", mean_val_accuracy, \"metric\", metric, \"val_auc_std\", np.std(all_val_auc), \"val_accuracy_std\", np.std(all_val_accuracy))\n",
        "\n",
        "  end = time.time()\n",
        "\n",
        "  return metric, print_data, ys_val_hat, ys_val, pref_val, pct_diffs\n",
        "\n",
        "def train_eval(hparams, model, xs, ys, prefs, num_train, num_val, epochs):\n",
        "    \n",
        "    print(\"Model:\", model)\n",
        "    print(\"Shape of input:\", xs.shape)\n",
        "    print(\"Hyperparameters:\", hparams)\n",
        "    print(\"Number of training data:\", num_train)\n",
        "    print(\"Number of validation data:\", num_val)\n",
        "    print(\"Number of epochs:\", epochs)\n",
        "    \n",
        "    metric, print_data, ys_val_hat, ys_val, pref_val, pct_diffs = multiple_train_ave(hparams, model, xs, ys, prefs, num_train, num_val, epochs)\n",
        "    print(\"Results\", print_data)\n",
        "\n",
        "    return ys_val_hat, ys_val, pref_val, pct_diffs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zAVXpBKrnYj9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def error_analysis_plot(y_pred, y_true, prefs):\n",
        "    y_pred = (y_pred.flatten() > 0.5).astype(int)\n",
        "    errors = (y_pred != y_true)\n",
        "    prefs[prefs == -100] = -12\n",
        "    plt.hist(prefs, bins=13, label='True values')\n",
        "    plt.hist(prefs[errors], bins=13, label='Prediction errors')\n",
        "    plt.ylabel('Number of occurrences')\n",
        "    plt.xlabel('Human preference value')\n",
        "    plt.legend()\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "89i3D1jwoVTx",
        "colab_type": "text"
      },
      "source": [
        "# Experiments"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3NH3YJntuNe7",
        "colab_type": "text"
      },
      "source": [
        "1. Train on 10K observation data\n",
        "2. Train on all possible combination of concatenations with 50 data points"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QyoA1bP7o7E3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ys = (prefs > -6).astype(int)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6hTKvFyTvQ8M",
        "colab_type": "text"
      },
      "source": [
        "## Experiment (1) - Train on 10K Observation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XZQ0ljbTuW_k",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 889
        },
        "outputId": "a9117953-bbf9-4dc1-fc7a-3dd6895146a3"
      },
      "source": [
        "# Experiment No. 1\n",
        "hparam = {\"reg_amount\": 0.0, \"drop_rate\": 0.0, \"xs_shape\": obs.shape}\n",
        "ys_val_hat, ys_val, pref_val = train_eval(hparam, cnn, obs, ys, prefs, num_train=10000, num_val=2000, epochs=1000)\n",
        "error_analysis_plot(ys_val_hat, ys_val, pref_val)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: <function cnn at 0x7fe12dbdcf28>\n",
            "Shape of input: (23750, 14, 16, 5)\n",
            "Hyperparameters: {'reg_amount': 0.0, 'drop_rate': 0.0, 'xs_shape': (23750, 14, 16, 5)}\n",
            "Number of training data: 10000\n",
            "Number of validation data: 2000\n",
            "Number of epochs: 1000\n",
            "Doing training iteration no. 1\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 13, 15, 64)        1344      \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 13, 15, 32)        2080      \n",
            "_________________________________________________________________\n",
            "average_pooling2d (AveragePo (None, 6, 7, 32)          0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 1344)              0         \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 1344)              0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 1)                 1345      \n",
            "=================================================================\n",
            "Total params: 4,769\n",
            "Trainable params: 4,769\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Best train accuracy: 0.9079999923706055\n",
            "Number of epochs: 105\n",
            "Doing training iteration no. 2\n",
            "Doing training iteration no. 3\n",
            "Doing training iteration no. 4\n",
            "Doing training iteration no. 5\n",
            "Results ('mean_val_auc', 0.9325024247169494, 'mean_val_accuracy', 0.8850000023841857, 'metric', 0.9087512135505675, 'val_auc_std', 0.014382437879937725, 'val_accuracy_std', 0.007368851520081325)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5xVdb3/8ddbLuItFBsJRYU63lAu0oAYRzAx1EBRQrNTAaV5OdaxG0rl8fIrf5GZ1/pJZBcsT+Il0xINRBGyg3IRr2igYkKoEyoKijrw+f2xv7PZwFzWDLP3nhnez8djHnut7/qutT5rYOYz6/td6/tVRGBmZgawQ7kDMDOzlsNJwczM8pwUzMwsz0nBzMzynBTMzCyvfbkD2BYf/vCHo0ePHuUOw8ysVVm4cOG/IqKitm2tOin06NGDBQsWlDsMM7NWRdJLdW1z85GZmeU5KZiZWV7RkoKkgyQtLvh6S9LXJXWRNFPS0vS5R6ovSddJWibpCUn9ixWbmZnVrmh9ChHxHNAPQFI7YCVwJzARmBURkyRNTOsXAicAB6SvI4Ab0qeZlcAHH3zAihUrWL9+fblDsWbSqVMnunfvTocOHTLvU6qO5mHA8xHxkqRRwNGpfCowm1xSGAXcFLnBmOZJ2l1St4hYVaIYzbZrK1asYLfddqNHjx5IKnc4to0igtWrV7NixQp69uyZeb9S9SmcDvw+LXct+EX/CtA1Le8DvFywz4pUthlJZ0laIGlBVVVVseI12+6sX7+ePffc0wmhjZDEnnvu2eg7v6InBUkdgZOA27bclu4KGjVMa0RMiYjKiKisqKj1MVszayInhLalKf+epbhTOAFYFBGvpvVXJXUDSJ+vpfKVwL4F+3VPZWZmViKl6FP4HJuajgDuBsYBk9LnXQXlX5V0C7kO5jXuTzArnx4T72nW4y2fNKLe7atXr2bYsGEAvPLKK7Rr146a1oBHH32Ujh07Nms8WRx99NFceeWVVFZWlvzc5VLUpCBpF+BTwNkFxZOAWyWdAbwEnJbKpwOfBpYB7wBfKmZszf0fvi4N/SCYWc6ee+7J4sWLAbj00kvZdddd+fa3v53fXl1dTfv2rXoQhlahqM1HEbEuIvaMiDUFZasjYlhEHBARx0bE66k8IuK8iPhYRPSOCI9fYbadGz9+POeccw5HHHEEF1xwAZdeeilXXnllfvthhx3G8uXLAfjd737HwIED6devH2effTYbNmzY7Fj33Xcfp556an599uzZjBw5EoBzzz2XyspKDj30UC655JJaY9l1113zy7fffjvjx48HoKqqis985jMMGDCAAQMG8PDDDwPw0EMP0a9fP/r168fhhx/O22+/vc3fj1LwG81m1qKtWLGCv/3tb1x11VV11lmyZAnTpk3j4YcfZvHixbRr146bb755szrHHnssjzzyCOvWrQNg2rRpnH766QBcfvnlLFiwgCeeeIKHHnqIJ554InN8559/Pt/4xjeYP38+d9xxB2eeeSYAV155JT/72c9YvHgxc+fOZaeddmrspZeF78XMrEU79dRTadeuXb11Zs2axcKFCxkwYAAA7777Lnvttddmddq3b8/xxx/Pn/70J8aMGcM999zDFVdcAcCtt97KlClTqK6uZtWqVTzzzDP06dMnU3z3338/zzzzTH79rbfeYu3atQwePJhvfvObfP7zn2f06NF07969MZddNk4KZtai7bLLLvnl9u3bs3Hjxvx6zTP4EcG4ceP44Q9/WO+xTj/9dH7605/SpUsXKisr2W233XjxxRe58sormT9/PnvssQfjx4+v9dn+wsc7C7dv3LiRefPm0alTp83qT5w4kREjRjB9+nQGDx7MX/7yFw4++ODGXXwZuPnIzFqNHj16sGjRIgAWLVrEiy++CMCwYcO4/fbbee213BPur7/+Oi+9tPXo0EOHDmXRokX84he/yDcdvfXWW+yyyy507tyZV199lXvvvbfWc3ft2pUlS5awceNG7rzzznz58OHDuf766/PrNZ3lzz//PL179+bCCy9kwIABPPvss83wHSg+3ymYWa1a4pNzn/nMZ7jppps49NBDOeKIIzjwwAMB6NWrFz/4wQ8YPnw4GzdupEOHDvzsZz9j//3332z/du3aMXLkSH7zm98wdepUAPr27cvhhx/OwQcfzL777svgwYNrPfekSZMYOXIkFRUVVFZWsnbtWgCuu+46zjvvPPr06UN1dTVDhgxh8uTJXHPNNTz44IPssMMOHHrooZxwwglF/M40H+VeKm6dKisro6mT7PiRVLPNLVmyhEMOOaTcYVgzq+3fVdLCiKj15Qs3H5mZWZ6TgpmZ5TkpmJlZnpOCmZnlOSmYmVmek4KZmeX5PQUzq92lnZv5eGsarNKuXTt69+5NdXU1hxxyCFOnTmXnnXdu0unGjx/PyJEjGTNmDGeeeSbf/OY36dWrV611Z8+eTceOHfnEJz4BwOTJk9l5550ZO3Zsk87dmvlOwcxajJ122onFixfz1FNP0bFjRyZPnrzZ9urq6iYd98Ybb6wzIUAuKfztb3/Lr59zzjklSwgRsdnQHVuu16Wp34uGOCmYWYt01FFHsWzZMmbPns1RRx3FSSedRK9evdiwYQMTJkxgwIAB9OnTh5///OdA7pfpV7/6VQ466CCOPfbY/JAXkJssp+ZF1/vuu4/+/fvTt29fhg0bxvLly5k8eTJXX301/fr1Y+7cuZsN0b148WIGDRpEnz59OOWUU3jjjTfyx7zwwgsZOHAgBx54IHPnzq31On784x/nY60Zlnv58uUcdNBBjB07lsMOO4y5c+dutv7yyy8zYcIEDjvsMHr37s20adMAtvperFu3jhEjRtC3b18OO+ywfL1t4eYjM2txqquruffeezn++OOB3DhHTz31FD179mTKlCl07tyZ+fPn89577zF48GCGDx/OY489xnPPPcczzzzDq6++Sq9evfjyl7+82XGrqqr4yle+wpw5c+jZsyevv/46Xbp04ZxzztlsUp9Zs2bl9xk7dizXX389Q4cO5eKLL+ayyy7jmmuuycf56KOPMn36dC677DLuv//+zc43Y8YMli5dyqOPPkpEcNJJJzFnzhz2228/li5dytSpUxk0aBDLly/fbP2OO+5g8eLFPP744/zrX/9iwIABDBkyZKvvxR133MHee+/NPffkRmhYs6bhJrqG+E7BzFqMd999l379+lFZWcl+++3HGWecAcDAgQPp2bMnkPtFe9NNN9GvXz+OOOIIVq9ezdKlS5kzZw6f+9znaNeuHXvvvTfHHHPMVsefN28eQ4YMyR+rS5cu9cazZs0a3nzzTYYOHQrAuHHjmDNnTn776NGjAfj4xz+en+yn0IwZM5gxYwaHH344/fv359lnn2Xp0qUA7L///gwaNChft3D9r3/9a/5aunbtytChQ5k/f/5W34vevXszc+ZMLrzwQubOnUvnztveD+Q7BTNrMWr6FLZUOHx2RHD99ddz3HHHbVZn+vTpRY9vSzvuuCOQ6yCvrY0/IvjOd77D2WefvVn58uXLN7smYKv1uhTWO/DAA1m0aBHTp0/noosuYtiwYVx88cWNvYzN+E7BzFqV4447jhtuuIEPPvgAgL///e+sW7eOIUOGMG3aNDZs2MCqVat48MEHt9p30KBBzJkzJz/k9uuvvw7AbrvtVut0mZ07d2aPPfbI9xf89re/zd81ZI31V7/6VX5E1ZUrV27W11GXo446Kn8tVVVVzJkzh4EDB25V75///Cc777wzX/jCF5gwYUJ+WPFt4TsFM6tdhkdIy+HMM89k+fLl9O/fn4igoqKCP/7xj5xyyik88MAD9OrVi/32248jjzxyq30rKiqYMmUKo0ePZuPGjey1117MnDmTE088kTFjxnDXXXdtNjcCwNSpUznnnHN45513+OhHP8qvf/3rzLEOHz6cJUuW5GPZdddd+d3vftfgTHKnnHIK//u//0vfvn2RxBVXXMFHPvKRreZkePLJJ5kwYQI77LADHTp04IYbbsgcW108dHaReehsay08dHbb1KKGzpa0u6TbJT0raYmkIyV1kTRT0tL0uUeqK0nXSVom6QlJ/YsZm5mZba3YfQrXAvdFxMFAX2AJMBGYFREHALPSOsAJwAHp6yxg2++DzMysUYqWFCR1BoYAvwSIiPcj4k1gFDA1VZsKnJyWRwE3Rc48YHdJ3YoVn5ltrTU3J9vWmvLvWcw7hZ5AFfBrSY9JulHSLkDXiFiV6rwCdE3L+wAvF+y/IpVtRtJZkhZIWlBVVVXE8M22L506dWL16tVODG1ERLB69Wo6derUqP2K+fRRe6A/8LWIeETStWxqKgIgIkJSo/4HRsQUYArkOpqbK1iz7V337t1ZsWIF/mOr7ejUqRPdu3dv1D7FTAorgBUR8Uhav51cUnhVUreIWJWah2oe2l0J7Fuwf/dUZmYl0KFDh/ybsrb9KlrzUUS8Arws6aBUNAx4BrgbGJfKxgF3peW7gbHpKaRBwJqCZiYzMyuBYr+89jXgZkkdgReAL5FLRLdKOgN4CTgt1Z0OfBpYBryT6pqZWQkVNSlExGKgthckhtVSN4DzihmPmZnVz2MfmZlZnpOCmZnlOSmYmVmek4KZmeU5KZiZWZ6TgpmZ5TkpmJlZnpOCmZnlOSmYmVmek4KZmeU1mBQkXSHpQ5I6SJolqUrSF0oRnJmZlVaWO4XhEfEWMBJYDvwbMKGYQZmZWXlkSQo1g+aNAG6LiDVFjMfMzMooyyipf5b0LPAucK6kCmB9ccMyM7NyaPBOISImAp8AKiPiA3JzHYwqdmBmZlZ6WTqadwb+E7ghFe1N7XMkmJlZK5elT+HXwPvk7hYgN2/yD4oWkZmZlU2WpPCxiLgC+AAgIt4BVNSozMysLLIkhfcl7QQEgKSPAe8VNSozMyuLLE8fXQLcB+wr6WZgMDC+mEGZmVl5NJgUImKmpEXAIHLNRudHxL+KHpmZmZVclqePTgGqI+KeiPgzUC3p5CwHl7Rc0pOSFktakMq6SJopaWn63COVS9J1kpZJekJS/225MDMza7wsfQqXFL7FHBFvkmtSyuqTEdEvImoeY50IzIqIA4BZaR3gBOCA9HUWmx6BNTOzEsmSFGqrk6Uvoi6jgKlpeSpwckH5TZEzD9hdUrdtOI+ZmTVSlqSwQNJVkj6Wvq4CFmY8fgAzJC2UdFYq6xoRq9LyK0DXtLwP8HLBvitS2WYknSVpgaQFVVVVGcMwM7MssiSFr5F7eW1a+noPOC/j8f89IvqTaxo6T9KQwo0REaRHXbOKiCkRURkRlRUVFY3Z1czMGpDl6aN1bGr3b5SIWJk+X5N0JzAQeFVSt4hYlZqHXkvVVwL7FuzePZWZmVmJZHn66EBJUyTNkPRAzVeG/XaRtFvNMjAceAq4GxiXqo0D7krLdwNj01NIg4A1Bc1MZmZWAlk6jG8DJgM3AhsaceyuwJ2Sas7zPxFxn6T5wK2SzgBeAk5L9acDnwaWkRuJ9UuNOJeZmTWDLEmhOiIa/XhoRLwA9K2lfDUwrJbyIHtfhZmZFUGWjuY/SfpPSd3Si2ddJHUpemRmZlZyWe4Uatr/C+dlDuCjzR+OmZmVU5anj3qWIhAzMyu/TDOvSbpI0pS0foCkkcUPzczMSs0zr5mZWZ5nXjMzszzPvGZmZnmeec3MzPLqTQqSdgD2AEbjmdfMzNq8epNCRGyUdEFE3ArcU6KYzMysTLL0Kdwv6duS9vUbzWZmbVuWPoXPps/CcYn8RrOZWRuUpU9hYkRMK1E8ZmZWRln6FCaQm3HNzMwa0GNiabpfl08aUZTjuk/BzMzy3KdgZmZ5HiXVzMzyGkwKksbWVh4RNzV/OGZmVk5Zmo8GFCx3IjeV5iLAScHMrI3J0nz0tcJ1SbsDtxQtIjMzK5ssTx9taR3gfgYzszYoS5/Cn0jDZpNLIr2AW7OeQFI7YAGwMiJGSupJ7k5jT2Ah8MWIeF/SjuSapD4OrAY+GxHLG3EtZma2jbL0KVxZsFwNvBQRKxpxjvOBJcCH0vqPgKsj4hZJk4EzgBvS5xsR8W+STk/1PlvbAc3MrDiyNB/9A3gkIh6KiIeB1ZJ6ZDm4pO7ACODGtC7gGOD2VGUqcHJaHpXWSduHpfpmZlYiWZLCbcDGgvUNqSyLa4ALCvbfE3gzIqrT+gpgn7S8D/AyQNq+JtXfjKSzJC2QtKCqqipjGGZmlkWWpNA+It6vWUnLHRvaSdJI4LWIWLgN8W0lIqZERGVEVFZUVDTnoc3MtntZkkKVpJNqViSNArLMvDYYOEnScnIdy8cA1wK7S6rpy+gOrEzLK4F90znaA53JdTibmVmJZEkK5wDflfQPSf8ALgTObminiPhORHSPiB7A6cADEfF54EFgTKo2DrgrLd+d1knbH4iIwMzMSibLy2vPA4Mk7ZrW127jOS8EbpH0A+Ax4Jep/JfAbyUtA14nl0jMzKyEsryn8H+BKyLizbS+B/CtiLgo60kiYjYwOy2/AAyspc564NSsxzQzs+aXpfnohJqEABARbwCfLl5IZmZWLlmSQrv0tjEAknYCdqynvpmZtVJZ3mi+GZgl6ddp/UtsesnMzMzakCwdzT+S9DhwbCr6fkT8pbhhmZlZOWS5U4DcU0IdyA2M91jxwjEzs3JqsE9B0mnAo+TeHTgNeETSmPr3MjOz1ijLncL3gAER8RqApArgfjYNamdmZm1ElqePdqhJCMnqjPuZmVkrk+VO4T5JfwF+n9Y/C0wvXkhmZlYuWZ4+miBpNPDvqWhKRNxZ3LDMzKwcMj19FBF/AP5Q5FjMzKzM3DdgZmZ5TgpmZpZXZ1KQNCt9/qh04ZiZWTnV16fQTdInyM2edgugwo0RsaiokZmZWcnVlxQuBv6b3JSZV22xLchNr2lmZm1InUkhIm4Hbpf03xHx/RLGZGZmZZLlPYXvSzoJGJKKZkfEn4sblpmZlUOWAfF+CJwPPJO+zk9TdJqZWRuT5eW1EUC/iNgIIGkqueGzv1vMwMzMrPSyvqewe8Fy52IEYmZm5ZclKfwQeEzSb9JdwkLg8oZ2ktRJ0qOSHpf0tKTLUnlPSY9IWiZpmqSOqXzHtL4sbe/R9MsyM7OmaDApRMTvgUHkxj66AzgyIqZlOPZ7wDER0RfoBxwvaRDwI+DqiPg34A3gjFT/DOCNVH51qmdmZiWUqfkoIlZFxN3p65WM+0RErE2rHdg0necxbJqgZypwcloeldZJ24dJ2uyFOTMzK66ijn0kqZ2kxcBrwEzgeeDNiKhOVVYA+6TlfYCXAdL2NcCetRzzLEkLJC2oqqoqZvhmZtudoiaFiNgQEf3IvRU9EDi4GY45JSIqI6KyoqJim2M0M7NN6k0K6S/9Z7f1JBHxJvAgcCSwu6SaR2G7AyvT8kpg33Te9uSeclq9rec2M7Ps6k0KEbEBeE7Sfo09sKQKSbun5Z2ATwFLyCWHManaOOCutHx3WidtfyAiorHnNTOzpsvy8toewNOSHgXW1RRGxEkN7NcNmCqpHbnkc2tE/FnSM8Atkn5A7iW4X6b6vwR+K2kZ8DpweuMuxczMtlWWpPDfTTlwRDwBHF5L+Qvk+he2LF8PnNqUc5mZWfPIMiDeQ5L2Bw6IiPsl7Qy0K35oZmZWalkGxPsKufcGfp6K9gH+WMygzMysPLI8knoeMBh4CyAilgJ7FTMoMzMrjyxJ4b2IeL9mJT0u6qeCzMzaoCxJ4SFJ3wV2kvQp4DbgT8UNy8zMyiFLUpgIVAFPAmcD04GLihmUmZmVR5anjzamIbMfIdds9JxfKjMza5saTAqSRgCTyQ1mJ6CnpLMj4t5iB2fWWvWYeE9JzrN80oiSnMe2H1leXvsJ8MmIWAYg6WPAPYCTgplZG5OlT+HtmoSQvAC8XaR4zMysjOq8U5A0Oi0ukDQduJVcn8KpwPwSxGZmZiVWX/PRiQXLrwJD03IVsFPRIjIzs7KpMylExJdKGYiZmZVflqePegJfA3oU1s8wdLaZmbUyWZ4++iO5uQ7+BGwsbjhmZlZOWZLC+oi4ruiRmJlZ2WVJCtdKugSYAbxXUxgRi4oWlZmZlUWWpNAb+CJwDJuajyKtm5lZG5IlKZwKfLRw+GwzM2ubsrzR/BSwe7EDMTOz8styp7A78Kyk+Wzep+BHUs3M2pgsSeGSphxY0r7ATUBXcn0QUyLiWkldgGnk3ntYDpwWEW9IEnAt8GngHWC8O7PNzEory3wKDzXx2NXAtyJikaTdgIWSZgLjgVkRMUnSRHKT+FwInAAckL6OAG5In2ZmViIN9ilIelvSW+lrvaQNkt5qaL+IWFXzl35EvA0sAfYBRgFTU7WpwMlpeRRwU+TMA3aX1K0J12RmZk2U5U5ht5rl1MQzChjUmJNI6gEcTm72tq4RsSpteoVc8xLkEsbLBbutSGWrMDOzksjy9FFe+iv+j8BxWfeRtCtwB/D1iNjsDiNN69moqT0lnSVpgaQFVVVVjdnVzMwakGVAvNEFqzsAlcD6LAeX1IFcQrg5Iv6Qil+V1C0iVqXmoddS+Upg34Ldu6eyzUTEFGAKQGVlpeeKNjNrRlnuFE4s+DqO3KxroxraKTU1/RJYEhFXFWy6GxiXlscBdxWUj1XOIGBNQTOTmZmVQJY+habOqzCY3PAYT0panMq+C0wCbpV0BvAScFraNp3c46jLyD2S6vkczMxKrL7pOC+uZ7+IiO/Xd+CI+CugOjYPq+2AwHn1HdPMzIqrvjuFdbWU7QKcAewJ1JsUzMys9alvOs6f1Cynl8/OJ9ekcwvwk7r2MzOz1qvePoU0JMU3gc+Te9Gsf0S8UYrAzMys9OrrU/gxMJrc45+9I2JtyaIyM7OyqO+R1G8BewMXAf8sGOri7SzDXJiZWetTX59Co952NjOz1s+/+M3MLM9JwczM8pwUzMwsz0nBzMzynBTMzCzPScHMzPKcFMzMLK/BobPN2poeE+8pdwhmLZbvFMzMLM9JwczM8pwUzMwsz0nBzMzynBTMzCzPScHMzPL8SKpZK1aKx2uXTxpR9HNYy1G0OwVJv5L0mqSnCsq6SJopaWn63COVS9J1kpZJekJS/2LFZWZmdStm89FvgOO3KJsIzIqIA4BZaR3gBOCA9HUWcEMR4zIzszoULSlExBzg9S2KRwFT0/JU4OSC8psiZx6wu6RuxYrNzMxqV+qO5q4RsSotvwJ0Tcv7AC8X1FuRyszMrITK9vRRRAQQjd1P0lmSFkhaUFVVVYTIzMy2X6VOCq/WNAulz9dS+Upg34J63VPZViJiSkRURkRlRUVFUYM1M9velPqR1LuBccCk9HlXQflXJd0CHAGsKWhmMrMyakujyvrx2oYVLSlI+j1wNPBhSSuAS8glg1slnQG8BJyWqk8HPg0sA94BvlSsuMzMrG5FSwoR8bk6Ng2rpW4A5xUrFjMzy8ZvNFuL0paaKsxaI499ZGZmeU4KZmaW56RgZmZ5TgpmZpbnjmYz2274QYaG+U7BzMzynBTMzCzPScHMzPLcp2BWBMs7/Ue5Q2g2Pdb/T7lDsBLynYKZmeU5KZiZWZ6TgpmZ5TkpmJlZnjuazcyaUekeMlhTlKP6TsHMzPKcFMzMLM9JwczM8tynYNudtvRiWSmU6vvll+Rahu02KbT2zqBS8+iSZtsHNx+ZmVnednunYC2Tm3bMyqtFJQVJxwPXAu2AGyNiUplDMrMSKcUfBO63aFiLSQqS2gE/Az4FrADmS7o7Ip4pb2Qtn9v7zay5tJikAAwElkXECwCSbgFGAa06KZTiF7abXMysubSkpLAP8HLB+grgiC0rSToLOCutrpX0XBPP92HgX03ctxFGFv0MKtm1lERbuZa2ch3Qpq5lZNGvRcU8eKHLtC3Xsn9dG1pSUsgkIqYAU7b1OJIWRERlM4RUdr6WlqetXAf4WlqqYl1LS3okdSWwb8F691RmZmYl0pKSwnzgAEk9JXUETgfuLnNMZmbblRbTfBQR1ZK+CvyF3COpv4qIp4t4ym1ugmpBfC0tT1u5DvC1tFRFuRZFRDGOa2ZmrVBLaj4yM7Myc1IwM7O87S4pSDpV0tOSNkqqLCj/lKSFkp5Mn8eUM84s6rqWtO07kpZJek7SceWKsSkk9ZM0T9JiSQskDSx3TE0l6WuSnk3/TleUO55tJelbkkLSh8sdS1NJ+nH6N3lC0p2Sdi93TI0h6fj0c71M0sTmPv52lxSAp4DRwJwtyv8FnBgRvYFxwG9LHVgT1HotknqRe3rrUOB44P+lYURaiyuAyyKiH3BxWm91JH2S3Fv5fSPiUODKMoe0TSTtCwwH/lHuWLbRTOCwiOgD/B34TpnjyaxgOKATgF7A59LPe7PZ7pJCRCyJiK3ego6IxyLin2n1aWAnSTuWNrrGqetayP0iuiUi3ouIF4Fl5IYRaS0C+FBa7gz8s566Ldm5wKSIeA8gIl4rczzb6mrgAnL/Pq1WRMyIiOq0Oo/cO1GtRX44oIh4H6gZDqjZbHdJIaPPAItqfphbodqGDNmnTLE0xdeBH0t6mdxf163mL7ktHAgcJekRSQ9JGlDugJpK0ihgZUQ8Xu5YmtmXgXvLHUQjFP1nu8W8p9CcJN0PfKSWTd+LiLsa2PdQ4EfkbpPLbluupSWr77qAYcA3IuIOSacBvwSOLWV8WTVwHe2BLsAgYABwq6SPRgt9DryBa/kuLeRnIossPzeSvgdUAzeXMraWrk0mhYho0i8QSd2BO4GxEfF880bVNE28lhY/ZEh91yXpJuD8tHobcGNJgmqCBq7jXOAPKQk8KmkjucHlqkoVX2PUdS2SegM9gcclQe7/0yJJAyPilRKGmFlDPzeSxpMbrXJYS03SdSj6z7abj5L0BMI9wMSIeLjc8Wyju4HTJe0oqSdwAPBomWNqjH8CQ9PyMcDSMsayLf4IfBJA0oFAR1rhaKMR8WRE7BURPSKiB7kmi/4tNSE0JE3mdQFwUkS8U+54GqnowwFtd280SzoFuB6oAN4EFkfEcZIuItd2XfgLaHhL7hys61rStu+Ray+tBr4eEa2m3VTSv5Obga89sB74z4hYWN6oGi/90P4K6Ae8D3w7Ih4ob1TbTiRVTS8AAARKSURBVNJyoDIiWl2CA5C0DNgRWJ2K5kXEOWUMqVEkfRq4hk3DAV3erMff3pKCmZnVzc1HZmaW56RgZmZ5TgpmZpbnpGBmZnlOCmZmluekYCUhae0W6+Ml/bRc8TQnSQenEV0fk/SxcsfTHCTN3nLkXds+OCmYZSCpvrf/TwZuj4jDs74J38pGrbXtiJOClZ2k30gaU7C+Nn0enQaSu0vSC5ImSfq8pEfTvBcfS/VOTIPOPSbpfkldU/mlkn6V/up9QdJ/1XH+tZKuTnMezJJUkcpnS7pG0gLgfEkfT/EslPQXSd3Si0RfB86V9GDa7wspxsWSfl6TANJ5fiLpceDIBupdLulx5eaVqLmermn8/8fT1yfqO1/B9R0v6baC9aMl/Tkt36DcnBVPS7qsru9PwfIYSb9JyxWS7pA0P30NzvhPbi2Yk4KVyk7pl9ZiSYuB/5Nxv77AOcAhwBeBAyNiILnxkL6W6vwVGBQRh5MbSviCgv0PBo4jN+TwJZI61HKOXYAFac6Dh4BLCrZ1jIhK4Dpyb4+PiYiPk3tT+fKImA5MBq6OiE9KOgT4LDA4zQexAfh8wXkeiYi+5N6mra/evFRvDvCVVH4d8FAq7w883cD5atwPHCFpl7T+2fR9gtwAcZVAH2CopD61fH/qcm267gHkRhZusWNUWXZtckA8a5HeTb+0gPyAZFnarOdHxKq0z/PAjFT+JGlcIXKDgk2T1I3c+EIvFux/TxoC/T1JrwFdyY3dU2gjMC0t/w74Q8G2mvKDgMOAmWlQuHbAqlriHQZ8HJif6u0E1AyVsgG4I0O994E/p+WFwKfS8jHAWICI2ACskfTFeo5Dqlst6T7gREm3AyPYlDhPk3QWud8F3chN3PJELddVm2OBXum8AB+StGtErK1nH2vhnBSsJagm3bVK2oHcL/YahXNabCxY38im/7/XA1dFxN2SjgYurWP/DWT7P1849su69Cng6Yg4soF9BUyNiNrmgFiffpk3VO+DgpE7G4q5vuMUugX4KvA6ubuit5UbLPHbwICIeCM1C3WqZd/C70fh9h3I3aGtb+Dc1oq4+chaguXk/toFOAmorYmnPp3ZNHzwuCacfwegpk/jP8g1R23pOaBC0pEAkjooN/fGlmYBYyTtlep1kbT/NtTbcp9zU/12kjo34jgPkWty+gqbmo4+RC7prUn9FifUcd5XJR2SEvYpBeUz2NSEh6R+W+1prY6TgrUEvyDXnv04cCSb/jrP6lLgNkkLadrQ1OuAgZKeItdEs1V/R5r6cAzwoxTnYuATtdR7BrgImCHpCXLzAXdrar0tnA98UtKT5JqVejXifBvINUmdkD5Js6g9BjwL/A9Q15DxE9M+f2PzJrP/AiolPSHpGXJ9P9bKeZRU2+5JWhsRu5Y7DrOWwHcKZmaW5zsFMzPL852CmZnlOSmYmVmek4KZmeU5KZiZWZ6TgpmZ5f1/+bDGLpYkoBAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "agIR2QfnvQOO",
        "colab_type": "text"
      },
      "source": [
        "## Experiment (2) - Train on all possible concatenations of dense and conv activations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MdlT1OMesos7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "conv_acts_flat = conv_acts.reshape((23750, 12 * 14 * 16))\n",
        "\n",
        "dens_acts_ch16 = dens_acts.reshape((23750, 2, 2, 16))\n",
        "dens_act_pad_ch16 = np.zeros((23750, 12, 14, 16))\n",
        "dens_act_pad_ch16[:dens_acts_ch16.shape[0], :dens_acts_ch16.shape[1],\n",
        "                  :dens_acts_ch16.shape[2], :dens_acts_ch16.shape[3]] = dens_acts_ch16\n",
        "\n",
        "dens_acts_ch64 = dens_acts.reshape((23750, 1, 1, 64))\n",
        "dens_act_pad_ch64 = np.zeros((23750, 12, 14, 64))\n",
        "dens_act_pad_ch64[:dens_acts_ch64.shape[0], :dens_acts_ch64.shape[1],\n",
        "                  :dens_acts_ch64.shape[2], :dens_acts_ch64.shape[3]] = dens_acts_ch64\n",
        "\n",
        "all_acts_flat = np.concatenate([conv_acts_flat, dens_acts], axis=1)\n",
        "conv_dens_act_conc_on_x1 = np.concatenate([conv_acts, dens_act_pad_ch16], axis=1)\n",
        "conv_dens_act_conc_on_x2 = np.concatenate([conv_acts, dens_act_pad_ch16], axis=2)\n",
        "conv_dens_act_conc_on_x3 = np.concatenate([conv_acts, dens_act_pad_ch16], axis=3)\n",
        "conv_dens_act_conc_on_ch = np.concatenate([conv_acts, dens_act_pad_ch64], axis=3)\n",
        "\n",
        "del conv_acts, dens_acts, conv_acts_flat, dens_acts_ch16, dens_act_pad_ch16, dens_acts_ch64, dens_act_pad_ch64"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "afCE8P-XoXj1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "cb550c0e-fd9b-4908-d14d-47980cbe9aa2"
      },
      "source": [
        "# Experiment No. 2\n",
        "\n",
        "for xs in [\n",
        "    dens_acts,\n",
        "    all_acts_flat,\n",
        "    conv_acts,\n",
        "    conv_dens_act_conc_on_x1, \n",
        "    conv_dens_act_conc_on_x2,\n",
        "    conv_dens_act_conc_on_x3,\n",
        "    conv_dens_act_conc_on_ch]:\n",
        "\n",
        "    if len(xs.shape) == 2:\n",
        "        hparam = {\"reg_amount\": 0.0, \"drop_rate\": 0.0, \"layer_sizes\": (32,)}\n",
        "        mod = dnn\n",
        "    else:\n",
        "        hparam = {\"reg_amount\": 0.0, \"drop_rate\": 0.0, \"xs_shape\": xs.shape}\n",
        "        mod = cnn\n",
        "    \n",
        "    ys_val_hat, ys_val, pref_val = train_eval(hparam, mod, xs, ys, prefs, num_train=50, num_val=400, epochs=400)\n",
        "    error_analysis_plot(ys_val_hat, ys_val, pref_val)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: <function dnn at 0x7fcdf56a37b8>\n",
            "Shape of input: (23750, 64)\n",
            "Hyperparameters: {'reg_amount': 0.0, 'drop_rate': 0.0, 'layer_sizes': (32,)}\n",
            "Number of training data: 50\n",
            "Number of validation data: 400\n",
            "Number of epochs: 400\n",
            "Doing training iteration no. 1\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "dense (Dense)                multiple                  2080      \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            multiple                  0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              multiple                  33        \n",
            "=================================================================\n",
            "Total params: 2,113\n",
            "Trainable params: 2,113\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Best train accuracy: 1.0\n",
            "Number of epochs: 31\n",
            "Doing training iteration no. 2\n",
            "Doing training iteration no. 3\n",
            "Doing training iteration no. 4\n",
            "Doing training iteration no. 5\n",
            "Doing training iteration no. 6\n",
            "Doing training iteration no. 7\n",
            "Doing training iteration no. 8\n",
            "Doing training iteration no. 9\n",
            "Doing training iteration no. 10\n",
            "Results ('mean_val_auc', 0.8714653313159942, 'mean_val_accuracy', 0.8435000061988831, 'metric', 0.8574826687574386, 'val_auc_std', 0.02263668332293567, 'val_accuracy_std', 0.02372235332354643)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5yVZbn/8c+Xk4AHFEVSUSHzhJykAVG2YGKIgaiIptsUSlNM25aJYrk9/MqdKalpbonMxHQnHjI1UUGSQxpyEk+ggYqJoo54BEUduH5/PM8sFjCHNYe11szwfb9e81rP+b6egZlrnvu+n/tWRGBmZgbQrNgBmJlZw+GkYGZmGU4KZmaW4aRgZmYZTgpmZpbRotgB1MVOO+0UnTt3LnYYZmaNyoIFC96LiA4V7WvUSaFz587Mnz+/2GGYmTUqkl6vbJ+rj8zMLMNJwczMMpwUzMwsI29tCpJuBYYB70ZEt032/QQYD3SIiPckCfgN8C3gU2B0RCzMV2xmtrkvv/ySFStWsHbt2mKHYvWkdevWdOrUiZYtW+Z8Tj4bmm8Dfgvcnr1R0u7AYODfWZuPAvZOvw4Cbk4/zaxAVqxYwbbbbkvnzp1J/k6zxiwiWLVqFStWrKBLly45n5e36qOImAW8X8Gu64ALgeyR+I4Bbo/EHGB7SbvkKzYz29zatWvZcccdnRCaCEnsuOOONX7yK2ibgqRjgDcj4tlNdu0GvJG1viLdVtE1zpQ0X9L80tLSPEVqtmVyQmhaavPvWbCkIKkt8FPg0rpcJyImRkRJRJR06FDhuxdmZlZLhXx5bS+gC/Bsmr06AQsl9QXeBHbPOrZTus3MiqTzuIfr9XrLrxpa5f5Vq1YxaNAgAN5++22aN29O+R9+c+fOpVWrVvUaTy4OO+wwxo8fT0lJScHLLpaCJYWIeB7YuXxd0nKgJO199CBwrqS7SBqYP4qIlfmMp77/w2+quh8AM9vYjjvuyKJFiwC4/PLL2Wabbbjgggsy+8vKymjRolEPwtAo5K36SNKfgX8C+0paIen0Kg6fArwKLAN+D/wgX3GZWeMxevRoxowZw0EHHcSFF17I5Zdfzvjx4zP7u3XrxvLlywG444476Nu3L7169eKss85i3bp1G13r0Ucf5YQTTsisz5gxg2HDhgFw9tlnU1JSwgEHHMBll11WYSzbbLNNZvnee+9l9OjRAJSWlnL88cfTp08f+vTpw5NPPgnAzJkz6dWrF7169eLAAw/kk08+qfP3oxDy2fvo5IjYJSJaRkSniPjDJvs7R8R76XJExDkRsVdEdI8ID2hkZkDSVfapp57i2muvrfSYJUuWMHnyZJ588kkWLVpE8+bNufPOOzc65ogjjuDpp59mzZo1AEyePJmTTjoJgCuvvJL58+fz3HPPMXPmTJ577rmc4zvvvPP48Y9/zLx587jvvvs444wzABg/fjw33XQTixYtYvbs2bRp06amt14UfhYzswbthBNOoHnz5lUeM336dBYsWECfPn0A+Oyzz9h55503OqZFixYMGTKEhx56iJEjR/Lwww9z9dVXA3D33XczceJEysrKWLlyJYsXL6ZHjx45xff444+zePHizPrHH3/M6tWr6d+/P+effz6nnHIKI0aMoFOnTjW57aJxUjCzBm3rrbfOLLdo0YL169dn1sv74EcEo0aN4pe//GWV1zrppJP47W9/S/v27SkpKWHbbbfltddeY/z48cybN48ddtiB0aNHV9i3P7t7Z/b+9evXM2fOHFq3br3R8ePGjWPo0KFMmTKF/v3789hjj7HffvvV7OaLwGMfmVmj0blzZxYuTEbAWbhwIa+99hoAgwYN4t577+Xdd98F4P333+f11zcfHXrgwIEsXLiQ3//+95mqo48//pitt96adu3a8c477/DII49UWHbHjh1ZsmQJ69ev5/77789sHzx4MDfeeGNmvbyx/JVXXqF79+5cdNFF9OnTh5deeqkevgP55ycFM6tQQ+xBd/zxx3P77bdzwAEHcNBBB7HPPvsA0LVrV37xi18wePBg1q9fT8uWLbnpppvYc889Nzq/efPmDBs2jNtuu41JkyYB0LNnTw488ED2228/dt99d/r3719h2VdddRXDhg2jQ4cOlJSUsHr1agBuuOEGzjnnHHr06EFZWRkDBgxgwoQJXH/99TzxxBM0a9aMAw44gKOOOiqP35n6o4io/qgGqqSkJGo7yY67pJptbMmSJey///7FDsPqWUX/rpIWRESFL1+4+sjMzDKcFMzMLMNJwczMMpwUzMwsw0nBzMwynBTMzCzD7ymYWcUub1fP1/uo2kOaN29O9+7dKSsrY//992fSpEm0bdu2VsWNHj2aYcOGMXLkSM444wzOP/98unbtWuGxM2bMoFWrVhxyyCEATJgwgbZt23LaaafVquzGzE8KZtZgtGnThkWLFvHCCy/QqlUrJkyYsNH+srKyWl33lltuqTQhQJIUnnrqqcz6mDFjCpYQImKjoTs2Xa9Mbb8X1XFSMLMG6dBDD2XZsmXMmDGDQw89lOHDh9O1a1fWrVvH2LFj6dOnDz169OB3v/sdkPwyPffcc9l333054ogjMkNeQDJZTvmLro8++ii9e/emZ8+eDBo0iOXLlzNhwgSuu+46evXqxezZszcaonvRokX069ePHj16cNxxx/HBBx9krnnRRRfRt29f9tlnH2bPnl3hfVxzzTWZWMuH5V6+fDn77rsvp512Gt26dWP27Nkbrb/xxhuMHTuWbt260b17dyZPngyw2fdizZo1DB06lJ49e9KtW7fMcXXh6iMza3DKysp45JFHGDJkCJCMc/TCCy/QpUsXJk6cSLt27Zg3bx6ff/45/fv3Z/DgwTzzzDO8/PLLLF68mHfeeYeuXbvyve99b6PrlpaW8v3vf59Zs2bRpUsX3n//fdq3b8+YMWM2mtRn+vTpmXNOO+00brzxRgYOHMill17KFVdcwfXXX5+Jc+7cuUyZMoUrrriCxx9/fKPypk6dytKlS5k7dy4RwfDhw5k1axZ77LEHS5cuZdKkSfTr14/ly5dvtH7fffexaNEinn32Wd577z369OnDgAEDNvte3Hfffey66648/HAyQsNHH1VfRVcdPymYWYPx2Wef0atXL0pKSthjjz04/fRkbq6+ffvSpUsXIPlFe/vtt9OrVy8OOuggVq1axdKlS5k1axYnn3wyzZs3Z9ddd+Xwww/f7Ppz5sxhwIABmWu1b9++yng++ugjPvzwQwYOHAjAqFGjmDVrVmb/iBEjAPj617+emewn29SpU5k6dSoHHnggvXv35qWXXmLp0qUA7LnnnvTr1y9zbPb6P/7xj8y9dOzYkYEDBzJv3rzNvhfdu3dn2rRpXHTRRcyePZt27ereDuQnBTNrMMrbFDaVPXx2RHDjjTdy5JFHbnTMlClT8h7fprbaaisgaSCvqI4/Irj44os566yzNtq+fPnyje4J2Gy9MtnH7bPPPixcuJApU6ZwySWXMGjQIC699NKa3sZG/KRgZo3KkUceyc0338yXX34JwL/+9S/WrFnDgAEDmDx5MuvWrWPlypU88cQTm53br18/Zs2alRly+/333wdg2223rXC6zHbt2rHDDjtk2gv+9Kc/ZZ4aco311ltvzYyo+uabb27U1lGZQw89NHMvpaWlzJo1i759+2523FtvvUXbtm35zne+w9ixYzPDiteFnxTMrGI5dCEthjPOOIPly5fTu3dvIoIOHTrw17/+leOOO46///3vdO3alT322IODDz54s3M7dOjAxIkTGTFiBOvXr2fnnXdm2rRpHH300YwcOZIHHnhgo7kRACZNmsSYMWP49NNP+epXv8of//jHnGMdPHgwS5YsycSyzTbbcMcdd1Q7k9xxxx3HP//5T3r27Ikkrr76ar7yla9sNifD888/z9ixY2nWrBktW7bk5ptvzjm2ynjo7Dzx0NnW2Hjo7KbJQ2ebmVmt5S0pSLpV0ruSXsjado2klyQ9J+l+Sdtn7btY0jJJL0s6suKrmplZPuXzSeE2YMgm26YB3SKiB/Av4GIASV2Bk4AD0nP+V1LVlW5mVu8ac3Wyba42/555SwoRMQt4f5NtUyOivN/WHKBTunwMcFdEfB4RrwHLgM2b2s0sb1q3bs2qVaucGJqIiGDVqlW0bt26RucVs/fR94Dyd7J3I0kS5Vak2zYj6UzgTIA99tgjn/GZbVE6derEihUrKC0tLXYoVk9at25Np06dqj8wS1GSgqSfAWXAnTU9NyImAhMh6X1Uz6GZbbFatmyZeVPWtlwFTwqSRgPDgEGx4Tn1TWD3rMM6pdvMzKyACtolVdIQ4EJgeER8mrXrQeAkSVtJ6gLsDcwtZGxmZpbHJwVJfwYOA3aStAK4jKS30VbANEkAcyJiTES8KOluYDFJtdI5EbEuX7GZmVnF8pYUIuLkCjb/oYrjrwSuzFc8ZmZWPb/RbGZmGU4KZmaW4aRgZmYZTgpmZpbhpGBmZhlOCmZmluGkYGZmGU4KZmaW4aRgZmYZ1SYFSVdL2k5SS0nTJZVK+k4hgjMzs8LK5UlhcER8TDKy6XLga8DYfAZlZmbFkUtSKB8faShwT0R8lMd4zMysiHIZEO9vkl4CPgPOltQBWJvfsMzMrBiqfVKIiHHAIUBJRHwJfEoyp7KZmTUxuTQ0twV+ANycbtoVKMlnUGZmVhy5tCn8EfiC5GkBkmkyf5G3iMzMrGhySQp7RcTVwJcA6TSaymtUZmZWFLkkhS8ktQECQNJewOd5jcrMzIoil95HlwGPArtLuhPoD4zOZ1BmZlYc1SaFiJgmaSHQj6Ta6LyIeC/vkZmZWcHl0vvoOKAsIh6OiL8BZZKOzX9oZmZWaLm0KVyW/RZzRHxIUqVUJUm3SnpX0gtZ29pLmiZpafq5Q7pdkm6QtEzSc5J61+ZmzMysbnJJChUdk0tbxG3AkE22jQOmR8TewPR0HeAoYO/060w2vBNhZmYFlEtSmC/pWkl7pV/XAguqOykiZgHvb7L5GGBSujwJODZr++2RmANsL2mX3G7BzMzqSy5J4YckL69NTr8+B86pZXkdI2Jluvw20DFd3g14I+u4Fem2zUg6U9J8SfNLS0trGYaZmVUkl95Ha9hQzVNvIiIkRS3OmwhMBCgpKanx+WZmVrlqk4KkfYALgM7Zx0fE4bUo7x1Ju0TEyrR66N10+5vA7lnHdUq3mZlZAeXSYHwPMAG4BVhXx/IeBEYBV6WfD2RtP1fSXcBBwEdZ1UxmZlYguSSFsoiocW8gSX8GDgN2krSCpBvrVcDdkk4HXgdOTA+fAnwLWEYyNPd3a1qemZnVXS5J4SFJPwDuJ2vMo4jYtGfRRiLi5Ep2Darg2KD2jddmZlZPckkKo9LP7HmZA/hq/YdjZmbFlEvvoy6FCMTMzIovp5nXJF0iaWK6vrekYfkPzczMCs0zr5mZWYZnXjMzswzPvGZmZhmeec3MzDKqTAqSmgE7ACPwzGtmZk1elUkhItZLujAi7gYeLlBMZmZWJLm0KTwu6QJJu6czp7WX1D7vkZmZWcHl0qbw7fQzexgKv9FsZtYE5dKmMC4iJhcoHjMzK6Iqq48iYj0bj3lkZmZNmNsUzMwsw20KZmaW4VFSzcwsI5c5mk+raHtE3F7/4ZiZWTHlUn3UJ2u5NcnMaQsBJwUzsyYml+qjH2avS9oeuCtvEZmZWdHk0vtoU2sAtzOYmTVBubQpPEQ6bDZJEukK3J3PoMzMrDhyaVMYn7VcBrweESvqUqikHwNnkCSb54HvAruQVEvtCCwATo2IL+pSjpmZ1Uwu1Uf/Bp6OiJkR8SSwSlLn2hYoaTfgv4CSiOgGNAdOAn4FXBcRXwM+AE6vbRlmZlY7uSSFe4D1Wevr0m110QJoI6kF0BZYCRwO3JvunwQcW8cyzMyshnJJCi2yq3HS5Va1LTAi3iSpkvo3STL4iKS66MOIKEsPWwHsVtH5ks6UNF/S/NLS0tqGYWZmFcglKZRKGl6+IukYoNYzr0naATiGpAfTrsDWwJBcz4+IiRFREhElHTp0qG0YZmZWgVwamscAd0r6bbq+AqjwLeccHQG8FhGlAJL+QjLv8/aSWqRPC52AN+tQhpmZ1UIuL6+9AvSTtE26vrqOZf47vV5b4DOSN6TnA08AI0l6II0CHqhjOWZmVkPVVh9J+h9J20fE6ohYLWkHSb+obYER8TRJg/JCku6ozYCJwEXA+ZKWkXRL/UNtyzAzs9rJpU3hqIj4sHwlIj4AvlWXQiPisojYLyK6RcSpEfF5RLwaEX0j4msRcUJEfF6XMszMrOZySQrNJW1VviKpDbBVFcebmVkjlUtD853AdEl/TNe/S/IegZmZNTG5NDT/StKzJL2GAH4eEY/lNywzMyuGXJ4UAJ4BWpKMVfRM/sIxM7NiyqX30YnAXJLuoicCT0same/AzMys8HJ5UvgZ0Cci3gWQ1AF4nA3jFJmZWRORS++jZuUJIbUqx/PMzKyRyeVJ4VFJjwF/Tte/DUzJX0hmZlYsufQ+GitpBPAf6aaJEXF/fsMyM7NiyKn3UUT8BfhLnmMxM7Mic9uAmZllOCmYmVlGpUlB0vT081eFC8fMzIqpqjaFXSQdAgyXdBeg7J0RsTCvkZmZWcFVlRQuBf6bZBa0azfZF8Dh+QrKzMyKo9KkEBH3AvdK+u+I+HkBYzIzsyLJ5T2Fn0saDgxIN82IiL/lNywzMyuGXAbE+yVwHrA4/TpP0v/kOzAzMyu8XF5eGwr0ioj1AJImkQyf/dN8BmZmZoWX63wK2wPvp8vt8hSLmVmT0Xncw3m9/vKrhublurkkhV8Cz0h6gqRb6gBgXF6iMTOzosqlofnPkmYAfdJNF0XE23UpVNL2wC1AN5Lurd8DXgYmA52B5cCJEfFBXcoxM7OayWmYi4hYGREPpl91Sgip3wCPRsR+QE9gCcnTx/SI2BuYjp9GzMwKruBjH0lqR1IF9QeAiPgiIj4EjgEmpYdNAo4tdGxmZlu6YgyI1wUoBf4o6RlJt0jaGugYESvTY94GOlZ0sqQzJc2XNL+0tLRAIZuZbRmqTAqSmkt6qZ7LbAH0Bm6OiAOBNWxSVRQRQdLWsJmImBgRJRFR0qFDh3oOzcxsy1ZlUoiIdcDLkvaoxzJXACsi4ul0/V6SJPGOpF0A0s93KznfzMzyJJcuqTsAL0qaS/JXPQARMbw2BUbE25LekLRvRLwMDGLD29KjgKvSzwdqc30zM6u9XJLCf+eh3B8Cd0pqBbwKfJfkqeVuSacDrwMn5qFcMzOrQi7vKcyUtCewd0Q8Lqkt0LwuhUbEIqCkgl2D6nJdMzOrm2qTgqTvA2cC7YG9gN2ACfgXuG1h8jlsQb6GLDCrqVy6pJ4D9Ac+BoiIpcDO+QzKzMyKI5ek8HlEfFG+IqkFlXQXNTOzxi2XpDBT0k+BNpK+CdwDPJTfsMzMrBhySQrjSN5Afh44C5gCXJLPoMzMrDhy6X20Pp1Y52mSaqOX0zeOzcysicml99FQkt5Gr5DMp9BF0lkR8Ui+gzMzs8LK5eW1XwPfiIhlAJL2Ah4GnBTMzJqYXNoUPilPCKlXgU/yFI+ZmRVRpU8Kkkaki/MlTQHuJmlTOAGYV4DYzMyswKqqPjo6a/kdYGC6XAq0yVtEZmZWNJUmhYj4biEDMTOz4sul91EXklFNO2cfX9uhs83MrOHKpffRX0nmU34IWJ/fcMzMrJhySQprI+KGvEdiZmZFl0tS+I2ky4CpwOflGyNiYd6iMjOzosglKXQHTgUOZ0P1UaTrZmbWhOSSFE4Avpo9fLaZmTVNubzR/AKwfb4DMTOz4svlSWF74CVJ89i4TcFdUs3MmphcksJleY/CzMwahFzmU5iZj4IlNQfmA29GxLD0Jbm7gB2BBcCpbscwMyusatsUJH0i6eP0a62kdZI+roeyzwOWZK3/CrguIr4GfACcXg9lmJlZDVSbFCJi24jYLiK2IxkI73jgf+tSqKROwFDglnRdJF1c700PmQQcW5cyzMys5nLpfZQRib8CR9ax3OuBC9nw3sOOwIcRUZaurwB2q+hESWdKmi9pfmlpaR3DMDOzbLkMiDcia7UZUAKsrW2BkoYB70bEAkmH1fT8iJgITAQoKSnxXNFmZvUol95H2fMqlAHLgWPqUGZ/YLikbwGtge2A3wDbS2qRPi10At6sQxlmZlYLufQ+qtd5FSLiYuBigPRJ4YKIOEXSPcBIkh5Io4AH6rNcMzOrXlXTcV5axXkRET+v51guAu6S9AvgGZLhus3MrICqelJYU8G2rUm6iu4I1DkpRMQMYEa6/CrQt67XNDOz2qtqOs5fly9L2pbkvYLvklTv/Lqy88zMrPGqsk1BUnvgfOAUkncHekfEB4UIzMzMCq+qNoVrgBEk3T+7R8TqgkVlZmZFUdXLaz8BdgUuAd7KGurik3oa5sLMzBqYqtoUavS2s5mZNX7+xW9mZhlOCmZmluGkYGZmGU4KZmaW4aRgZmYZuYySatYodB73cLFDqLV8x778qqF5vb41HX5SMDOzDCcFMzPLcFIwM7MMJwUzM8twUjAzswwnBTMzy3BSMDOzDCcFMzPLcFIwM7MMJwUzM8soeFKQtLukJyQtlvSipPPS7e0lTZO0NP3codCxmZlt6Yox9lEZ8JOIWChpW2CBpGnAaGB6RFwlaRwwDrioCPGZ1cry1v9Z63M7r/2/eozErPYK/qQQESsjYmG6/AmwBNgNOAaYlB42CTi20LGZmW3pitqmIKkzcCDwNNAxIlamu94GOlZyzpmS5kuaX1paWpA4zcy2FEUbOlvSNsB9wI8i4mNJmX0REZKiovMiYiIwEaCkpKTCY8zMctGYh1vPl6I8KUhqSZIQ7oyIv6Sb35G0S7p/F+DdYsRmZrYlK/iTgpJHgj8ASyLi2qxdDwKjgKvSzwcKHZtZU5XPv4g9gU/TUozqo/7AqcDzkhal235KkgzulnQ68DpwYhFiM7MachVM01LwpBAR/wBUye5BhYzFzMw25jeazcwsw0nBzMwynBTMzCzDScHMzDKK9vKamW1Ql3GTwGMnWf3xk4KZmWU4KZiZWYaTgpmZZTgpmJlZhhuarclwY61Z3TkpWEHldWC21nm7tFm9q+sfMfBRvcSxKVcfmZlZhpOCmZllOCmYmVmGk4KZmWU4KZiZWYaTgpmZZTgpmJlZht9TsM14zl2zLZeTgpnV6UUqvwnetLj6yMzMMhpcUpA0RNLLkpZJGlfseMzMtiQNqvpIUnPgJuCbwApgnqQHI2JxcSNrWJpynX/dx4OxxsZVVw1LQ3tS6Assi4hXI+IL4C7gmCLHZGa2xVBEFDuGDEkjgSERcUa6fipwUEScm3XMmcCZ6eq+wMu1LG4n4L06hNuQ+F4apqZyL03lPsD3Um7PiOhQ0Y4GVX2Ui4iYCEys63UkzY+IknoIqeh8Lw1TU7mXpnIf4HvJRUOrPnoT2D1rvVO6zczMCqChJYV5wN6SukhqBZwEPFjkmMzMthgNqvooIsoknQs8BjQHbo2IF/NUXJ2roBoQ30vD1FTupancB/heqtWgGprNzKy4Glr1kZmZFZGTgpmZZWxxSUHSCZJelLReUknW9m9KWiDp+fTz8GLGmYvK7iXdd3E6VMjLko4sVoy1IamXpDmSFkmaL6lvsWOqLUk/lPRS+u90dbHjqStJP5EUknYqdiy1Jema9N/kOUn3S9q+2DHVRL6HAtrikgLwAjACmLXJ9veAoyOiOzAK+FOhA6uFCu9FUleSnlsHAEOA/02HEGksrgauiIhewKXpeqMj6Rskb+T3jIgDgPFFDqlOJO0ODAb+XexY6mga0C0iegD/Ai4ucjw5yxoK6CigK3By+vNeb7a4pBARSyJis7egI+KZiHgrXX0RaCNpq8JGVzOV3QvJL6K7IuLziHgNWEYyhEhjEcB26XI74K0qjm3IzgauiojPASLi3SLHU1fXAReS/Ps0WhExNSLK0tU5JO9DNRZ5Hwpoi0sKOToeWFj+w9wI7Qa8kbW+It3WWPwIuEbSGyR/XTeav+Q2sQ9wqKSnJc2U1KfYAdWWpGOANyPi2WLHUs++BzxS7CBqIO8/2w3qPYX6Iulx4CsV7PpZRDxQzbkHAL8ieUwuurrcS0NW1X0Bg4AfR8R9kk4E/gAcUcj4clXNfbQA2gP9gD7A3ZK+Gg20H3g19/JTGsjPRC5y+bmR9DOgDLizkLE1dE0yKURErX6BSOoE3A+cFhGv1G9UtVPLe2nww4VUdV+SbgfOS1fvAW4pSFC1UM19nA38JU0CcyWtJxnErLRQ8dVEZfciqTvQBXhWEiT/nxZK6hsRbxcwxJxV93MjaTQwDBjUUJN0JfL+s+3qo1TaA+FhYFxEPFnseOroQeAkSVtJ6gLsDcwtckw18RYwMF0+HFhaxFjq4q/ANwAk7QO0ohGO0BkRz0fEzhHROSI6k1RZ9G6oCaE6koaQtI0Mj4hPix1PDeV9KKAt7o1mSccBNwIdgA+BRRFxpKRLSOqus38BDW7IjYOV3Uu672ck9aVlwI8iotHUm0r6D+A3JE+ya4EfRMSC4kZVc+kP7a1AL+AL4IKI+Htxo6o7ScuBkohodAkOQNIyYCtgVbppTkSMKWJINSLpW8D1bBgK6Mp6vf6WlhTMzKxyrj4yM7MMJwUzM8twUjAzswwnBTMzy3BSMDOzDCcFKwhJqzdZHy3pt8WKpz5J2i8d0fUZSXsVO576IGnGpiPv2pbBScEsB5Kqevv/WODeiDgw1zfhG9motbYFcVKwopN0m6SRWeur08/D0oHkHpD0qqSrJJ0iaW4678Ve6XFHp4POPSPpcUkd0+2XS7o1/av3VUn/VUn5qyVdl855MF1Sh3T7DEnXS5oPnCfp62k8CyQ9JmmX9EWiHwFnS3oiPe87aYyLJP2uPAGk5fxa0rPAwdUcd6WkZ5XMK1F+Px3T8f+fTb8Oqaq8rPsbIumerPXDJP0tXb5ZyZwVL0q6orLvT9bySEm3pcsdJN0naV761T/Hf3JrwJwUrFDapL+0FklaBPy/HM/rCYwB9gdOBfaJiL4k4yH9MD3mH0C/iDiQZCjhC/BAaeEAAALeSURBVLPO3w84kmTI4csktaygjK2B+emcBzOBy7L2tYqIEuAGkrfHR0bE10neVL4yIqYAE4DrIuIbkvYHvg30T+eDWAecklXO0xHRk+Rt2qqOm5MeNwv4frr9BmBmur038GI15ZV7HDhI0tbp+rfT7xMkA8SVAD2AgZJ6VPD9qcxv0vvuQzKycIMdo8py1yQHxLMG6bP0lxaQGZAslzrreRGxMj3nFWBquv150nGFSAYFmyxpF5LxhV7LOv/hdAj0zyW9C3QkGbsn23pgcrp8B/CXrH3l2/cFugHT0kHhmgMrK4h3EPB1YF56XBugfKiUdcB9ORz3BfC3dHkB8M10+XDgNICIWAd8JOnUKq5DemyZpEeBoyXdCwxlQ+I8UdKZJL8LdiGZuOW5Cu6rIkcAXdNyAbaTtE1ErK7iHGvgnBSsISgjfWqV1IzkF3u57Dkt1metr2fD/98bgWsj4kFJhwGXV3L+OnL7P5899sua9FPAixFxcDXnCpgUERXNAbE2/WVe3XFfZo3cWV3MVV0n213AucD7JE9FnygZLPECoE9EfJBWC7Wu4Nzs70f2/mYkT2hrqynbGhFXH1lDsJzkr12A4UBFVTxVaceG4YNH1aL8ZkB5m8Z/klRHbeploIOkgwEktVQy98ampgMjJe2cHtde0p51OG7Tc85Oj28uqV0NrjOTpMrp+2yoOtqOJOl9lLZbHFVJue9I2j9N2MdlbZ/Khio8JPXa7ExrdJwUrCH4PUl99rPAwWz46zxXlwP3SFpA7YamXgP0lfQCSRXNZu0d6dSHI4FfpXEuAg6p4LjFwCXAVEnPkcwHvEttj9vEecA3JD1PUq3UtQblrSOpkjoq/SSdRe0Z4CXg/4DKhowfl57zFBtXmf0XUCLpOUmLSdp+rJHzKKm2xZO0OiK2KXYcZg2BnxTMzCzDTwpmZpbhJwUzM8twUjAzswwnBTMzy3BSMDOzDCcFMzPL+P9tFUE7e6ALNQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Model: <function dnn at 0x7fcdf56a37b8>\n",
            "Shape of input: (23750, 2752)\n",
            "Hyperparameters: {'reg_amount': 0.0, 'drop_rate': 0.0, 'layer_sizes': (32,)}\n",
            "Number of training data: 50\n",
            "Number of validation data: 400\n",
            "Number of epochs: 400\n",
            "Doing training iteration no. 1\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "dense (Dense)                multiple                  88096     \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            multiple                  0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              multiple                  33        \n",
            "=================================================================\n",
            "Total params: 88,129\n",
            "Trainable params: 88,129\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Best train accuracy: 0.9599999785423279\n",
            "Number of epochs: 35\n",
            "Doing training iteration no. 2\n",
            "Doing training iteration no. 3\n",
            "Doing training iteration no. 4\n",
            "Doing training iteration no. 5\n",
            "Doing training iteration no. 6\n",
            "Doing training iteration no. 7\n",
            "Doing training iteration no. 8\n",
            "Doing training iteration no. 9\n",
            "Doing training iteration no. 10\n",
            "Results ('mean_val_auc', 0.8381119489669799, 'mean_val_accuracy', 0.7775000035762787, 'metric', 0.8078059762716293, 'val_auc_std', 0.023772682868251443, 'val_accuracy_std', 0.02921899779108061)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5xd873/8dc7N4lLQxgpIpL2uEXunUTIISoa0QQVoXq0kpYSpxetI0TruPzKr0qKUkeaqorSikuLVhBSkZRDbuKW0LiMCkHENSGY5HP+WGu2PclcVmZm7z2TeT8fj/3Y674+a8/M/sz6ftf3+1VEYGZmBtCm1AGYmVnz4aRgZmY5TgpmZpbjpGBmZjlOCmZmltOu1AE0xg477BA9evQodRhmZi3KwoUL34qIsprWteik0KNHDxYsWFDqMMzMWhRJL9e2zsVHZmaW46RgZmY5TgpmZpbTousUzKzpfPrppyxfvpy1a9eWOhRrIh07dqRbt260b98+8z5OCmYGwPLly9lmm23o0aMHkkodjjVSRLBq1SqWL19Oz549M+/n4iMzA2Dt2rVsv/32TgibCUlsv/32m3zn56RgZjlOCJuXhvw8nRTMzCzHdQpmVqMek+5u0uNVXDyqzvWrVq1i+PDhALz++uu0bduWsrKk0e28efPo0KFDk8aTxUEHHcTkyZMpLy8v+rlLpWBJQdJ1wGjgzYjonS67FDgc+AR4Afh2RLybrjsbOBFYB/wwIu4rVGzQ9L/wG6rvD8DMqtt+++1ZvHgxAOeffz5bb701Z5xxRm59ZWUl7dr5/9hCK2Tx0fXAyA2W3Q/0joi+wD+BswEk9QKOA/ZJ9/kfSW0LGJuZtQDjx49nwoQJ7Lvvvpx55pmcf/75TJ48Obe+d+/eVFRUAHDjjTcyePBg+vfvzymnnMK6deuqHevee+/lmGOOyc3Pnj2b0aNHA3DqqadSXl7OPvvsw3nnnVdjLFtvvXVu+rbbbmP8+PEArFy5kqOPPppBgwYxaNAgHn74YQAeeugh+vfvT//+/RkwYAAffPBBoz+PYihYUoiIOcDbGyybGRGV6eyjQLd0+kjg5oj4OCJeAp4HBhcqNjNrOZYvX84jjzzCZZddVus2S5cuZfr06Tz88MMsXryYtm3bctNNN1Xb5pBDDuGxxx5jzZo1AEyfPp3jjjsOgIsuuogFCxbw5JNP8tBDD/Hkk09mju+0007jxz/+MfPnz+f222/npJNOAmDy5MlcffXVLF68mLlz59KpU6dNvfSSKOW92HeA6en0LiRJosrydNlGJJ0MnAzQvXv3QsZnZs3AMcccQ9u2dRcczJo1i4ULFzJo0CAAPvroI3bcccdq27Rr146RI0fy17/+lbFjx3L33XdzySWXAHDLLbcwdepUKisrWbFiBUuWLKFv376Z4nvggQdYsmRJbv79999n9erVDB06lNNPP53jjz+eMWPG0K1btzqO0nyUJClI+ilQCdxU37YbioipwFSA8vLyaOLQzKyZ2WqrrXLT7dq1Y/369bn5qmfwI4Jx48bx85//vM5jHXfccfz617+mS5culJeXs8022/DSSy8xefJk5s+fz3bbbcf48eNrfLY///HO/PXr16/n0UcfpWPHjtW2nzRpEqNGjWLGjBkMHTqU++67j7322mvTLr4Eiv5IqqTxJBXQx0dE1Zf6q8CueZt1S5eZmeX06NGDRYsWAbBo0SJeeuklAIYPH85tt93Gm2++CcDbb7/Nyy9v3Dv0sGHDWLRoEb/97W9zRUfvv/8+W221FZ07d+aNN97gnnvuqfHcXbt2ZenSpaxfv56//OUvueUjRozgqquuys1XVZa/8MIL9OnTh7POOotBgwbx7LPPNsEnUHhFvVOQNBI4ExgWER/mrboL+KOky4Cdgd2BecWMzcyqa45P0B199NHccMMN7LPPPuy7777sscceAPTq1YsLL7yQESNGsH79etq3b8/VV1/NbrvtVm3/tm3bMnr0aK6//nqmTZsGQL9+/RgwYAB77bUXu+66K0OHDq3x3BdffDGjR4+mrKyM8vJyVq9eDcCVV17J9773Pfr27UtlZSUHHnggU6ZM4YorruDBBx+kTZs27LPPPhx22GEF/GSajj77Z72JDyz9CTgI2AF4AziP5GmjLYBV6WaPRsSEdPufktQzVAI/ioia03We8vLyaOggO34k1ay6pUuXsvfee5c6DGtiNf1cJS2MiBobXxTsTiEivlHD4t/Vsf1FwEWFisfMzOrnbi7MzCzHScHMzHKcFMzMLMdJwczMcpwUzMwsx10OmlnNzu/cxMd7r95N2rZtS58+faisrGTvvfdm2rRpbLnllg063fjx4xk9ejRjx47lpJNO4vTTT6dXr141bjt79mw6dOjA/vvvD8CUKVPYcsstOeGEExp07pbMdwpm1mx06tSJxYsX8/TTT9OhQwemTJlSbX1lZWUte9bt2muvrTUhQJIUHnnkkdz8hAkTipYQIqJa1x0bztemoZ9FfZwUzKxZOuCAA3j++eeZPXs2BxxwAEcccQS9evVi3bp1TJw4kUGDBtG3b19+85vfAMmX6fe//3323HNPDjnkkFyXF5AMllPV0PXee+9l4MCB9OvXj+HDh1NRUcGUKVO4/PLL6d+/P3Pnzq3WRffixYsZMmQIffv25aijjuKdd97JHfOss85i8ODB7LHHHsydO7fG67j00ktzsVZ1y11RUcGee+7JCSecQO/evZk7d261+VdeeYWJEyfSu3dv+vTpw/TpSd+hG34Wa9asYdSoUfTr14/evXvntmsMFx+ZWbNTWVnJPffcw8iRyZAsixYt4umnn6Znz55MnTqVzp07M3/+fD7++GOGDh3KiBEjePzxx3nuuedYsmQJb7zxBr169eI73/lOteOuXLmS7373u8yZM4eePXvy9ttv06VLFyZMmFBtUJ9Zs2bl9jnhhBO46qqrGDZsGOeeey4XXHABV1xxRS7OefPmMWPGDC644AIeeOCBauebOXMmy5YtY968eUQERxxxBHPmzKF79+4sW7aMadOmMWTIECoqKqrN33777SxevJgnnniCt956i0GDBnHggQdu9Fncfvvt7Lzzztx9d9JDw3vv1V9EVx/fKZhZs/HRRx/Rv39/ysvL6d69OyeeeCIAgwcPpmfPnkDyRXvDDTfQv39/9t13X1atWsWyZcuYM2cO3/jGN2jbti0777wzBx988EbHf/TRRznwwANzx+rSpUud8bz33nu8++67DBs2DIBx48YxZ86c3PoxY8YA8KUvfSk32E++mTNnMnPmTAYMGMDAgQN59tlnWbZsGQC77bYbQ4YMyW2bP/+Pf/wjdy1du3Zl2LBhzJ8/f6PPok+fPtx///2cddZZzJ07l86dG18P5DsFM2s2quoUNpTffXZEcNVVV3HooYdW22bGjBkFj29DW2yxBZBUkNdUxh8RnH322ZxyyinVlldUVFS7JmCj+drkb7fHHnuwaNEiZsyYwTnnnMPw4cM599xzN/UyqvGdgpm1KIceeijXXHMNn376KQD//Oc/WbNmDQceeCDTp09n3bp1rFixggcffHCjfYcMGcKcOXNyXW6//XYyOOQ222xT43CZnTt3ZrvttsvVF/zhD3/I3TVkjfW6667L9aj66quvVqvrqM0BBxyQu5aVK1cyZ84cBg/eeDDK1157jS233JJvfvObTJw4MdeteGP4TsHMapbhEdJSOOmkk6ioqGDgwIFEBGVlZdxxxx0cddRR/P3vf6dXr150796d/fbbb6N9y8rKmDp1KmPGjGH9+vXsuOOO3H///Rx++OGMHTuWO++8s9rYCADTpk1jwoQJfPjhh3zhC1/g97//feZYR4wYwdKlS3OxbL311tx44431jiR31FFH8b//+7/069cPSVxyySV8/vOf32hMhqeeeoqJEyfSpk0b2rdvzzXXXJM5ttoUrOvsYnDX2WZNx11nb542tetsFx+ZmVmOk4KZmeU4KZhZTksuTraNNeTn6aRgZgB07NiRVatWOTFsJiKCVatW0bFjx03az08fmRkA3bp1Y/ny5axcubLUoVgT6dixI926ddukfZwUzAyA9u3b51rKWuvl4iMzM8txUjAzsxwnBTMzy3FSMDOzHCcFMzPLKVhSkHSdpDclPZ23rIuk+yUtS9+3S5dL0pWSnpf0pKSBhYrLzMxqV8g7heuBkRssmwTMiojdgVnpPMBhwO7p62Sg8V39mZnZJitYUoiIOcDbGyw+EpiWTk8Dvpa3/IZIPApsK2mnQsVmZmY1K3adQteIWJFOvw50Tad3AV7J2255umwjkk6WtEDSAre8NDNrWiWraI6kg5VN7mQlIqZGRHlElJeVlRUgMjOz1qvYSeGNqmKh9L1qXLpXgV3ztuuWLjMzsyIqdlK4CxiXTo8D7sxbfkL6FNIQ4L28YiYzMyuSgnWIJ+lPwEHADpKWA+cBFwO3SDoReBk4Nt18BvBV4HngQ+DbhYrLzMxqV29SkHQJcCHwEXAv0Bf4cUTcWNd+EfGNWlYNr2HbAL5Xb7RmZlZQWYqPRkTE+8BooAL4N2BiIYMyM7PSyJIUqu4mRgG3RsR7BYzHzMxKKEudwt8kPUtSfHSqpDJgbWHDMjOzUqj3TiEiJgH7A+UR8SlJRfCRhQ7MzMyKr96kIGlL4D/5rD+inYHyQgZlZmalkaVO4ffAJyR3C5A0KruwYBGZmVnJZEkKX4yIS4BPASLiQ0AFjcrMzEoiS1L4RFIn0n6KJH0R+LigUZmZWUlkefroPJJGa7tKugkYCowvZFBmZlYa9SaFiLhf0iJgCEmx0WkR8VbBIzMzs6LL8vTRUUBlRNwdEX8DKiV9rb79zMys5clSp3BefivmiHiXpEjJzMw2M1mSQk3bFKx3VTMzK50sSWGBpMskfTF9XQYsLHRgZmZWfFmSwg9IGq9NT18f426uzcw2S1mePloDTCpCLGZmVmJZBtnZAzgD6JG/fUQcXLiwzMysFLJUGN8KTAGuBdYVNhwzMyulLEmhMiKuqX8zMzNr6bJUNP9V0n9K2klSl6pXwSMzM7Oiy3KnMC59zx+XOYAvNH04ZmZWSlmePupZjEDMzKz0Mo28JukcSVPT+d0ljS58aGZmVmweec3MzHI88pqZmeWUZOQ1ST+W9IykpyX9SVJHST0lPSbpeUnTJXVozDnMzGzTZeo6m+ojr80CzmzoCSXtAvwQKI+I3kBb4DjgF8DlEfFvwDvAiQ09h5mZNUydSUFSG2A7YAzJEJx/Ivkyn93I87YDOklqB2wJrAAOBm5L108DPJCPmVmR1ZkUImI9cGZErKoaea2xQ3FGxKvAZOBfJMngPZKuuN+NiMp0s+XALjXtL+lkSQskLVi5cmVjQjEzsw1kKT56QNIZknZtihbNkrYDjgR6AjsDWwEjs+4fEVMjojwiysvKyhoahpmZ1SBLi+avp+/5Yyg0pkXzIcBLEbESQNKfgaHAtpLapXcL3UgefTUza5F6TLq7oMevuHhUQY6bpU5hUkT03ODVmC4u/gUMSRvFCRgOLAEeBMam24wD7mzEOczMrAGy1ClMrGubTRURj5FUKC8CnkpjmAqcBZwu6Xlge+B3TXleMzOrX5biowcknUEyFOeaqoUR8XZDTxoR55E86prvRWBwQ49pZmaNV4o6BTMza6bcS6qZmeVkGaP5hJqWR8QNTR+OmZmVUpbio0F50x1JnhZaBDgpmJltZrIUH/0gf17StsDNBYvIzMxKJkuL5g2tIWmNbGZmm5ksdQp/Je02mySJ9AJuKWRQZmZWGlnqFCbnTVcCL0fE8gLFY2ZmJZQlKfwLWBERawEkdZLUIyIqChqZmZkVXZY6hVuB9Xnz69JlZma2mcmSFNpFxCdVM+m0h8o0M9sMZUkKKyUdUTUj6UigUQPtmJlZ85SlTmECcJOkX6fzy4EaWzmbmVnLlqXx2gsk4x9snc6vLnhUZmZWEvUWH0n6/5K2jYjVEbFa0naSLixGcGZmVlxZ6hQOi4h3q2Yi4h3gq4ULyczMSiVLUmgraYuqGUmdgC3q2N7MzFqoLBXNNwGzJP0+nf82MK1wIZmZWalkqWj+haQngEPSRT+LiPsKG5aZmZVCljsFgMeB9iQd4z1euHDMzKyUsjx9dCwwDxgLHAs8JmlsoQMzM7Piy3Kn8FNgUES8CSCpDHgAuK2QgZmZWfFlefqoTVVCSK3KuJ+ZmbUwWe4U7pV0H/CndP7rwIzChWRmZqWS5emjiZLGAP+eLpoaEX8pbFhmZlYKmZ4+iog/A39uqpNK2ha4FuhN8kTTd4DngOlAD6ACODZtPW1mZkVSqrqBXwH3RsReQD9gKTAJmBURuwOz0nkzMyuioicFSZ2BA4HfQTJoT9q30pF81lJ6GvC1YsdmZtba1ZoUJM1K33/RxOfsCawEfi/pcUnXStoK6BoRK9JtXge61hLXyZIWSFqwcuXKJg7NzKx1q+tOYSdJ+wNHSBogaWD+qxHnbAcMBK6JiAHAGjYoKoqIIKlr2EhETI2I8ogoLysra0QYZma2oboqms8F/hvoBly2wboADm7gOZcDyyPisXT+NpKk8IaknSJihaSdgDdrPYKZmRVErUkhIm4DbpP03xHxs6Y6YUS8LukVSXtGxHPAcGBJ+hoHXJy+39lU5zQzs2yytFP4maQjSCqHAWZHxN8aed4fkIz73AF4kaQ77jbALZJOBF4m6WfJzMyKqN6kIOnnwGCScRUATpO0f0T8pKEnjYjFQHkNq4Y39JhmZtZ4WRqvjQL6R8R6AEnTSLrPbnBSMDOz5ilrO4Vt86Y7FyIQMzMrvSx3Cj8HHpf0ICCSugW3NjYz2wxlqWj+k6TZwKB00VkR8XpBozIzs5LI2iHeCuCuAsdiZmYl5sFyzMwsx0nBzMxy6kwKktpKerZYwZiZWWnVmRQiYh3wnKTuRYrHzMxKKEtF83bAM5LmkfRoCkBEHFGwqMzMrCSyJIX/LngUZmbWLGRpp/CQpN2A3SPiAUlbAm0LH5qZmRVbvU8fSfouyZgHv0kX7QLcUcigzMysNLI8kvo9YCjwPkBELAN2LGRQZmZWGlmSwscR8UnVjKR21DJUppmZtWxZksJDkn4CdJL0FeBW4K+FDcvMzEohS1KYBKwEngJOAWYA5xQyKDMzK40sTx+tTwfWeYyk2Oi5iHDxkZnZZijLcJyjgCnACyTjKfSUdEpE3FPo4MzMrLiyNF77JfDliHgeQNIXgbsBJwVrVXpMurtgx664eFTBjm22KbLUKXxQlRBSLwIfFCgeMzMroVrvFCSNSScXSJoB3EJSp3AMML8IsZmZWZHVVXx0eN70G8CwdHol0KlgEZmZWcnUmhQi4tvFDMTMzEovy9NHPYEfAD3yt3fX2WZmm58sTx/dAfyOpBXz+qY6saS2wALg1YgYnSafm4HtgYXAt/K71zAzs8LLkhTWRsSVBTj3acBS4HPp/C+AyyPiZklTgBOBawpwXjMzq0WWR1J/Jek8SftJGlj1asxJJXUDRgHXpvMCDibpohtgGvC1xpzDzMw2XZY7hT7At0i+tKuKjyKdb6grgDOBbdL57YF3I6IynV9OMm7DRiSdDJwM0L27h442M2tKWZLCMcAXmqp8X9Jo4M2IWCjpoE3dPyKmAlMBysvL3QeTmVkTypIUnga2Bd5sonMOBY6Q9FWgI0mdwq+AbSW1S+8WugGvNtH5zMwsoyx1CtsCz0q6T9JdVa+GnjAizo6IbhHRAzgO+HtEHA88CIxNNxsH3NnQc5iZWcNkuVM4r+BRJM4CbpZ0IfA4yWOwZmZWRFnGU3ioUCePiNnA7HT6RWBwoc5lZmb1y9Ki+QM+G5O5A9AeWBMRn6t9LzMza4my3ClUPTZa1Z7gSGBIIYMyM7PSyFLRnBOJO4BDCxSPmZmVUJbiozF5s22AcmBtwSIyM7OSyfL0Uf64CpVABUkRkpmZbWay1Cl4XAUzs1airuE4z61jv4iInxUgHrMG6zHp7lKHYNbi1XWnsKaGZVuRdGm9PeCkYGa2malrOM5fVk1L2oZk/INvkwyE88va9jMzs5arzjoFSV2A04HjScY4GBgR7xQjMDMzK7666hQuBcaQdFPdJyJWFy0qMzMriboar/0XsDNwDvCapPfT1weS3i9OeGZmVkx11SlsUmtnMzNr+fzFb2ZmOU4KZmaW46RgZmY5Wfo+MjOrVaFbkldcPKqgx7fqfKdgZmY5TgpmZpbjpGBmZjlOCmZmluOkYGZmOU4KZmaW46RgZmY5bqdg1gz4WX9rLop+pyBpV0kPSloi6RlJp6XLu0i6X9Ky9H27YsdmZtbaleJOoRL4r4hYlI7otlDS/cB4YFZEXCxpEjAJOKsE8ZlZK1GKcb0rOv5HEx3pvSY6TnVFv1OIiBURsSid/gBYCuwCHEkyuhvp+9eKHZuZWWtX0opmST2AAcBjQNeIWJGueh3oWss+J0taIGnBypUrixKnmVlrUbKkIGlr4HbgRxFRbSS3iAggatovIqZGRHlElJeVlRUhUjOz1qMkSUFSe5KEcFNE/Dld/IakndL1OwFvliI2M7PWrBRPHwn4HbA0Ii7LW3UXMC6dHgfcWezYzMxau1I8fTQU+BbwlKTF6bKfABcDt0g6EXgZOLYEsZmZtWpFTwoR8Q9AtaweXsxYzMysOndzYWZmOe7mwqyRmqIxUo+1f2yCSDZPpWhg1po5KZi1Av5itaxcfGRmZjlOCmZmluOkYGZmOa5TsKJy2bZZ8+akYC3S5vbET1N1p9ycrslaJhcfmZlZjpOCmZnlOCmYmVmOk4KZmeU4KZiZWY6TgpmZ5fiRVDOrxo/Htm6+UzAzsxwnBTMzy3FSMDOzHNcp2EbcP5FtblxPkp2TgpkVxOb4RdxU19ScufjIzMxynBTMzCzHScHMzHKcFMzMLMcVzZZZa6hkM2vtmt2dgqSRkp6T9LykSaWOx8ysNWlWdwqS2gJXA18BlgPzJd0VEUtKG1nz4nYEZlYoze1OYTDwfES8GBGfADcDR5Y4JjOzVkMRUeoYciSNBUZGxEnp/LeAfSPi+3nbnAycnM7uCTzXwNPtALzViHCbE19L87S5XMvmch3ga6myW0SU1bSiWRUfZRERU4GpjT2OpAURUd4EIZWcr6V52lyuZXO5DvC1ZNHcio9eBXbNm++WLjMzsyJobklhPrC7pJ6SOgDHAXeVOCYzs1ajWRUfRUSlpO8D9wFtgesi4pkCna7RRVDNiK+ledpcrmVzuQ7wtdSrWVU0m5lZaTW34iMzMyshJwUzM8tpdUlB0jGSnpG0XlJ53vKvSFoo6an0/eBSxplFbdeSrjs77SrkOUmHlirGhpDUX9KjkhZLWiBpcKljaihJP5D0bPpzuqTU8TSWpP+SFJJ2KHUsDSXp0vRn8qSkv0jattQxbYpCdwXU6pIC8DQwBpizwfK3gMMjog8wDvhDsQNrgBqvRVIvkie39gFGAv+TdiHSUlwCXBAR/YFz0/kWR9KXSVrk94uIfYDJJQ6pUSTtCowA/lXqWBrpfqB3RPQF/gmcXeJ4MsvrCugwoBfwjfTvvcm0uqQQEUsjYqNW0BHxeES8ls4+A3SStEVxo9s0tV0LyRfRzRHxcUS8BDxP0oVISxHA59LpzsBrdWzbnJ0KXBwRHwNExJsljqexLgfOJPn5tFgRMTMiKtPZR0naQ7UUBe8KqNUlhYyOBhZV/TG3QLsAr+TNL0+XtRQ/Ai6V9ArJf9ct5j+5DewBHCDpMUkPSRpU6oAaStKRwKsR8USpY2li3wHuKXUQm6Dgf9vNqp1CU5H0APD5Glb9NCLurGfffYBfkNwml1xjrqU5q+u6gOHAjyPidknHAr8DDilmfFnVcx3tgC7AEGAQcIukL0QzfQ68nmv5Cc3kbyKLLH83kn4KVAI3FTO25m6zTAoR0aAvEEndgL8AJ0TEC00bVcM08FqafXchdV2XpBuA09LZW4FrixJUA9RzHacCf06TwDxJ60k6MVtZrPg2RW3XIqkP0BN4QhIkv0+LJA2OiNeLGGJm9f3dSBoPjAaGN9ckXYuC/227+CiVPoFwNzApIh4udTyNdBdwnKQtJPUEdgfmlTimTfEaMCydPhhYVsJYGuMO4MsAkvYAOtACe+iMiKciYseI6BERPUiKLAY214RQH0kjSepGjoiID0sdzyYqeFdAra5Fs6SjgKuAMuBdYHFEHCrpHJKy6/wvoBHNuXKwtmtJ1/2UpLy0EvhRRLSYclNJ/w78iuROdi3wnxGxsLRRbbr0j/Y6oD/wCXBGRPy9tFE1nqQKoDwiWlyCA5D0PLAFsCpd9GhETChhSJtE0leBK/isK6CLmvT4rS0pmJlZ7Vx8ZGZmOU4KZmaW46RgZmY5TgpmZpbjpGBmZjlOClYUklZvMD9e0q9LFU9TkrRX2qPr45K+WOp4moKk2Rv2vGutg5OCWQaS6mr9/zXgtogYkLUlfAvrtdZaEScFKzlJ10samze/On0/KO1I7k5JL0q6WNLxkual4158Md3u8LTTucclPSCpa7r8fEnXpf/1vijph7Wcf7Wky9MxD2ZJKkuXz5Z0haQFwGmSvpTGs1DSfZJ2ShsS/Qg4VdKD6X7fTGNcLOk3VQkgPc8vJT0B7FfPdhdJekLJuBJV19M17f//ifS1f13ny7u+kZJuzZs/SNLf0ulrlIxZ8YykC2r7fPKmx0q6Pp0uk3S7pPnpa2jGH7k1Y04KViyd0i+txZIWA/8v4379gAnA3sC3gD0iYjBJf0g/SLf5BzAkIgaQdCV8Zt7+ewGHknQ5fJ6k9jWcYytgQTrmwUPAeXnrOkREOXAlSevxsRHxJZKWyhdFxAxgCnB5RHxZ0t7A14Gh6XgQ64Dj887zWET0I2lNW9d2j6bbzQG+my6/EngoXT4QeKae81V5ANhX0lbp/NfTzwmSDuLKgb7AMEl9a/h8avOr9LoHkfQs3Gz7qLLsNssO8axZ+ij90gJyHZJlKbOeHxEr0n1eAGamy58i7VeIpFOw6ZJ2Iulf6KW8/e9Ou0D/WNKbQFeSvnvyrQemp9M3An/OW1e1fE+gN3B/2ilcW2BFDfEOB74EzE+36wlU93wAAAH3SURBVARUdZWyDrg9w3afAH9LpxcCX0mnDwZOAIiIdcB7kr5Vx3FIt62UdC9wuKTbgFF8ljiPlXQyyXfBTiQDtzxZw3XV5BCgV3pegM9J2joiVtexjzVzTgrWHFSS3rVKakPyxV4lf0yL9Xnz6/ns9/cq4LKIuEvSQcD5tey/jmy/8/l9v6xJ3wU8ExH71bOvgGkRUdMYEGvTL/P6tvs0r+fO+mKu6zj5bga+D7xNclf0gZLOEs8ABkXEO2mxUMca9s3/PPLXtyG5Q1tbz7mtBXHxkTUHFST/7QIcAdRUxFOXznzWffC4Bpy/DVBVp/EfJMVRG3oOKJO0H4Ck9krG3tjQLGCspB3T7bpI2q0R2224z6np9m0ldd6E4zxEUuT0XT4rOvocSdJ7L623OKyW874hae80YR+Vt3wmnxXhIan/Rntai+OkYM3Bb0nKs58A9uOz/86zOh+4VdJCGtY19RpgsKSnSYpoNqrvSIc+HAv8Io1zMbB/DdstAc4BZkp6kmQ84J0aut0GTgO+LOkpkmKlXptwvnUkRVKHpe+ko6g9DjwL/BGorcv4Sek+j1C9yOyHQLmkJyUtIan7sRbOvaRaqydpdURsXeo4zJoD3ymYmVmO7xTMzCzHdwpmZpbjpGBmZjlOCmZmluOkYGZmOU4KZmaW838eZbcye8r5gQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Model: <function cnn at 0x7fcdf56a3c80>\n",
            "Shape of input: (23750, 12, 14, 16)\n",
            "Hyperparameters: {'reg_amount': 0.0, 'drop_rate': 0.0, 'xs_shape': (23750, 12, 14, 16)}\n",
            "Number of training data: 50\n",
            "Number of validation data: 400\n",
            "Number of epochs: 400\n",
            "Doing training iteration no. 1\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 11, 13, 32)        2080      \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 11, 13, 16)        528       \n",
            "_________________________________________________________________\n",
            "average_pooling2d (AveragePo (None, 5, 6, 16)          0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 480)               0         \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 480)               0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 1)                 481       \n",
            "=================================================================\n",
            "Total params: 3,089\n",
            "Trainable params: 3,089\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Best train accuracy: 1.0\n",
            "Number of epochs: 35\n",
            "Doing training iteration no. 2\n",
            "Doing training iteration no. 3\n",
            "Doing training iteration no. 4\n",
            "Doing training iteration no. 5\n",
            "Doing training iteration no. 6\n",
            "Doing training iteration no. 7\n",
            "Doing training iteration no. 8\n",
            "Doing training iteration no. 9\n",
            "Doing training iteration no. 10\n",
            "Results ('mean_val_auc', 0.8118107438087463, 'mean_val_accuracy', 0.7614999949932099, 'metric', 0.7866553694009781, 'val_auc_std', 0.0461885203069655, 'val_accuracy_std', 0.041985113778413484)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5xWZbn/8c+Xk+AhFEVSEaESFTlJA6JswcSNGoiJWJYllKaYtS23KJZb8bdz54HUNH8SlYnpTjzkoUAFSYQ05SSeQMPDmCgq4hEUdeDaf6w1jw8wh8XMPM8zM3zfr9e8Zh3utda1Bua55r7vte5bEYGZmRlAi1IHYGZmjYeTgpmZ5TgpmJlZjpOCmZnlOCmYmVlOq1IHUB+77LJLdO3atdRhmJk1KYsWLXorIjpWta9JJ4WuXbuycOHCUodhZtakSHq5un1uPjIzsxwnBTMzy3FSMDOznCbdp2BmDefTTz9lxYoVrFu3rtShWANp27YtnTt3pnXr1pmPcVIwMwBWrFjBDjvsQNeuXZFU6nCsniKC1atXs2LFCrp165b5ODcfmRkA69atY+edd3ZCaCYksfPOO29xzc9JwcxynBCal7r8ezopmJlZjvsUzKxKXSdMb9DzlV8yvMb9q1evZujQoQC8/vrrtGzZko4dk5du58+fT5s2bRo0niwOPfRQJk2aRFlZWdGvXSpbbVJo6P/wm6rtF8DMNrbzzjuzZMkSACZOnMj222/P2WefndtfUVFBq1Zb7UdW0bj5yMwarbFjxzJu3DgOPPBAzjnnHCZOnMikSZNy+3v27El5eTkAN910EwMGDKBv376cdtpprF+/fqNz3XfffRx//PG59Tlz5jBixAgATj/9dMrKyth///258MILq4xl++23zy3ffvvtjB07FoBVq1Zx3HHH0b9/f/r378/DDz8MwEMPPUTfvn3p27cvBxxwAB988EG9fx7F4KRgZo3aihUreOSRR7jiiiuqLbNs2TKmTZvGww8/zJIlS2jZsiU333zzRmUOP/xwHnvsMdauXQvAtGnTOOGEEwC4+OKLWbhwIU8++SQPPfQQTz75ZOb4zjzzTH7yk5+wYMEC7rjjDk455RQAJk2axLXXXsuSJUuYN28e7dq129JbLwnXxcysUTv++ONp2bJljWVmz57NokWL6N+/PwAfffQRu+6660ZlWrVqxZFHHslf/vIXRo8ezfTp07nssssAuPXWW5kyZQoVFRWsXLmSpUuX0rt370zxPfDAAyxdujS3/v7777NmzRoGDRrEWWedxYknnsioUaPo3Lnzltx2yTgpmFmjtt122+WWW7VqxYYNG3Lrlc/gRwRjxozhF7/4RY3nOuGEE/j1r39Nhw4dKCsrY4cdduCll15i0qRJLFiwgJ122omxY8dW+Wx//uOd+fs3bNjAo48+Stu2bTcqP2HCBIYPH86MGTMYNGgQ999/P/vuu++W3XwJFKz5SNL1kt6U9HTetsslPSvpSUl3Stoxb995kp6X9JykIwoVl5k1XV27dmXx4sUALF68mJdeegmAoUOHcvvtt/Pmm28C8Pbbb/Pyy5uPDj1kyBAWL17Mb3/721zT0fvvv892221H+/bteeONN7j33nurvHanTp1YtmwZGzZs4M4778xtHzZsGNdcc01uvbKz/IUXXqBXr16ce+659O/fn2effbYBfgKFV8iawg3Ar4Eb87bNAs6LiApJlwLnAedK6gGcAOwP7A48IKl7RKzHzEqiMT5Bd9xxx3HjjTey//77c+CBB9K9e3cAevTowc9//nOGDRvGhg0baN26Nddeey177bXXRse3bNmSESNGcMMNNzB16lQA+vTpwwEHHMC+++7LnnvuyaBBg6q89iWXXMKIESPo2LEjZWVlrFmzBoCrr76aM844g969e1NRUcHgwYOZPHkyV111FQ8++CAtWrRg//3356ijjirgT6bhKCIKd3KpK/DXiOhZxb5jgdERcaKk8wAi4hfpvvuBiRHxj5rOX1ZWFnWdZMePpJptbNmyZey3336lDsMaWFX/rpIWRUSVL1+U8umj7wGV9bQ9gFfy9q1It21G0qmSFkpauGrVqgKHaGa2dSlJUpD0M6ACuLm2spuKiCkRURYRZZVvO5qZWcMo+tNHksYCI4Ch8Vnb1avAnnnFOqfbzMysiIpaU5B0JHAOMDIiPszbdQ9wgqRtJHUD9gbmFzM2MzMrYE1B0p+AQ4FdJK0ALiR52mgbYFb6zO+jETEuIp6RdCuwlKRZ6Qw/eWRmVnwFSwoR8c0qNv++hvIXAxcXKh4zM6ud32g2s6pNbN/A53uv1iItW7akV69eVFRUsN9++zF16lS23XbbOl1u7NixjBgxgtGjR3PKKadw1lln0aNHjyrLzpkzhzZt2nDwwQcDMHnyZLbddltOOumkOl27KfOAeGbWaLRr144lS5bw9NNP06ZNGyZPnrzR/oqKijqd93e/+121CQGSpPDII4/k1seNG1e0hBARGw3dsel6der6s6iNk4KZNUqHHHIIzz//PHPmzOGQQw5h5MiR9OjRg/Xr1zN+/Hj69+9P7969+c1vfgMkH6Y//OEP2WeffTj88MNzQ15AMllO5Yuu9913H/369aNPnz4MHTqU8vJyJk+ezJVXXknfvn2ZN2/eRkN0L1myhIEDB9K7d2+OPfZY3nnnndw5zz33XAYMGED37t2ZN29elfdx+eWX52KtHJa7vLycffbZh5NOOomePXsyb968jdZfeeUVxo8fT8+ePenVqxfTpk0D2OxnsXbtWoYPH06fPn3o2bNnrlx9uPnIzBqdiooK7r33Xo488kggGefo6aefplu3bkyZMoX27duzYMECPv74YwYNGsSwYcN4/PHHee6551i6dClvvPEGPXr04Hvf+95G5121ahXf//73mTt3Lt26dePtt9+mQ4cOjBs3bqNJfWbPnp075qSTTuKaa65hyJAhXHDBBVx00UVcddVVuTjnz5/PjBkzuOiii3jggQc2ut7MmTNZvnw58+fPJyIYOXIkc+fOpUuXLixfvpypU6cycOBAysvLN1q/4447WLJkCU888QRvvfUW/fv3Z/DgwZv9LO644w523313pk9PRmh4773am+hq45qCmTUaH330EX379qWsrIwuXbpw8sknAzBgwAC6desGJB+0N954I3379uXAAw9k9erVLF++nLlz5/LNb36Tli1bsvvuu3PYYYdtdv5HH32UwYMH587VoUOHGuN57733ePfddxkyZAgAY8aMYe7cubn9o0aNAuDLX/5ybrKffDNnzmTmzJkccMAB9OvXj2effZbly5cDsNdeezFw4MBc2fz1v//977l76dSpE0OGDGHBggWb/Sx69erFrFmzOPfcc5k3bx7t29e/H8g1BTNrNCr7FDaVP3x2RHDNNddwxBEbD6Y8Y8aMgse3qW222QZIOsirauOPCM477zxOO+20jbaXl5dvdE/AZuvVyS/XvXt3Fi9ezIwZMzj//PMZOnQoF1xwwZbexkZcUzCzJuWII47guuuu49NPPwXgn//8J2vXrmXw4MFMmzaN9evXs3LlSh588MHNjh04cCBz587NDbn99ttvA7DDDjtUOV1m+/bt2WmnnXL9BX/84x9ztYassV5//fW5EVVfffXVjfo6qnPIIYfk7mXVqlXMnTuXAQMGbFbutddeY9ttt+Xb3/4248ePzw0rXh+uKZhZ1TI8QloKp5xyCuXl5fTr14+IoGPHjtx1110ce+yx/O1vf6NHjx506dKFgw46aLNjO3bsyJQpUxg1ahQbNmxg1113ZdasWRx99NGMHj2au+++e6O5EQCmTp3KuHHj+PDDD/nCF77AH/7wh8yxDhs2jGXLluVi2X777bnppptqnUnu2GOP5R//+Ad9+vRBEpdddhmf//znN5uT4amnnmL8+PG0aNGC1q1bc91112WOrToFHTq70Dx0tlnD8dDZzVNTGjrbzMwaGScFMzPLcVIws5ym3Jxsm6vLv6eTgpkB0LZtW1avXu3E0ExEBKtXr6Zt27ZbdJyfPjIzADp37syKFSvwNLfNR9u2bencufMWHeOkYGYAtG7dOvemrG293HxkZmY5TgpmZpbjpGBmZjlOCmZmluOkYGZmOU4KZmaW46RgZmY5TgpmZpbjpGBmZjkFSwqSrpf0pqSn87Z1kDRL0vL0+07pdkm6WtLzkp6U1K9QcZmZWfUKWVO4AThyk20TgNkRsTcwO10HOArYO/06Faj/9EFmZrbFCpYUImIu8PYmm48BpqbLU4Gv5W2/MRKPAjtK2q1QsZmZWdWK3afQKSJWpsuvA53S5T2AV/LKrUi3bUbSqZIWSlro0RzNzBpWrUlB0mWSPieptaTZklZJ+nZ9LxzJoO1bPHB7REyJiLKIKOvYsWN9wzAzszxZagrDIuJ9YARQDnwJGF/H671R2SyUfn8z3f4qsGdeuc7pNjMzK6IsSaFyzoXhwG0R8V49rncPMCZdHgPcnbf9pPQppIHAe3nNTGZmViRZJtn5q6RngY+A0yV1BNbVdpCkPwGHArtIWgFcCFwC3CrpZOBl4Otp8RnAV4HngQ+B727hfZiZWQOoNSlExARJl5H89b5e0ockTwvVdtw3q9k1tIqyAZxR2znNzKywsnQ0bwv8gM/eHdgdKCtkUGZmVhpZ+hT+AHwCHJyuvwr8vGARmZlZyWRJCl+MiMuATwEi4kNABY3KzMxKIktS+ERSO9J3CiR9Efi4oFGZmVlJZHn66ELgPmBPSTcDg4CxhQzKzMxKI8vTR7MkLQYGkjQbnRkRbxU8MjMzK7osTx8dC1RExPSI+CtQIelrtR1nZmZNT5Y+hQvz32KOiHdJmpTMzKyZyZIUqiqTpS/CzMyamCxJYaGkKyR9Mf26AlhU6MDMzKz4siSFH5G8vDYt/foYD0lhZtYsZXn6aC2fTZtpZmbNWK1JQVJ34Gyga375iDiscGGZmVkpZOkwvg2YDPwOWF/YcMzMrJSyJIWKiLiu9mJmZtbUZelo/oukH0jaTVKHyq+CR2ZmZkWXpaZQOX1m/rzMAXyh4cMxM7NSyvL0UbdiBGJmZqWXaeY1SedLmpKu7y1pROFDMzOzYvPMa2ZmluOZ18zMLMczr5mZWY5nXjMzs5wak4KkFsBOwCg885qZWbNXY/NRRGwAzomI1ZUzrzVEQpD0E0nPSHpa0p8ktZXUTdJjkp6XNE1Sm/pex8zMtkyWPoUHJJ0tac+GeKNZ0h7AfwBlEdETaAmcAFwKXBkRXwLeAU6u6zXMzKxusvQpfCP9nj+HQn3faG4FtJP0KbAtsBI4DPhWun8qMBHwmEtmZkWUpU9hQkRMa6gLRsSrkiYB/wI+AmaSzOT2bkRUpMVWAHtUE9OpwKkAXbp0aaiwzMyMbH0K42sqs6Uk7QQcA3QDdge2A47MenxETImIsogo69ixY0OGZma21St6nwJwOPBSRKyKiE+BP5M85rqjpMqaS2eSN6fNzKyIStGn8C9goKRtSZqPhgILgQeB0cAtJCOz3l3H85uZWR0VfZTUiHhM0u3AYqACeByYAkwHbpH083Tb7xvyumZmVrssczSfVNX2iLixrheNiAtJ3pTO9yIwoK7nNDOz+svSfNQ/b7ktSXPPYqDOScHMzBqnLM1HP8pfl7QjSbu/mZk1M1mePtrUWpLHSc3MrJnJ0qfwF9Jhs0mSSA/g1kIGZWZmpZGlT2FS3nIF8HJErChQPGZmVkJZksK/gJURsQ5AUjtJXSOivKCRmZlZ0WXpU7gN2JC3vj7dZmZmzUyWpNAqIj6pXEmXPdeBmVkzlCUprJI0snJF0jGAZ14zM2uGsvQpjANulvTrdH0FUOVbzmZmlug6YXpBz19+yfCCnDfLy2svkAxgt326vqYgkZiZWcnV2nwk6X8k7RgRayJijaSd0kHrzMysmcnSp3BURLxbuRIR7wBfLVxIZmZWKlmSQktJ21SuSGoHbFNDeTMza6KydDTfDMyW9Id0/bvA1MKFZGZmpZKlo/lSSU+QTKMJ8N8RcX9hwzIzs1LIUlOAZCa01iQD4z1euHDMzKyUsjx99HVgPsn8yV8HHpM0utCBmZlZ8WWpKfwM6B8RbwJI6gg8ANxeyMDMzKz4sjx91KIyIaRWZzzOzMyamCw1hfsk3Q/8KV3/BjCjcCGZmVmpZHn6aLykUcC/pZumRMSdhQ3LzMxKIdPTRxHxZ+DPBY7FzMxKzH0DZmaWU5KkIGlHSbdLelbSMkkHSeogaZak5en3nUoRm5nZ1qzapCBpdvr90gJc91fAfRGxL9AHWAZMAGZHxN7A7HTdzMyKqKY+hd0kHQyMlHQLoPydEbG4LheU1B4YDIxNz/MJ8Ek6o9uhabGpwBzg3Lpcw8zM6qampHAB8F9AZ+CKTfYFcFgdr9kNWAX8QVIfYBFwJtApIlamZV4HOlV1sKRTgVMBunTpUscQzMysKtU2H0XE7RFxFHBZRHxlk6+6JgRIElE/4LqIOABYyyZNRRERJImnqrimRERZRJR17NixHmGYmdmmsryn8N+SRpI0+QDMiYi/1uOaK4AVEfFYun47SVJ4Q9JuEbFS0m7Am9WewczMCiLLgHi/IGneWZp+nSnpf+p6wYh4HXhF0j7ppqHpee8BxqTbxgB31/UaZmZWN1leXhsO9I2IDQCSppIMn/3Telz3R8DNktoAL5JM3NMCuFXSycDLJCOymplZEWWdT2FH4O10uX19LxoRS4CyKnYNre+5zcys7rIkhV8Aj0t6kOSx1MH4HQIzs2YpS0fznyTNAfqnm85N+wXMzKyZyTog3kqSjmAzM2vGPCCemZnlOCmYmVlOjUlBUktJzxYrGDMzK60ak0JErAeek+RBhszMtgJZOpp3Ap6RNJ9knCIAImJkwaIyM7OSyJIU/qvgUZiZWaOQ5T2FhyTtBewdEQ9I2hZoWfjQzMys2LIMiPd9kpFMf5Nu2gO4q5BBmZlZaWR5JPUMYBDwPkBELAd2LWRQZmZWGlmSwsfplJkASGpFNRPgmJlZ05YlKTwk6adAO0n/DtwG/KWwYZmZWSlkefpoAnAy8BRwGjAD+F0hgzJrjLpOmF6wc5dfMrxg5zbbElmePtqQTqzzGEmz0XPpHMpmZtbM1JoUJA0HJgMvkMyn0E3SaRFxb6GDMzOz4srSfPRL4CsR8TyApC8C0wEnBTOzZiZLR/MHlQkh9SLwQYHiMTOzEqq2piBpVLq4UNIM4FaSPoXjgQVFiM3MzIqspuajo/OW3wCGpMurgHYFi8jMzEqm2qQQEd8tZiBmZlZ6WZ4+6gb8COiaX95DZ5uZNT9Znj66C/g9yVvMGwobjpmZlVKWpLAuIq5u6AtLagksBF6NiBFpjeQWYGdgEfCd/DGXzMys8LI8kvorSRdKOkhSv8qvBrj2mcCyvPVLgSsj4kvAOyRDa5iZWRFlqSn0Ar4DHMZnzUeRrteJpM7AcOBi4CxJSs/3rbTIVGAicF1dr2FmZlsuS1I4HvhCAzflXAWcA+yQru8MvBsRFen6CpLJfMzMrIiyNB89DezYUBeUNAJ4MyIW1fH4UyUtlLRw1apVDRWWmZmRraawI/CspAXAx5Ub6/FI6iBgpKSvAm2BzwG/AnaU1CqtLXQGXq3q4IiYAkwBKCsr82itZmYNKEtSuLAhLxgR5wHnAUg6FDg7Ik6UdBswmuQJpDHA3Q15XTMzq12W+RQeKkYgwLnALZJ+DjxO8m6EmZkVUZY3mj/gszmZ2wCtgbUR8bn6Xjwi5gBz0uUXgQH1PaeZmdVdlppC5RNCpI+OHgMMLGRQZmZWGlmePsqJxF3AEQWKx8zMSihL89GovNUWQBmwrmARmZlZyWR5+ih/XoUKoJykCcnMzJqZLH0KnlfBzGwrUdN0nBfUcFxExH8XIB4zMyuhmmoKa6vYth3J6KU7A04KZmbNTE3Tcf6yclnSDiRDXX+X5I3jX1Z3nJmZNV019ilI6gCcBZxIMpx1v4h4pxiBmZlZ8dXUp3A5MIpk8LleEbGmaFGZmVlJ1PTy2n8CuwPnA69Jej/9+kDS+8UJz8zMiqmmPoUtetvZzMyaPn/wm5lZTpY3ms2ahK4Tppc6BLMmzzUFMzPLcVIwM7McJwUzM8txUjAzsxx3NJuZFUB5228V+ArvFeSsTgpmGRX2l7wwv+BmW8rNR2ZmluOkYGZmOU4KZmaW4z4Fs0ag0G9jl18yvGDnbsqx2+ZcUzAzs5yiJwVJe0p6UNJSSc9IOjPd3kHSLEnL0+87FTs2M7OtXSlqChXAf0ZED2AgcIakHsAEYHZE7A3MTtfNzKyIip4UImJlRCxOlz8AlgF7AMeQTPlJ+v1rxY7NzGxrV9KOZkldgQOAx4BOEbEy3fU60KmaY04FTgXo0qVL4YM0awY8rLhlVbKkIGl74A7gxxHxvqTcvogISVHVcRExhWTeaMrKyqosY42XP5zMGreSPH0kqTVJQrg5Iv6cbn5D0m7p/t2AN0sRm5nZ1qzoNQUlVYLfA8si4oq8XfcAY4BL0u93FzKOpjpYlZlZIZWi+WgQ8B3gKUlL0m0/JUkGt0o6GXgZ+HoJYjMz26oVPSlExN8BVbN7aDFjMbPGr5D9UH5benN+o9nMzHKcFMzMLMcD4lmzUfiHB6y5KWjTVNuCnbqgXFMwM7McJwUzM8txUjAzsxwnBTMzy3FSMDOzHCcFMzPLcVIwM7Mcv6dgZvVS6PdDuq7734Ke3zbmmoKZmeW4pmDWCPhtbGssXFMwM7Mc1xRsM4UdD8Z/EZs1Zk4KZtaoFfIPCXdib87NR2ZmluOkYGZmOU4KZmaW4z4FM9tq+cGHzbmmYGZmOU4KZmaW4+Yj24yr1GZbLyeFpmhi+1JHYGbNVKNrPpJ0pKTnJD0vaUKp4zEz25o0qqQgqSVwLXAU0AP4pqQepY3KzGzr0aiSAjAAeD4iXoyIT4BbgGNKHJOZ2VajsfUp7AG8kre+Ajgwv4CkU4FT09U1kp6r47V2Ad6q47G1u0gFO3UVCnsvxeV7aXyay31Ac7qXi1Sfe9mruh2NLSnUKiKmAFPqex5JCyOirAFCKjnfS+PUXO6ludwH+F6yaGzNR68Ce+atd063mZlZETS2pLAA2FtSN0ltgBOAe0ock5nZVqNRNR9FRIWkHwL3Ay2B6yPimQJdrt5NUI2I76Vxai730lzuA3wvtVJEFOK8ZmbWBDW25iMzMyshJwUzM8vZ6pKCpOMlPSNpg6SyvO3/LmmRpKfS74eVMs4sqruXdN956VAhz0k6olQx1oWkvpIelbRE0kJJA0odU11J+pGkZ9N/p8tKHU99SfpPSSFpl1LHUleSLk//TZ6UdKekHUsd05Yo9FBAW11SAJ4GRgFzN9n+FnB0RPQCxgB/LHZgdVDlvaRDg5wA7A8cCfz/dAiRpuIy4KKI6AtckK43OZK+QvJGfp+I2B+YVOKQ6kXSnsAw4F+ljqWeZgE9I6I38E/gvBLHk1kxhgLa6pJCRCyLiM3ego6IxyPitXT1GaCdpG2KG92Wqe5eSD6IbomIjyPiJeB5kiFEmooAPpcutwdeq6FsY3Y6cElEfAwQEW+WOJ76uhI4h+Tfp8mKiJkRUZGuPkryPlRTUfChgLa6pJDRccDiyl/mJqiq4UL2KFEsdfFj4HJJr5D8dd1k/pLbRHfgEEmPSXpIUv9SB1RXko4BXo2IJ0odSwP7HnBvqYPYAgX/3W5U7yk0FEkPAJ+vYtfPIuLuWo7dH7iUpJpccvW5l8aspvsChgI/iYg7JH0d+D1weDHjy6qW+2gFdAAGAv2BWyV9IRrpc+C13MtPaSS/E1lk+b2R9DOgAri5mLE1ds0yKUREnT5AJHUG7gROiogXGjaquqnjvTT64UJqui9JNwJnpqu3Ab8rSlB1UMt9nA78OU0C8yVtIBmQbVWx4tsS1d2LpF5AN+AJSZD8f1osaUBEvF7EEDOr7fdG0lhgBDC0sSbpahT8d9vNR6n0CYTpwISIeLjU8dTTPcAJkraR1A3YG5hf4pi2xGvAkHT5MGB5CWOpj7uArwBI6g60oQmO0BkRT0XErhHRNSK6kjRZ9GusCaE2ko4k6RsZGREfljqeLVTwoYC2ujeaJR0LXAN0BN4FlkTEEZLOJ2m7zv8AGtaYOweru5d0389I2ksrgB9HRJNpN5X0b8CvSGqy64AfRMSi0ka15dJf2uuBvsAnwNkR8bfSRlV/ksqBsohocgkOQNLzwDbA6nTToxExroQhbRFJXwWu4rOhgC5u0PNvbUnBzMyq5+YjMzPLcVIwM7McJwUzM8txUjAzsxwnBTMzy3FSsKKQtGaT9bGSfl2qeBqSpH3TEV0fl/TFUsfTECTN2XTkXds6OCmYZSCpprf/vwbcHhEHZH0TvomNWmtbEScFKzlJN0ganbe+Jv1+aDqQ3N2SXpR0iaQTJc1P5734Ylru6HTQucclPSCpU7p9oqTr0796X5T0H9Vcf42kK9M5D2ZL6phunyPpKkkLgTMlfTmNZ5Gk+yXtlr5I9GPgdEkPpsd9O41xiaTfVCaA9Dq/lPQEcFAt5S6W9ISSeSUq76dTOv7/E+nXwTVdL+/+jpR0W976oZL+mi5fp2TOimckXVTdzydvebSkG9LljpLukLQg/RqU8Z/cGjEnBSuWdumH1hJJS4D/l/G4PsA4YD/gO0D3iBhAMh7Sj9IyfwcGRsQBJEMJn5N3/L7AESRDDl8oqXUV19gOWJjOefAQcGHevjYRUQZcTfL2+OiI+DLJm8oXR8QMYDJwZUR8RdJ+wDeAQel8EOuBE/Ou81hE9CF5m7amco+m5eYC30+3Xw08lG7vBzxTy/UqPQAcKGm7dP0b6c8JkgHiyoDewBBJvav4+VTnV+l99ycZWbjRjlFl2TXLAfGsUfoo/dACcgOSZWmzXhARK9NjXgBmptufIh1XiGRQsGmSdiMZX+ilvOOnp0OgfyzpTaATydg9+TYA09Llm4A/5+2r3L4P0BOYlQ4K1xJYWUW8Q4EvAwvScu2AyqFS1gN3ZCj3CfDXdHkR8O/p8mHASQARsR54T9J3ajgPadkKSfcBR0u6HRjOZ4nz65JOJfks2I1k4pYnq7ivqk6k970AAAG1SURBVBwO9EivC/A5SdtHxJoajrFGzknBGoMK0lqrpBYkH+yV8ue02JC3voHP/v9eA1wREfdIOhSYWM3x68n2fz5/7Je16XcBz0TEQbUcK2BqRFQ1B8S69MO8tnKf5o3cWVvMNZ0n3y3AD4G3SWpFHygZLPFsoH9EvJM2C7Wt4tj8n0f+/hYkNbR1tVzbmhA3H1ljUE7y1y7ASKCqJp6atOez4YPH1OH6LYDKPo1vkTRHbeo5oKOkgwAktVYy98amZgOjJe2alusgaa96lNv0mNPT8i0ltd+C8zxE0uT0fT5rOvocSdJ7L+23OKqa674hab80YR+bt30mnzXhIanvZkdak+OkYI3Bb0nas58ADuKzv86zmgjcJmkRdRuaei0wQNLTJE00m/V3pFMfjgYuTeNcAhxcRbmlwPnATElPkswHvFtdy23iTOArkp4iaVbqsQXXW0/SJHVU+p10FrXHgWeB/wWqGzJ+QnrMI2zcZPYfQJmkJyUtJen7sSbOo6TaVk/SmojYvtRxmDUGrimYmVmOawpmZpbjmoKZmeU4KZiZWY6TgpmZ5TgpmJlZjpOCmZnl/B+kBIvHDvgR4AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Model: <function cnn at 0x7fcdf56a3c80>\n",
            "Shape of input: (23750, 24, 14, 16)\n",
            "Hyperparameters: {'reg_amount': 0.0, 'drop_rate': 0.0, 'xs_shape': (23750, 24, 14, 16)}\n",
            "Number of training data: 50\n",
            "Number of validation data: 400\n",
            "Number of epochs: 400\n",
            "Doing training iteration no. 1\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 23, 13, 32)        2080      \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 23, 13, 16)        528       \n",
            "_________________________________________________________________\n",
            "average_pooling2d (AveragePo (None, 11, 6, 16)         0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 1056)              0         \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 1056)              0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 1)                 1057      \n",
            "=================================================================\n",
            "Total params: 3,665\n",
            "Trainable params: 3,665\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Best train accuracy: 1.0\n",
            "Number of epochs: 60\n",
            "Doing training iteration no. 2\n",
            "Doing training iteration no. 3\n",
            "Doing training iteration no. 4\n",
            "Doing training iteration no. 5\n",
            "Doing training iteration no. 6\n",
            "Doing training iteration no. 7\n",
            "Doing training iteration no. 8\n",
            "Doing training iteration no. 9\n",
            "Doing training iteration no. 10\n",
            "Results ('mean_val_auc', 0.8793354988098144, 'mean_val_accuracy', 0.8144999921321869, 'metric', 0.8469177454710006, 'val_auc_std', 0.028125135179792708, 'val_accuracy_std', 0.033124762159971566)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5xWZb338c+Xk+AhFCNSEaEeURFkoAFRtmBiiBvEVCzblbDTFLe6LbcoltvDUz6ZksfcEpWJ6U4U81CioiRCmnISj2igTol5GPEIhjrwe/5Ya25vYA6Lmbnvew7f9+s1r1nXOl2/NcPcP9Z1rXVdigjMzMwA2pU6ADMzaz6cFMzMLMdJwczMcpwUzMwsx0nBzMxyOpQ6gMb47Gc/G7179y51GGZmLcrSpUvfiojuNW1r0Umhd+/eLFmypNRhmJm1KJL+Vts2Nx+ZmVmOk4KZmeU4KZiZWU6L7lMws6bzySefsHr1atavX1/qUKyJdO7cmZ49e9KxY8fMxzgpmBkAq1evZocddqB3795IKnU41kgRwZo1a1i9ejV9+vTJfJybj8wMgPXr17Pzzjs7IbQSkth55523+s7PScHMcpwQWpeG/D6dFMzMLMd9CmZWo95T72nS81VcMrbO7WvWrGHUqFEAvP7667Rv357u3ZOXbhctWkSnTp2aNJ4sDj74YKZNm0Z5eXnR6y6VNpsUmvof/Obq+wMws03tvPPOLF++HIALL7yQ7bffnrPOOiu3vaqqig4d2uxHVtG4+cjMmq1JkyYxefJk9t9/f84++2wuvPBCpk2bltvev39/KioqALjpppsYOnQoZWVlnHzyyWzYsGGTc913330ce+yxufL8+fMZN24cAKeccgrl5eXsu+++XHDBBTXGsv322+eWZ8+ezaRJkwCorKzkmGOOYciQIQwZMoRHHnkEgIcffpiysjLKysoYNGgQH3zwQaN/HsVQsKQg6XpJb0p6Jm/dZZKel/SUpDsk7Zi37VxJqyS9IOmwQsVlZi3L6tWrefTRR7n88str3WfFihXMmjWLRx55hOXLl9O+fXtuvvnmTfY59NBDefzxx1m3bh0As2bN4rjjjgPg4osvZsmSJTz11FM8/PDDPPXUU5njO+OMM/j+97/P4sWLuf322znxxBMBmDZtGtdeey3Lly9n4cKFdOnSZWsvvSQKeadwAzBms3UPAP0jYj/gr8C5AJL6AccB+6bH/I+k9gWMzcxaiGOPPZb27ev+OJg3bx5Lly5lyJAhlJWVMW/ePF566aVN9unQoQNjxozhD3/4A1VVVdxzzz0ceeSRANx6660MHjyYQYMG8eyzz/Lcc89lju/BBx/ktNNOo6ysjPHjx/P++++zdu1ahg8fzplnnsnVV1/Nu+++22KavgoWZUQskNR7s3Vz84qPARPS5SOBWyLiI+BlSauAocBfChWfmbUM2223XW65Q4cObNy4MVeufgY/Ipg4cSI/+clP6jzXcccdx89//nO6detGeXk5O+ywAy+//DLTpk1j8eLF7LTTTkyaNKnGZ/vzH+/M375x40Yee+wxOnfuvMn+U6dOZezYscyZM4fhw4dz//33s/fee2/dxZdAKfsUvgPcmy7vBrySt211um4Lkk6StETSksrKygKHaGbNSe/evVm2bBkAy5Yt4+WXXwZg1KhRzJ49mzfffBOAt99+m7/9bcvRoUeOHMmyZcv45S9/mWs6ev/999luu+3o2rUrb7zxBvfee+8WxwH06NGDFStWsHHjRu64447c+tGjR3PNNdfkytWd5S+++CIDBgzgnHPOYciQITz//PNN8BMovJLcz0j6IVAF3FzfvpuLiBnADIDy8vJo4tDMLNUcn6A75phjuPHGG9l3333Zf//96du3LwD9+vXjxz/+MaNHj2bjxo107NiRa6+9lj322GOT49u3b8+4ceO44YYbmDlzJgADBw5k0KBB7L333uy+++4MHz68xrovueQSxo0bR/fu3SkvL2ft2rUAXH311Zx66qnst99+VFVVMWLECKZPn86VV17JQw89RLt27dh33305/PDDC/iTaTqKKNznatp89MeI6J+3bhJwMjAqIj5M150LEBE/Scv3AxdGRJ3NR+Xl5dHQSXb8SKrZplasWME+++xT6jCsidX0e5W0NCJqfPmiqM1HksYAZwPjqxNC6m7gOEnbSOoD7AksKmZsZmZWwOYjSb8DDgY+K2k1cAHJ00bbAA+knTaPRcTkiHhW0q3AcyTNSqdGxIaaz2xmZoVSyKePvlHD6l/Xsf/FwMWFisfMzOrnN5rNzCzHScHMzHKcFMzMLKdlvHdtZsV3YdcmPt979e7Svn17BgwYQFVVFfvssw8zZ85k2223bVB1kyZNYty4cUyYMIETTzyRM888k379+tW47/z58+nUqRMHHnggANOnT2fbbbfl+OOPb1DdLZnvFMys2ejSpQvLly/nmWeeoVOnTkyfPn2T7VVVVQ06769+9ataEwIkSeHRRx/NlSdPnly0hBARmwzdsXm5Ng39WdTHScHMmqWDDjqIVatWMX/+fA466CDGjx9Pv3792LBhA1OmTGHIkCHst99+/OIXvwCSD9PTTjuNvfbai0MPPTQ35AUkk+VUv+h63333MXjwYAYOHMioUaOoqKhg+vTpXHHFFZSVlbFw4cJNhuhevnw5w4YNY7/99uOoo47inXfeyZ3znHPOYejQofTt25eFCxfWeB2XXXZZLtbqYbkrKirYa6+9OP744+nfvz8LFy7cpPzKK68wZcoU+vfvz4ABA5g1axbAFj+LdevWMXbsWAYOHEj//v1z+zWGm4/MrNmpqqri3nvvZcyYZKDlZcuW8cwzz9CnTx9mzJhB165dWbx4MR999BHDhw9n9OjRPPHEE7zwwgs899xzvPHGG/Tr14/vfOc7m5y3srKS7373uyxYsIA+ffrw9ttv061bNyZPnrzJpD7z5s3LHXP88cdzzTXXMHLkSM4//3wuuugirrzyylycixYtYs6cOVx00UU8+OCDm9Q3d+5cVq5cyaJFi4gIxo8fz4IFC+jVqxcrV65k5syZDBs2jIqKik3Kt99+O8uXL+fJJ5/krbfeYsiQIYwYMWKLn8Xtt9/Orrvuyj33JCM0vPde/U109fGdgpk1G//85z8pKyujvLycXr16ccIJJwAwdOhQ+vTpAyQftDfeeCNlZWXsv//+rFmzhpUrV7JgwQK+8Y1v0L59e3bddVcOOeSQLc7/2GOPMWLEiNy5unXrVmc87733Hu+++y4jR44EYOLEiSxYsCC3/eijjwbgS1/6Um6yn3xz585l7ty5DBo0iMGDB/P888+zcuVKAPbYYw+GDRuW2ze//Oc//zl3LT169GDkyJEsXrx4i5/FgAEDeOCBBzjnnHNYuHAhXbs2vh/Idwpm1mxU9ylsLn/47Ijgmmuu4bDDNp2La86cOQWPb3PbbLMNkHSQ19TGHxGce+65nHzyyZusr6io2OSagC3Ktcnfr2/fvixbtow5c+Zw3nnnMWrUKM4///ytvYxN+E7BzFqUww47jOuuu45PPvkEgL/+9a+sW7eOESNGMGvWLDZs2MBrr73GQw89tMWxw4YNY8GCBbkht99++20Adthhhxqny+zatSs77bRTrr/gt7/9be6uIWus119/fW5E1VdffXWTvo7aHHTQQblrqaysZMGCBQwdOnSL/f7xj3+w7bbb8q1vfYspU6bkhhVvDN8pmFnNMjxCWgonnngiFRUVDB48mIige/fu3HnnnRx11FH86U9/ol+/fvTq1YsDDjhgi2O7d+/OjBkzOProo9m4cSOf+9zneOCBBzjiiCOYMGECd9111yZzIwDMnDmTyZMn8+GHH/KFL3yB3/zmN5ljHT16NCtWrMjFsv3223PTTTfVO5PcUUcdxV/+8hcGDhyIJC699FI+//nPbzEnw9NPP82UKVNo164dHTt25LrrrsscW20KOnR2oXnobLOm46GzW6dmPXS2mZk1b04KZmaW46RgZjktuTnZttSQ36eTgpkB0LlzZ9asWePE0EpEBGvWrKFz585bdZyfPjIzAHr27Mnq1auprKwsdSjWRDp37kzPnj236hgnBTMDoGPHjrk3Za3tcvORmZnlOCmYmVmOk4KZmeU4KZiZWY6TgpmZ5TgpmJlZTsGSgqTrJb0p6Zm8dd0kPSBpZfp9p3S9JF0taZWkpyQNLlRcZmZWu0LeKdwAjNls3VRgXkTsCcxLywCHA3umXycBjR//1czMtlrBkkJELADe3mz1kcDMdHkm8NW89TdG4jFgR0m7FCo2MzOrWbH7FHpExGvp8utAj3R5N+CVvP1Wp+u2IOkkSUskLfHr+GZmTatkHc2RjLq11SNvRcSMiCiPiPLu3bsXIDIzs7ar2EnhjepmofR79WSlrwK75+3XM11nZmZFVOykcDcwMV2eCNyVt/749CmkYcB7ec1MZmZWJPUmBUmXSvqMpI6S5kmqlPStDMf9DvgLsJek1ZJOAC4BviJpJXBoWgaYA7wErAJ+CfxHA6/HzMwaIcvQ2aMj4mxJRwEVwNHAAuCmug6KiG/UsmlUDfsGcGqGWMzMrICyNB9VJ46xwG0R8V4B4zEzsxLKcqfwR0nPA/8ETpHUHVhf2LDMzKwU6r1TiIipwIFAeUR8AnxI8rKZmZm1Mlk6mrcl6fitHnpiV6C8kEGZmVlpZOlT+A3wMcndAiTvD/y4YBGZmVnJZEkKX4yIS4FPACLiQ0AFjcrMzEoiS1L4WFIX0iEpJH0R+KigUZmZWUlkefroAuA+YHdJNwPDgUmFDMrMzEqj3qQQEQ9IWgYMI2k2OiMi3ip4ZGZmVnRZnj46CqiKiHsi4o9AlaSv1necmZm1PFn6FC7If4s5It4laVIyM7NWJktSqGmfLH0RZmbWwmRJCkskXS7pi+nX5cDSQgdmZmbFlyUpnE7y8tqs9OsjPKKpmVmrlOXpo3XA1CLEYmZmJVZvUpDUFzgL6J2/f0QcUriwzMysFLJ0GN8GTAd+BWwobDhmZlZKWZJCVURcV/9uZmbW0mXpaP6DpP+QtIukbtVfBY/MzMyKLsudwsT0+5S8dQF8oenDMTOzUsry9FGfYgRiZmall2nmNUnnSZqRlveUNK7woZmZWbF55jUzM8vxzGtmZpbjmdfMzCwn09DZbDrz2jzg7MZUKun7kp6V9Iyk30nqLKmPpMclrZI0S1KnxtRhZmZbr86kIKkdsBNwNMkUnL8DyiNifkMrlLQb8J/pefoD7YHjgJ8CV0TE/wHeAU5oaB1mZtYwdSaFiNgInB0Ra6pnXmuiqTg7AF0kdQC2BV4DDgFmp9tnAp7dzcysyLI0Hz0o6SxJuzfFG80R8SowDfg7STJ4j2R+hncjoirdbTWwW03HSzpJ0hJJSyorKxsahpmZ1SDLG81fT7/nz6HQ4DeaJe0EHAn0Ad4lGXBvTNbjI2IGMAOgvLw8GhKDmZnVrM6kkPYpTI2IWU1Y56HAyxFRmdbxe2A4sKOkDundQk+S9yHMzKyIsvQpTKlrnwb4OzAsfVNawCjgOeAhYEK6z0Tgriau18zM6lGKPoXHSTqUlwFPpzHMAM4BzpS0CtgZ+HVD6zAzs4Ypep8CQERcQPL+Q76XgKENPaeZmTWeR0k1M7OcLHM0H1/T+oi4senDMTOzUsrSfDQkb7kzScfwMsBJwcyslcnSfHR6flnSjsAtBYvIzMxKJsvTR5tbR/LimZmZtTJZ+hT+QDpsNkkS6QfcWsigzMysNLL0KUzLW64C/hYRqwsUj5mZlVCWpPB34LWIWA8gqYuk3hFRUdDIzMys6LL0KdwGbMwrb0jXmZlZK5MlKXSIiI+rC+myZ0UzM2uFsiSFSknjqwuSjgSaYqIdMzNrZrL0KUwGbpb087S8GqjxLWczM0v0nnpPQc9fccnYgpw3y8trL5IMdb19Wl5bkEjMzKzk6m0+kvT/JO0YEWsjYq2knST9uBjBmZlZcWXpUzg8It6tLkTEO8C/Fi4kMzMrlSxJob2kbaoLkroA29Sxv5mZtVBZOppvBuZJ+k1a/ndgZuFCMjOzUsnS0fxTSU8Ch6arfhQR9xc2LDMzK4UsdwoATwAdSQbGe6Jw4ZiZWSllefroa8AiYALwNeBxSRMKHZiZmRVfljuFHwJDIuJNAEndgQeB2YUMzMzMii/L00ftqhNCak3G48zMrIXJcqdwn6T7gd+l5a8DcwoXkpmZlUqWp4+mSDoa+Jd01YyIuKOwYZmZWSlkevooIn4P/L7AsZiZWYmVpG9A0o6SZkt6XtIKSQdI6ibpAUkr0+87lSI2M7O2rFQdxlcB90XE3sBAYAUwFZgXEXsC89KymZkVUa1JQdK89PtPm7JCSV2BEcCvIZnJLR1w70g+HT5jJvDVpqzXzMzqV1efwi6SDgTGS7oFUP7GiFjWwDr7AJXAbyQNBJYCZwA9IuK1dJ/XgR41HSzpJOAkgF69ejUwBDMzq0ldSeF84L+BnsDlm20L4JBG1DkYOD0iHpd0FZs1FUVESIqaDo6IGcAMgPLy8hr3MTOzhqk1KUTEbGC2pP+OiB81YZ2rgdUR8Xhank2SFN6QtEtEvCZpF+DNWs9gZmYFUW9Hc0T8SNJ4SdPSr3GNqTAiXgdekbRXumoU8BxwNzAxXTcRuKsx9ZiZ2dar9z0FST8BhpLMqwBwhqQDI+IHjaj3dOBmSZ2Al0jmaGgH3CrpBOBvJIPvmZlZEWV5eW0sUBYRGwEkzSQZPrvBSSEilgPlNWwa1dBzmplZ42V9T2HHvOWuhQjEzMxKL8udwk+AJyQ9RPJY6gj8YpmZWauUZUC830maDwxJV52TdhabmVkrk3VAvNdIng4yM7NWLOsczWZtXu+p9xTs3BWXjC3Yuc22hmdQMzOznDqTgqT2kp4vVjBmZlZadSaFiNgAvCDJI8+ZmbUBWfoUdgKelbQIWFe9MiLGFywqMzMriSxJ4b8LHoWZmTULWd5TeFjSHsCeEfGgpG2B9oUPzczMiq3ep48kfZdkeOtfpKt2A+4sZFBmZlYaWR5JPRUYDrwPEBErgc8VMigzMyuNLEnho4j4uLogqQPJzGtmZtbKZEkKD0v6AdBF0leA24A/FDYsMzMrhSxJYSpQCTwNnAzMAc4rZFBmZlYaWZ4+2phOrPM4SbPRCxHh5iMzs1Yoy3ScY4HpwIsk8yn0kXRyRNxb6ODMzKy4sry89jPgyxGxCkDSF4F7ACcFM7NWJkufwgfVCSH1EvBBgeIxM7MSqvVOQdLR6eISSXOAW0n6FI4FFhchNjMzK7K6mo+OyFt+AxiZLlcCXQoWkZmZlUytSSEi/r2YgZiZWellefqoD3A60Dt/fw+dbWbW+mR5+uhO4NckbzFvbKqKJbUHlgCvRsS4NPncAuwMLAW+nT+8hpmZFV6WpLA+Iq4uQN1nACuAz6TlnwJXRMQtkqYDJwDXFaBeMzOrRZZHUq+SdIGkAyQNrv5qTKWSegJjgV+lZQGHkAzRDTAT+Gpj6jAzs62X5U5hAPBtkg/t6uajSMsNdSVwNrBDWt4ZeDciqtLyapJ5G7Yg6STgJIBevTx1tJlZU8qSFI4FvtBU7fuSxgFvRsRSSQdv7fERMQOYAVBeXu4xmMzMmlCWpPAMsCPwZhPVORwYL+lfgc4kfQpXATtK6pDeLfQEXm2i+szMLKMsfQo7As9Lul/S3dVfDa0wIs6NiJ4R0Rs4DvhTRHwTeAiYkO42EbiroXWYmVnDZLlTuKDgUSTOAW6R9GPgCZLHYM3MrIiyzKfwcKEqj4j5wPx0+SVgaKHqMjOz+mV5o/kDPp2TuRPQEVgXEZ+p/SgzM2uJstwpVD82Wv0+wZHAsEIGZWZmpZGlozknEncChxUoHjMzK6EszUdH5xXbAeXA+oJFZGZmJZPl6aP8eRWqgAqSJiQzM2tlsvQpeF4FM7M2oq7pOM+v47iIiB8VIB4zMyuhuu4U1tWwbjuSIa13BpwUzMxambqm4/xZ9bKkHUjmP/h3kolwflbbcWZm1nLV2acgqRtwJvBNkjkOBkfEO8UIzMzMiq+uPoXLgKNJhqkeEBFrixaVWQP0nnpPqUMwa/HqulP4L+Aj4Dzgh8nLzACIpKPZw1yYNZFCJ7SKS8YW9PzWetTVp7BVbzubmVnL5w9+MzPLcVIwM7McJwUzM8txUjAzsxwnBTMzy3FSMDOzHCcFMzPLcVIwM7OcLJPsmFmBVXT+tyY5T+/1/9sk57HCaarfNbzXROfZlO8UzMwsx0nBzMxynBTMzCyn6H0KknYHbgR6AAHMiIir0rkbZgG9gQrga567waz5a8kjvHq49S2V4k6hCviviOgHDANOldQPmArMi4g9gXlp2czMiqjoSSEiXouIZenyB8AKYDfgSJLZ3Ui/f7XYsZmZtXUl7VOQ1BsYBDwO9IiI19JNr5M0L9V0zEmSlkhaUllZWZQ4zczaipIlBUnbA7cD34uI9/O3RUSQ9DdsISJmRER5RJR37969CJGambUdJUkKkjqSJISbI+L36eo3JO2Sbt8FeLMUsZmZtWWlePpIwK+BFRFxed6mu4GJwCXp97uKHZtZa+WnbCyrUgxzMRz4NvC0pOXpuh+QJINbJZ0A/A34WgliMzNr04qeFCLiz4Bq2TyqmLGYmdmm/EazmZnlOCmYmVmOk4KZmeV4PgWzRmq68fHNSs9JwYrKj0aaNW9uPjIzsxzfKZjZJprb1KC+uywu3ymYmVmOk4KZmeW4+chapKZo4miq5g2rWXNrhrJsfKdgZmY5TgpmZpbjpGBmZjlOCmZmluOkYGZmOU4KZmaW46RgZmY5TgpmZpbjpGBmZjl+o9mKynMPWCn47ersfKdgZmY5TgpmZpbjpGBmZjnuU7At1DapifsDzFo/JwUzs4zawn+Mml3zkaQxkl6QtErS1FLHY2bWljSrpCCpPXAtcDjQD/iGpH6ljcrMrO1oVkkBGAqsioiXIuJj4BbgyBLHZGbWZigiSh1DjqQJwJiIODEtfxvYPyJOy9vnJOCktLgX8EIDq/ss8FYjwm1OfC3NU2u5ltZyHeBrqbZHRHSvaUOL62iOiBnAjMaeR9KSiChvgpBKztfSPLWWa2kt1wG+liyaW/PRq8DueeWe6TozMyuC5pYUFgN7SuojqRNwHHB3iWMyM2szmlXzUURUSToNuB9oD1wfEc8WqLpGN0E1I76W5qm1XEtruQ7wtdSrWXU0m5lZaTW35iMzMyshJwUzM8tpc0lB0rGSnpW0UVJ53vqvSFoq6en0+yGljDOL2q4l3XZuOlTIC5IOK1WMDSGpTNJjkpZLWiJpaKljaihJp0t6Pv09XVrqeBpL0n9JCkmfLXUsDSXpsvR38pSkOyTtWOqYtkahhwJqc0kBeAY4Gliw2fq3gCMiYgAwEfhtsQNrgBqvJR0a5DhgX2AM8D/pECItxaXARRFRBpyfllscSV8meSN/YETsC0wrcUiNIml3YDTw91LH0kgPAP0jYj/gr8C5JY4ns2IMBdTmkkJErIiILd6CjognIuIfafFZoIukbYob3dap7VpIPohuiYiPIuJlYBXJECItRQCfSZe7Av+oY9/m7BTgkoj4CCAi3ixxPI11BXA2ye+nxYqIuRFRlRYfI3kfqqUo+FBAbS4pZHQMsKz6j7kF2g14Ja+8Ol3XUnwPuEzSKyT/u24x/5PbTF/gIEmPS3pY0pBSB9RQko4EXo2IJ0sdSxP7DnBvqYPYCgX/225W7yk0FUkPAp+vYdMPI+Kueo7dF/gpyW1yyTXmWpqzuq4LGAV8PyJul/Q14NfAocWML6t6rqMD0A0YBgwBbpX0hWimz4HXcy0/oJn8TWSR5e9G0g+BKuDmYsbW3LXKpBARDfoAkdQTuAM4PiJebNqoGqaB19Lshwup67ok3QickRZvA35VlKAaoJ7rOAX4fZoEFknaSDKIWWWx4tsatV2LpAFAH+BJSZD8e1omaWhEvF7EEDOr7+9G0iRgHDCquSbpWhT8b9vNR6n0CYR7gKkR8Uip42mku4HjJG0jqQ+wJ7CoxDFtjX8AI9PlQ4CVJYylMe4EvgwgqS/QiRY4QmdEPB0Rn4uI3hHRm6TJYnBzTQj1kTSGpG9kfER8WOp4tlLBhwJqc280SzoKuAboDrwLLI+IwySdR9J2nf8BNLo5dw7Wdi3pth+StJdWAd+LiBbTbirpX4CrSO5k1wP/ERFLSxvV1kv/aK8HyoCPgbMi4k+ljarxJFUA5RHR4hIcgKRVwDbAmnTVYxExuYQhbRVJ/wpcyadDAV3cpOdva0nBzMxq5+YjMzPLcVIwM7McJwUzM8txUjAzsxwnBTMzy3FSsKKQtHaz8iRJPy9VPE1J0t7piK5PSPpiqeNpCpLmbz7yrrUNTgpmGUiq6+3/rwKzI2JQ1jfhW9iotdaGOClYyUm6QdKEvPLa9PvB6UByd0l6SdIlkr4paVE678UX0/2OSAede0LSg5J6pOsvlHR9+r/elyT9Zy31r5V0RTrnwTxJ3dP18yVdKWkJcIakL6XxLJV0v6Rd0heJvgecIumh9LhvpTEul/SL6gSQ1vMzSU8CB9Sz38WSnlQyr0T19fRIx/9/Mv06sK768q5vjKTb8soHS/pjunydkjkrnpV0UW0/n7zlCZJuSJe7S7pd0uL0a3jGX7k1Y04KVixd0g+t5ZKWA/8343EDgcnAPsC3gb4RMZRkPKTT033+DAyLiEEkQwmfnXf83sBhJEMOXyCpYw11bAcsSec8eBi4IG9bp4goB64meXt8QkR8ieRN5YsjYg4wHbgiIr4saR/g68DwdD6IDcA38+p5PCIGkrxNW9d+j6X7LQC+m66/Gng4XT8YeLae+qo9COwvabu0/PX05wTJAHHlwH7ASEn71fDzqc1V6XUPIRlZuNmOUWXZtcoB8axZ+mf6oQXkBiTL0ma9OCJeS495EZibrn+adFwhkkHBZknahWR8oZfzjr8nHQL9I0lvAj1Ixu7JtxGYlS7fBPw+b1v1+r2A/sAD6aBw7YHXaoh3FPAlYHG6XxegeqiUDcDtGfb7GPhjurwU+Eq6fAhwPEBEbADek/TtOs5Dum+VpPuAIyTNBsiPqnEAAAHMSURBVMbyaeL8mqSTSD4LdiGZuOWpGq6rJocC/dJ6AT4jafuIWFvHMdbMOSlYc1BFetcqqR3JB3u1/DktNuaVN/Lpv99rgMsj4m5JBwMX1nL8BrL9m88f+2Vd+l3AsxFxQD3HCpgZETXNAbE+/TCvb79P8kburC/mus6T7xbgNOBtkruiD5QMlngWMCQi3kmbhTrXcGz+zyN/ezuSO7T19dRtLYibj6w5qCD53y7AeKCmJp66dOXT4YMnNqD+dkB1n8a/kTRHbe4FoLukAwAkdVQy98bm5gETJH0u3a+bpD0asd/mx5yS7t9eUtetOM/DJE1O3+XTpqPPkCS999J+i8NrqfcNSfukCfuovPVz+bQJD0llWxxpLY6TgjUHvyRpz34SOIBP/3ee1YXAbZKW0rChqdcBQyU9Q9JEs0V/Rzr14QTgp2mcy4EDa9jvOeA8YK6kp0jmA96loftt5gzgy5KeJmlW6rcV9W0gaZI6PP1OOovaE8DzwP8CtQ0ZPzU95lE2bTL7T6Bc0lOSniPp+7EWzqOkWpsnaW1EbF/qOMyaA98pmJlZju8UzMwsx3cKZmaW46RgZmY5TgpmZpbjpGBmZjlOCmZmlvP/AYNzkXkrIqicAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Model: <function cnn at 0x7fcdf56a3c80>\n",
            "Shape of input: (23750, 12, 28, 16)\n",
            "Hyperparameters: {'reg_amount': 0.0, 'drop_rate': 0.0, 'xs_shape': (23750, 12, 28, 16)}\n",
            "Number of training data: 50\n",
            "Number of validation data: 400\n",
            "Number of epochs: 400\n",
            "Doing training iteration no. 1\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 11, 27, 32)        2080      \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 11, 27, 16)        528       \n",
            "_________________________________________________________________\n",
            "average_pooling2d (AveragePo (None, 5, 13, 16)         0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 1040)              0         \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 1040)              0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 1)                 1041      \n",
            "=================================================================\n",
            "Total params: 3,649\n",
            "Trainable params: 3,649\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Best train accuracy: 1.0\n",
            "Number of epochs: 58\n",
            "Doing training iteration no. 2\n",
            "Doing training iteration no. 3\n",
            "Doing training iteration no. 4\n",
            "Doing training iteration no. 5\n",
            "Doing training iteration no. 6\n",
            "Doing training iteration no. 7\n",
            "Doing training iteration no. 8\n",
            "Doing training iteration no. 9\n",
            "Doing training iteration no. 10\n",
            "Results ('mean_val_auc', 0.8932569205760956, 'mean_val_accuracy', 0.8285000026226044, 'metric', 0.86087846159935, 'val_auc_std', 0.017591903831538216, 'val_accuracy_std', 0.01997498529837964)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5yVZb338c+Xk+AhFBtJRQR9PCEncUCURzAxxEBMRNNdAaUhZm3LRLDaHp50Z8r2mFsiMzHdiWEeSjSQREhTTuIJNFBxO4qKmAdQ1IHf88e6Z7GAOdxzWGvNDN/36zWvuc/X7x6Y9Zvruu77uhQRmJmZAbQodgBmZtZ4OCmYmVmWk4KZmWU5KZiZWZaTgpmZZbUqdgD18cUvfjG6dOlS7DDMzJqUxYsXvxsRJZXta9JJoUuXLixatKjYYZiZNSmSXqtqn5uPzMwsy0nBzMyynBTMzCyrSfcpmFnD+fzzzykrK2PDhg3FDsUaSNu2benUqROtW7dOfY6TgpkBUFZWxi677EKXLl2QVOxwrJ4igrVr11JWVkbXrl1Tn+fmIzMDYMOGDey+++5OCM2EJHbfffda1/ycFMwsywmheanLv6eTgpmZZblPwcwq1WXSgw16vVVXDqt2/9q1axk8eDAAb731Fi1btqSkJPPS7YIFC2jTpk2DxpPGMcccw+TJkyktLS142cWy3SaFhv4Pv7WafgHMbEu77747S5cuBeDSSy9l55135oILLsjuLy8vp1Wr7fYjq2DcfGRmjdbYsWMZP348RxxxBBdeeCGXXnopkydPzu7v3r07q1atAuCOO+6gX79+9O7dm7PPPpuNGzduca2HH36YU089Nbs+d+5chg8fDsA555xDaWkphx56KJdcckmlsey8887Z5RkzZjB27FgA1qxZwymnnELfvn3p27cvjz/+OACPPfYYvXv3pnfv3hx22GF89NFH9f55FIKTgpk1amVlZTzxxBNcc801VR6zfPlypk+fzuOPP87SpUtp2bIld9555xbHHHfccTz11FOsX78egOnTp3P66acDcMUVV7Bo0SKeffZZHnvsMZ599tnU8Z133nn86Ec/YuHChdxzzz2cddZZAEyePJmbbrqJpUuXMn/+fNq1a1fbWy8K18XMrFE79dRTadmyZbXHzJkzh8WLF9O3b18APvnkE/bYY48tjmnVqhVDhw7lz3/+M6NGjeLBBx/kqquuAuDuu+9m6tSplJeXs3r1apYtW0bPnj1TxffII4+wbNmy7PqHH37IunXrGDBgAOeffz7f+MY3GDlyJJ06darNbReNk4KZNWo77bRTdrlVq1Zs2rQpu17xDH5EMGbMGH7xi19Ue63TTz+dX/3qV3To0IHS0lJ22WUXXn31VSZPnszChQvZbbfdGDt2bKXP9uc+3pm7f9OmTTz55JO0bdt2i+MnTZrEsGHDmDlzJgMGDOCvf/0rBx98cO1uvgjcfGRmTUaXLl1YsmQJAEuWLOHVV18FYPDgwcyYMYN33nkHgPfee4/XXtt2dOhBgwaxZMkSfvOb32Sbjj788EN22mkn2rdvz9tvv81DDz1UadkdO3Zk+fLlbNq0iXvvvTe7fciQIdx4443Z9YrO8pdffpkePXowceJE+vbty4svvtgAP4H8y1tNQdKtwHDgnYjovtW+HwOTgZKIeFeZFHw98FXgY2BsRCzJV2xmVrPG+ATdKaecwu23386hhx7KEUccwYEHHghAt27duPzyyxkyZAibNm2idevW3HTTTey7775bnN+yZUuGDx/ObbfdxrRp0wDo1asXhx12GAcffDD77LMPAwYMqLTsK6+8kuHDh1NSUkJpaSnr1q0D4IYbbuDcc8+lZ8+elJeXM3DgQKZMmcJ1113Ho48+SosWLTj00EM54YQT8viTaTiKiPxcWBoIrANuz00KkvYBbgEOBg5PksJXgR+QSQpHANdHxBE1lVFaWhp1nWTHj6SabWn58uUccsghxQ7DGlhl/66SFkdEpS9f5K35KCLmAe9Vsuta4EIgNxudRCZ5REQ8Cewqac98xWZmZpUraJ+CpJOANyLima127Q28nrNelmwzM7MCKtjTR5J2BH4CDKnndcYB4wA6d+7cAJGZmVmFQtYU9ge6As9IWgV0ApZI+hLwBrBPzrGdkm3biIipEVEaEaUV46KYmVnDKFhSiIjnImKPiOgSEV3INBH1iYi3gAeA0croD3wQEasLFZuZmWXkLSlI+gPwD+AgSWWSzqzm8JnAK8BK4DfA9/IVl5mZVS1vfQoRcUYN+7vkLAdwbr5iMbM6uLR9A1/vgxoPadmyJT169KC8vJxDDjmEadOmseOOO9apuLFjxzJ8+HBGjRrFWWedxfnnn0+3bt0qPXbu3Lm0adOGo446CoApU6aw4447Mnr06DqV3ZT5jWYzazTatWvH0qVLef7552nTpg1TpkzZYn95eXmdrnvLLbdUmRAgkxSeeOKJ7Pr48eMLlhAiYouhO7Zer0pdfxY1cVIws0bp6KOPZuXKlcydO5ejjz6aESNG0K1bNzZu3MiECRPo27cvPXv25Ne//jWQ+TD9/ve/z0EHHcRxxx2XHfICMpPlVLzo+vDDD9OnTx969erF4MGDWbVqFVOmTOHaa6+ld+/ezJ8/f4shupcuXUr//v3p2bMnJ598Mv/617+y15w4cSL9+vXjwAMPZP78+ZXex9VXX52NtWJY7lWrVnHQQQcxevRounfvzvz587dYf/3115kwYQLdu3enR48eTJ8+HWCbn8X69esZNmwYvXr1onv37tnj6sMD4plZo1NeXs5DDz3E0KFDgcw4R88//zxdu3Zl6tSptG/fnoULF/Lpp58yYMAAhgwZwtNPP81LL73EsmXLePvtt+nWrRvf+c53trjumjVr+O53v8u8efPo2rUr7733Hh06dGD8+PFbTOozZ86c7DmjR4/mxhtvZNCgQVx88cVcdtllXHfdddk4FyxYwMyZM7nssst45JFHtihv1qxZrFixggULFhARjBgxgnnz5tG5c2dWrFjBtGnT6N+/P6tWrdpi/Z577mHp0qU888wzvPvuu/Tt25eBAwdu87O455572GuvvXjwwcwIDR98UHMTXU1cUzCzRuOTTz6hd+/elJaW0rlzZ848M/N8Sr9+/ejatSuQ+aC9/fbb6d27N0cccQRr165lxYoVzJs3jzPOOIOWLVuy1157ceyxx25z/SeffJKBAwdmr9WhQ4dq4/nggw94//33GTRoEABjxoxh3rx52f0jR44E4PDDD89O9pNr1qxZzJo1i8MOO4w+ffrw4osvsmLFCgD23Xdf+vfvnz02d/3vf/979l46duzIoEGDWLhw4TY/ix49ejB79mwmTpzI/Pnzad++/v1ArimYWaNR0aewtdzhsyOCG2+8keOPP36LY2bOnJn3+La2ww47AJkO8sra+COCiy66iLPPPnuL7atWrdrinoBt1quSe9yBBx7IkiVLmDlzJj/72c8YPHgwF198cW1vYwuuKZhZk3L88cdz88038/nnnwPwz3/+k/Xr1zNw4ECmT5/Oxo0bWb16NY8++ug25/bv35958+Zlh9x+773M8Gy77LJLpdNltm/fnt122y3bX/D73/8+W2tIG+utt96aHVH1jTfe2KKvoypHH3109l7WrFnDvHnz6Nev3zbHvfnmm+y4445885vfZMKECdlhxevDNQUzq1yKR0iL4ayzzmLVqlX06dOHiKCkpIT77ruPk08+mb/97W9069aNzp07c+SRR25zbklJCVOnTmXkyJFs2rSJPfbYg9mzZ3PiiScyatQo7r///i3mRgCYNm0a48eP5+OPP2a//fbjd7/7XepYhwwZwvLly7Ox7Lzzztxxxx01ziR38skn849//INevXohiauuuoovfelL28zJ8NxzzzFhwgRatGhB69atufnmm1PHVpW8DZ1dCB4626zheOjs5qnRDJ1tZmZNj5OCmZllOSmYWVZTbk62bdXl39NJwcwAaNu2LWvXrnViaCYigrVr19K2bdtaneenj8wMgE6dOlFWVsaaNWuKHYo1kLZt29KpU6daneOkYGYAtG7dOvumrG2/3HxkZmZZTgpmZpblpGBmZllOCmZmluWkYGZmWU4KZmaWlbekIOlWSe9Iej5n29WSXpT0rKR7Je2as+8iSSslvSTp+MqvamZm+ZTPmsJtwNCtts0GukdET+CfwEUAkroBpwOHJuf8t6Tqx5Y1M7MGl7ekEBHzgPe22jYrIiqmJ3oSqHjV7iTgroj4NCJeBVYC284oYWZmeVXMPoXvAA8ly3sDr+fsK0u2bUPSOEmLJC3y6/hmZg2rKElB0k+BcuDO2p4bEVMjojQiSktKSho+ODOz7VjBxz6SNBYYDgyOzcMxvgHsk3NYp2SbmZkVUEFrCpKGAhcCIyLi45xdDwCnS9pBUlfgAGBBIWMzM7MUSUHSVZK+IKm1pDmS1kj6Zorz/gD8AzhIUpmkM4FfAbsAsyUtlTQFICJeAO4GlgEPA+dGxMZ63JeZmdVBmuajIRFxoaSTgVXASGAecEd1J0XEGZVs/m01x18BXJEiHjMzy5M0zUcViWMY8MeI+CCP8ZiZWRGlqSn8RdKLwCfAOZJKgA35DcvMzIqhxppCREwCjgJKI+Jz4GMyL5uZmVkzk6ajeUfge8DNyaa9gNJ8BmVmZsWRpk/hd8BnZGoLkHl/4PK8RWRmZkWTJinsHxFXAZ8DJO8XKK9RmZlZUaRJCp9JagcEgKT9gU/zGpWZmRVFmqePLiHzQtk+ku4EBgBj8xmUmZkVR41JISJmS1oC9CfTbHReRLyb98jMzKzg0jx9dDJQHhEPRsRfgHJJX8t/aGZmVmhp+hQuyX2LOSLeJ9OkZGZmzUyapFDZMQUfctvMzPIvTVJYJOkaSfsnX9cAi/MdmJmZFV6apPADMi+vTU++PgXOzWdQZmZWHGmePloPTCpALGZmVmQ1JgVJBwIXAF1yj4+IY/MXlpmZFUOaDuM/AlOAWwDPhmZm1oylSQrlEXFzzYeZmVlTl6aj+c+SvidpT0kdKr7yHpmZmRVcmqQwBpgAPEHmUdTFwKKaTpJ0q6R3JD2fs62DpNmSViTfd0u2S9INklZKelZSn7rdjpmZ1Ueamde6VvK1X4pr3wYM3WrbJGBORBwAzGHzU00nAAckX+PYPKGPmZkVUKqZ1yT9TNLUZP0AScNrOi8i5gHvbbX5JGBasjwN+FrO9tsj40lgV0l7pr0JMzNrGIWeea1jRKxOlt8COibLewOv5xxXlmzbhqRxkhZJWrRmzZo6hmFmZpUp2sxrEREkE/fU8rypEVEaEaUlJSX1DcPMzHIUeua1tyuahZLv7yTb3wD2yTmuU7LNzMwKKNXQ2Ww589oc4MI6lvcAmaeZSL7fn7N9dPIUUn/gg5xmJjMzK5BqX16T1ALYDRhJLWdek/QH4Bjgi5LKyCSXK4G7JZ0JvAaclhw+E/gqsBL4GPh2XW7GzMzqp9qkEBGbJF0YEXcDD9bmwhFxRhW7BldybOCRV83Mii5N89Ejki6QtI/faDYza97SjH309eR77l/yAaR5gc3MzJqQNH0KkyJieoHiMTOzIqq2+SgiNpEZ98jMzLYD7lMwM7Ms9ymYmVlWmjmauxYiEDMzK740czSPrmx7RNze8OGYmVkxpWk+6puz3JbMy2dLACcFM7NmJk3z0Q9y1yXtCtyVt4jMzKxo0jx9tLX1gPsZzMyaoTR9Cn9m87wHLYBuwN35DMrMzIojTZ/C5JzlcuC1iCjLUzxmZlZEaZLC/wKrI2IDgKR2krpExKq8RmZmZgWXpk/hj8CmnPWNyTYzM2tm0iSFVhHxWcVKstwmfyGZmVmxpEkKaySNqFiRdBJQ48xrZmbW9KTpUxgP3CnpV8l6GVDpW85mZta0pXl57WWgv6Sdk/V1eY/KzMyKosbmI0n/KWnXiFgXEesk7Sbp8voUKulHkl6Q9LykP0hqK6mrpKckrZQ0XZL7LczMCixNn8IJEfF+xUpE/Av4al0LlLQ38O9AaUR0B1oCpwO/BK6NiP8D/As4s65lmJlZ3aRJCi0l7VCxIqkdsEM1x6fRCmgnqRWwI7AaOBaYkeyfBnytnmWYmVktpelovhOYI+l3yfq3yXxo10lEvCFpMpmX4j4BZgGLgfcjojw5rAzYu7LzJY0DxgF07ty5rmGYmVklaqwpRMQvgcuBQ5Kvn0fEVXUtUNJuwElkBtXbC9gJGJr2/IiYGhGlEVFaUlJS1zDMzKwSaWoKAE8DrckMjPd0Pcs8Dng1ItYASPoTMADYVVKrpLbQCXijnuWYmVktpXn66DRgATAKOA14StKoepT5v2Qecd1RkshM2rMMeDQpA2AMcH89yjAzszpIU1P4KdA3It4BkFQCPMLmTuFaiYinJM0gM3tbOZmax1TgQeCu5HHXp4Hf1uX6ZmZWd2mSQouKhJBYS90m58mKiEuAS7ba/ArQrz7XNTOz+kmTFB6W9FfgD8n614GZ+QvJzMyKJc0wFxMkjQT+b7JpakTcm9+wzMysGFI9fRQRfwL+lOdYzMysyOrVN2BmZs2Lk4KZmWVVmRQkzUm+/7Jw4ZiZWTFV16ewp6SjgBGS7gKUuzMiluQ1MjMzK7jqksLFwH+QGXLimq32BZlRTc3MrBmpMilExAxghqT/iIifFzAmMzMrkjTvKfxc0ghgYLJpbkT8Jb9hmZlZMaQZEO8XwHlkBq1bBpwn6T/zHZiZmRVempfXhgG9I2ITgKRpZAas+0k+AzMzs8JL+57CrjnL7fMRiJmZFV+amsIvgKclPUrmsdSBwKS8RmVmZkWRpqP5D5LmAn2TTRMj4q28RmVmZkWRdkC81cADeY7FzMyKzGMfmZlZlpOCmZllVZsUJLWU9GKhgjEzs+KqNilExEbgJUmdG7JQSbtKmiHpRUnLJR0pqYOk2ZJWJN93a8gyzcysZmmaj3YDXpA0R9IDFV/1LPd64OGIOBjoBSwn85jrnIg4AJiDH3s1Myu4NE8f/UdDFiipPZl3HcYCRMRnwGeSTgKOSQ6bBswFJjZk2WZmVr0aawoR8RiwCmidLC8E6jOXQldgDfA7SU9LukXSTkDH5NFXgLeAjpWdLGmcpEWSFq1Zs6YeYZiZ2dbSDIj3XWAG8Otk097AffUosxXQB7g5Ig4D1rNVU1FEBJk5G7YREVMjojQiSktKSuoRhpmZbS1N89G5QD/gKYCIWCFpj3qUWQaURcRTyfoMMknhbUl7RsRqSXsC79SjDDOzouoy6cG8Xn/VlcPyct00Hc2fJu3+AEhqRRV/xaeRDJHxuqSDkk2DyQzJ/QAwJtk2Bri/rmWYmVndpKkpPCbpJ0A7SV8Bvgf8uZ7l/gC4U1Ib4BXg22QS1N2SzgReA06rZxlmZlZLaZLCJOBM4DngbGAmcEt9Co2IpUBpJbsG1+e6ZmZWP2lGSd2UTKzzFJlmo5eSjmAzM2tmakwKkoYBU4CXycyn0FXS2RHxUL6DMzOzwkrTfPRfwJcjYiWApP2BBwEnBTOzZibN00cfVSSExCvAR3mKx8zMiqjKmoKkkcniIkkzgbvJ9CmcSuatZjMza2aqaz46MWf5bWBQsrwGaJe3iMzMrGiqTAoR8e1CBmJmZsWX5umjrmReNuuSe3xEjMhfWGZmVgxpnj66D/gtmbeYN+U3HDMzK6Y0SWFDRNyQ90jMzKzo0iSF6yVdAswCPq3YGBH1mVPBzMwaoTRJoQfwLeBYNjcfRbJuZmbNSJqkcCqwX+7w2WZm1jylSQrPA7viSW9sO5fPSVPyNWGKWW2lSQq7Ai9KWsiWfQp+JNXMrJlJkxQuyXsUZmbWKKSZT+GxQgRiZmbFl+aN5o/YPCdzG6A1sD4ivpDPwMzMrPDS1BR2qViWJOAkoH8+gzIzs+JIM59CVmTcBxxf34IltZT0tKS/JOtdJT0laaWk6ZLa1LcMMzOrnTTNRyNzVlsApcCGBij7PGA5UNEM9Uvg2oi4S9IU4Ezg5gYox8zMUkpTUzgx5+t4MrOunVSfQiV1AoYBtyTrIvOG9IzkkGnA1+pThpmZ1V6aPoV8zKtwHXAhUNFfsTvwfkSUJ+tlwN6VnShpHDAOoHPnznkIzcxs+1XddJwXV3NeRMTP61KgpOHAOxGxWNIxtT0/IqYCUwFKS0ujhsPNzKwWqqsprK9k205k2vp3B+qUFIABwAhJXwXakulTuB7YVVKrpLbQCXijjtc3M7M6qrJPISL+q+KLzF/m7YBvA3cB+9W1wIi4KCI6RUQX4HTgbxHxDeBRYFRy2Bjg/rqWYWZmdVNtR7OkDpIuB54lU6voExETIyIfg+NNBM6XtJJMTeS3eSjDzMyqUV2fwtXASDK1hB4Rsa6hC4+IucDcZPkVoF9Dl2FmZulVV1P4MbAX8DPgTUkfJl8fSfqwMOGZmVkhVVlTiIhave1sZmZNnz/4zcwsy0nBzMyy0kyyY9Yk5HO6TLPthWsKZmaW5aRgZmZZTgpmZpblpGBmZllOCmZmluWnj8wagXw/ObXqymF5vb41H64pmJlZlpOCmZllOSmYmVmWk4KZmWU5KZiZWZaTgpmZZTkpmJlZlpOCmZllFTwpSNpH0qOSlkl6QdJ5yfYOkmZLWpF8363QsZmZbe+K8UZzOfDjiFgiaRdgsaTZwFhgTkRcKWkSMAmYWIT4zApuVdt/y3MJH+T5+tZcFDwpRMRqYHWy/JGk5cDewEnAMclh04C5OCmYWRPVVBN9UfsUJHUBDgOeAjomCQPgLaBjkcIyM9tuFW1APEk7A/cAP4yIDyVl90VESIoqzhsHjAPo3LlzIUI1a/LyOeCeB9trXoqSFCS1JpMQ7oyIPyWb35a0Z0SslrQn8E5l50bEVGAqQGlpaaWJw8wKxyO8Ni/FePpIwG+B5RFxTc6uB4AxyfIY4P5Cx2Zmtr0rRk1hAPAt4DlJS5NtPwGuBO6WdCbwGnBaEWIzM9uuFePpo78DqmL34ELGYoWX76YGM6sfv9FsZmZZTgpmZpblpGBmZllOCmZmllW0l9fMmpr8D1tgVnyuKZiZWZZrCmbWqHmIjsJyUjCz7VZeE07bvF06r9x8ZGZmWa4pWLPhjmCz+nNSMNsO5DNhdtnwP3m7thWem4/MzCzLNQUzq5d8N9u5JlJYrimYmVmWk4KZmWU5KZiZWZaTgpmZZW23Hc35f6b9gzxfP3/y+5an3yWw2vHjtIXlmoKZmWU5KZiZWVajaz6SNBS4HmgJ3BIRVxY5JDNrptycua1GlRQktQRuAr4ClAELJT0QEcuKG1nt5bNd3swsXxpb81E/YGVEvBIRnwF3AScVOSYzs+1Go6opAHsDr+eslwFH5B4gaRwwLlldJ+mlOpb1ReDdOp6bwvD8XXpbeb6XhqOaD2ky95JCc7mX5nIf0Jzu5TLV5172rWpHY0sKNYqIqcDU+l5H0qKIKG2AkIrO99I4NZd7aS73Ab6XNBpb89EbwD45652SbWZmVgCNLSksBA6Q1FVSG+B04IEix2Rmtt1oVM1HEVEu6fvAX8k8knprRLyQp+Lq3QTViPheGqfmci/N5T7A91IjRUQ+rmtmZk1QY2s+MjOzInJSMDOzrO0uKUg6VdILkjZJKs3Z/hVJiyU9l3w/tphxplHVvST7LpK0UtJLko4vVox1Iam3pCclLZW0SFK/YsdUV5J+IOnF5N/pqmLHU1+SfiwpJH2x2LHUlaSrk3+TZyXdK2nXYsdUG5KGJr/XKyVNaujrb3dJAXgeGAnM22r7u8CJEdEDGAP8vtCB1UGl9yKpG5kntw4FhgL/nQwh0lRcBVwWEb2Bi5P1JkfSl8m8kd8rIg4FJhc5pHqRtA8wBPjfYsdST7OB7hHRE/gncFGR40ktZyigE4BuwBnJ73uD2e6SQkQsj4ht3oKOiKcj4s1k9QWgnaQdChtd7VR1L2Q+iO6KiE8j4lVgJZkhRJqKAL6QLLcH3qzm2MbsHODKiPgUICLeKXI89XUtcCGZf58mKyJmRUR5svokmfehmoq8DwW03SWFlE4BllT8MjdBlQ0XsneRYqmLHwJXS3qdzF/XTeYvua0cCBwt6SlJj0nqW+yA6krSScAbEfFMsWNpYN8BHip2ELWQ99/tRvWeQkOR9AjwpUp2/TQi7q/h3EOBX5KpJhddfe6lMavuvoDBwI8i4h5JpwG/BY4rZHxp1XAfrYAOQH+gL3C3pP2ikT4HXsO9/IRG8juRRprfG0k/BcqBOwsZW2PXLJNCRNTpA0RSJ+BeYHREvNywUdVNHe+l0Q8XUt19SbodOC9Z/SNwS0GCqoMa7uMc4E9JElggaROZAdnWFCq+2qjqXiT1ALoCz0iCzP+nJZL6RcRbBQwxtZp+bySNJTNq5eDGmqSrkPffbTcfJZInEB4EJkXE48WOp54eAE6XtIOkrsABwIIix1QbbwKDkuVjgRVFjKU+7gO+DCDpQKANTXCEzoh4LiL2iIguEdGFTJNFn8aaEGqSTOR1ITAiIj4udjy1lPehgLa7N5olnQzcCJQA7wNLI+J4ST8j03ad+wE0pDF3DlZ1L8m+n5JpLy0HfhgRTabdVNL/JTP7XitgA/C9iFhc3KhqL/mlvRXoDXwGXBARfytuVPUnaRVQGhFNLsEBSFoJ7ACsTTY9GRHjixhSrUj6KnAdm4cCuqJBr7+9JQUzM6uam4/MzCzLScHMzLKcFMzMLMtJwczMspwUzMwsy0nBCkLSuq3Wx0r6VbHiaUiSDk5GdH1a0v7FjqchSJq79ci7tn1wUjBLQVJ1b/9/DZgREYelfRO+iY1aa9sRJwUrOkm3SRqVs74u+X5MMpDc/ZJekXSlpG9IWpDMe7F/ctyJyaBzT0t6RFLHZPulkm5N/up9RdK/V1H+OknXJnMezJFUkmyfK+k6SYuA8yQdnsSzWNJfJe2ZvEj0Q+AcSY8m530ziXGppF9XJICknP+S9AxwZA3HXSHpGWXmlai4n47J+P/PJF9HVVdezv0NlfTHnPVjJP0lWb5ZmTkrXpB0WVU/n5zlUZJuS5ZLJN0jaWHyNSDlP7k1Yk4KVijtkg+tpZKWAv8v5Xm9gPHAIcC3gAMjoqQhG7kAAALzSURBVB+Z8ZB+kBzzd6B/RBxGZijhC3POPxg4nsyQw5dIal1JGTsBi5I5Dx4DLsnZ1yYiSoEbyLw9PioiDifzpvIVETETmAJcGxFflnQI8HVgQDIfxEbgGznlPBURvci8TVvdcU8mx80DvptsvwF4LNneB3ihhvIqPAIcIWmnZP3ryc8JMgPElQI9gUGSelby86nK9cl99yUzsnCjHaPK0muWA+JZo/RJ8qEFZAckS9NmvTAiVifnvAzMSrY/RzKuEJlBwaZL2pPM+EKv5pz/YDIE+qeS3gE6khm7J9cmYHqyfAfwp5x9FdsPAroDs5NB4VoCqyuJdzBwOLAwOa4dUDFUykbgnhTHfQb8JVleDHwlWT4WGA0QERuBDyR9q5rrkBxbLulh4ERJM4BhbE6cp0kaR+azYE8yE7c8W8l9VeY4oFtSLsAXJO0cEeuqOccaOScFawzKSWqtklqQ+WCvkDunxaac9U1s/v97I3BNRDwg6Rjg0irO30i6//O5Y7+sT74LeCEijqzhXAHTIqKyOSA2JB/mNR33ec7InTXFXN11ct0FfB94j0yt6CNlBku8AOgbEf9KmoXaVnJu7s8jd38LMjW0DTWUbU2Im4+sMVhF5q9dgBFAZU081WnP5uGDx9Sh/BZARZ/Gv5FpjtraS0CJpCMBJLVWZu6Nrc0BRknaIzmug6R963Hc1ueckxzfUlL7WlznMTJNTt9lc9PRF8gkvQ+SfosTqij3bUmHJAn75Jzts9jchIek3tucaU2Ok4I1Br8h0579DHAkm/86T+tS4I+SFlO3oanXA/0kPU+miWab/o5k6sNRwC+TOJcCR1Vy3DLgZ8AsSc+SmQ94z7oet5XzgC9Leo5Ms1K3WpS3kUyT1AnJd5JZ1J4GXgT+B6hqyPhJyTlPsGWT2b8DpZKelbSMTN+PNXEeJdW2e5LWRcTOxY7DrDFwTcHMzLJcUzAzsyzXFMzMLMtJwczMspwUzMwsy0nBzMyynBTMzCzr/wN0jgk2c++oogAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Model: <function cnn at 0x7fcdf56a3c80>\n",
            "Shape of input: (23750, 12, 14, 32)\n",
            "Hyperparameters: {'reg_amount': 0.0, 'drop_rate': 0.0, 'xs_shape': (23750, 12, 14, 32)}\n",
            "Number of training data: 50\n",
            "Number of validation data: 400\n",
            "Number of epochs: 400\n",
            "Doing training iteration no. 1\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 11, 13, 32)        4128      \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 11, 13, 16)        528       \n",
            "_________________________________________________________________\n",
            "average_pooling2d (AveragePo (None, 5, 6, 16)          0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 480)               0         \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 480)               0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 1)                 481       \n",
            "=================================================================\n",
            "Total params: 5,137\n",
            "Trainable params: 5,137\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Best train accuracy: 1.0\n",
            "Number of epochs: 37\n",
            "Doing training iteration no. 2\n",
            "Doing training iteration no. 3\n",
            "Doing training iteration no. 4\n",
            "Doing training iteration no. 5\n",
            "Doing training iteration no. 6\n",
            "Doing training iteration no. 7\n",
            "Doing training iteration no. 8\n",
            "Doing training iteration no. 9\n",
            "Doing training iteration no. 10\n",
            "Results ('mean_val_auc', 0.8686852812767029, 'mean_val_accuracy', 0.8247500061988831, 'metric', 0.846717643737793, 'val_auc_std', 0.06532306181400693, 'val_accuracy_std', 0.04938940212156422)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5xVdb3/8ddbLgJeUBRJRYQ8oiI3aUCUI5gYYiImoumxhNIQ07JMBKvj5VeevJB3j0RmYnoSxUpNVJRESOMu3kADdUwUFfEKiTrw+f2x1mw3MJfNzOy95/J+Ph7zmLW+6/L9rBlmf1jf71rfryICMzMzgG2KHYCZmdUfTgpmZpbhpGBmZhlOCmZmluGkYGZmGc2LHUBt7LrrrtG5c+dih2Fm1qAsWrTo3YhoX9G2Bp0UOnfuzMKFC4sdhplZgyLptcq2ufnIzMwynBTMzCzDScHMzDIadJ+CmdWdzz//nJUrV7J+/fpih2J1pFWrVnTs2JEWLVrkfIyTgpkBsHLlSnbYYQc6d+6MpGKHY7UUEaxZs4aVK1fSpUuXnI9z85GZAbB+/Xp22WUXJ4RGQhK77LLLVt/5OSmYWYYTQuNSk99n3pKCpFslvSPp+Qq2/URSSNo1XZek6yWtkPSspD75isvMzCqXzz6F24AbgduzCyXtBQwB/pVVfDSwb/p1MHBz+t3MiqTzhAfr9Hyllx9T5fY1a9YwePBgAN566y2aNWtG+/bJS7fz58+nZcuWdRpPLg4//HAmTpxISUlJwesulrwlhYiYLalzBZuuAS4A7ssqOw64PZIZf+ZK2knS7hGxKl/x1fU/+M1V9wdgZpvaZZddWLJkCQCXXHIJ22+/Peeff35me1lZGc2b+9mYfCton4Kk44A3IuKZzTbtCbyetb4yLavoHGMkLZS0cPXq1XmK1Mzqg9GjRzN27FgOPvhgLrjgAi655BImTpyY2d69e3dKS0sBuOOOO+jXrx+9e/fmzDPPZMOGDZuc6+GHH+bEE0/MrM+aNYthw4YBcNZZZ1FSUsKBBx7IxRdfXGEs22+/fWZ52rRpjB49GoDVq1dzwgkn0LdvX/r27cuTTz4JwBNPPEHv3r3p3bs3Bx10EB9//HGtfx6FULCkIKkN8FPgotqcJyImR0RJRJSU31qaWeO1cuVKnnrqKa6++upK91m2bBlTp07lySefZMmSJTRr1ow777xzk32OPPJI5s2bx7p16wCYOnUqJ598MgCXXXYZCxcu5Nlnn+WJJ57g2WefzTm+c889lx//+McsWLCAe++9lzPOOAOAiRMnctNNN7FkyRLmzJlD69att/bSi6KQ92L7AF2AZ9Ie8Y7AYkn9gDeAvbL27ZiWmVkTd+KJJ9KsWbMq95k5cyaLFi2ib9++AHzyySfstttum+zTvHlzhg4dygMPPMDIkSN58MEHufLKKwG4++67mTx5MmVlZaxatYqlS5fSs2fPnOJ77LHHWLp0aWb9o48+Yu3atQwYMIDzzjuPU089lREjRtCxY8etueyiKVhSiIjngMxvSVIpUBIR70q6HzhH0l0kHcwf5rM/wcwaju222y6z3Lx5czZu3JhZL38GPyIYNWoUv/rVr6o818knn8yNN95Iu3btKCkpYYcdduDVV19l4sSJLFiwgJ133pnRo0dX+Gx/9uOd2ds3btzI3LlzadWq1Sb7T5gwgWOOOYbp06czYMAAHnnkEfbff/+tu/giyOcjqX8E/gHsJ2mlpNOr2H068AqwAvgt8P18xWVmDVfnzp1ZvHgxAIsXL+bVV18FYPDgwUybNo133nkHgPfee4/XXttydOhBgwaxePFifvvb32aajj766CO222472rZty9tvv81DDz1UYd0dOnRg2bJlbNy4kT//+c+Z8iFDhnDDDTdk1ss7y19++WV69OjB+PHj6du3Ly+++GId/ATyL59PH51SzfbOWcsBnJ2vWMxs69XHJ+hOOOEEbr/9dg488EAOPvhgunbtCkC3bt345S9/yZAhQ9i4cSMtWrTgpptuYu+9997k+GbNmjFs2DBuu+02pkyZAkCvXr046KCD2H///dlrr70YMGBAhXVffvnlDBs2jPbt21NSUsLatWsBuP766zn77LPp2bMnZWVlDBw4kEmTJnHttdfy+OOPs80223DggQdy9NFH5/EnU3eUfB43TCUlJVHTSXb8SKrZppYtW8YBBxxQ7DCsjlX0e5W0KCIqfPnCw1yYmVmGk4KZmWU4KZiZWYaTgpmZZTgpmJlZhpOCmZlleMhBM6vYJW3r+HwfVrtLs2bN6NGjB2VlZRxwwAFMmTKFNm3a1Ki60aNHM2zYMEaOHMkZZ5zBeeedR7du3Srcd9asWbRs2ZJDDz0UgEmTJtGmTRtOO+20GtXdkPlOwczqjdatW7NkyRKef/55WrZsyaRJkzbZXlZWVqPz3nLLLZUmBEiSwlNPPZVZHzt2bMESQkRsMnTH5uuVqenPojpOCmZWLx122GGsWLGCWbNmcdhhhzF8+HC6devGhg0bGDduHH379qVnz5785je/AZIP03POOYf99tuPI488MjPkBSST5ZS/6Prwww/Tp08fevXqxeDBgyktLWXSpElcc8019O7dmzlz5mwyRPeSJUvo378/PXv25Pjjj+f999/PnHP8+PH069ePrl27MmfOnAqv46qrrsrEWj4sd2lpKfvttx+nnXYa3bt3Z86cOZusv/7664wbN47u3bvTo0cPpk6dCrDFz2LdunUcc8wx9OrVi+7du2f2qw03H5lZvVNWVsZDDz3E0KFDgWSco+eff54uXbowefJk2rZty4IFC/j0008ZMGAAQ4YM4emnn+all15i6dKlvP3223Tr1o3vfve7m5x39erVfO9732P27Nl06dKF9957j3bt2jF27NhNJvWZOXNm5pjTTjuNG264gUGDBnHRRRdx6aWXcu2112binD9/PtOnT+fSSy/lscce26S+GTNmsHz5cubPn09EMHz4cGbPnk2nTp1Yvnw5U6ZMoX///pSWlm6yfu+997JkyRKeeeYZ3n33Xfr27cvAgQO3+Fnce++97LHHHjz4YDJCw4cfVt9EVx3fKZhZvfHJJ5/Qu3dvSkpK6NSpE6efnoyj2a9fP7p06QIkH7S33347vXv35uCDD2bNmjUsX76c2bNnc8opp9CsWTP22GMPjjjiiC3OP3fuXAYOHJg5V7t27aqM58MPP+SDDz5g0KBBAIwaNYrZs2dnto8YMQKAr3zlK5nJfrLNmDGDGTNmcNBBB9GnTx9efPFFli9fDsDee+9N//79M/tmr//973/PXEuHDh0YNGgQCxYs2OJn0aNHDx599FHGjx/PnDlzaNu29v1AvlMws3qjvE9hc9nDZ0cEN9xwA0cdddQm+0yfPj3v8W1u2223BZIO8ora+COCCy+8kDPPPHOT8tLS0k2uCdhivTLZ+3Xt2pXFixczffp0fv7znzN48GAuuqhW85j5TsHMGpajjjqKm2++mc8//xyAf/7zn6xbt46BAwcydepUNmzYwKpVq3j88ce3OLZ///7Mnj07M+T2e++9B8AOO+xQ4XSZbdu2Zeedd870F/zhD3/I3DXkGuutt96aGVH1jTfe2KSvozKHHXZY5lpWr17N7Nmz6dev3xb7vfnmm7Rp04ZvfetbjBs3LjOseG34TsHMKpbDI6TFcMYZZ1BaWkqfPn2ICNq3b89f/vIXjj/+eP72t7/RrVs3OnXqxCGHHLLFse3bt2fy5MmMGDGCjRs3sttuu/Hoo49y7LHHMnLkSO67775N5kYAmDJlCmPHjuXf//43X/7yl/n973+fc6xDhgxh2bJlmVi233577rjjjmpnkjv++OP5xz/+Qa9evZDElVdeyZe+9KUt5mR47rnnGDduHNtssw0tWrTg5ptvzjm2ynjo7Dzx0NnW0Hjo7MbJQ2ebmVmNOSmYmVmGk4KZZTTk5mTbUk1+n04KZgZAq1atWLNmjRNDIxERrFmzhlatWm3VcXl7+kjSrcAw4J2I6J6WXQUcC3wGvAx8JyI+SLddCJwObAB+GBGP5Cs2M9tSx44dWblyJatXry52KFZHWrVqRceOHbfqmHw+knobcCNwe1bZo8CFEVEm6QrgQmC8pG7AycCBwB7AY5K6RsSGPMZnZllatGiReVPWmq68NR9FxGzgvc3KZkRE+Wt/c4HyFHYccFdEfBoRrwIrgC3f1DAzs7wqZp/Cd4GH0uU9gdeztq1My8zMrICKkhQk/QwoA+6swbFjJC2UtNBtn2ZmdavgSUHSaJIO6FPji8cc3gD2ytqtY1q2hYiYHBElEVHSvn37vMZqZtbUFDQpSBoKXAAMj4h/Z226HzhZ0raSugD7AvMLGZuZmeX3kdQ/AocDu0paCVxM8rTRtsCjkgDmRsTYiHhB0t3AUpJmpbP95JGZWeHlLSlExCkVFP+uiv0vAy7LVzxmZlY9v9FsZmYZTgpmZpbhpGBmZhlOCmZmluGkYGZmGU4KZmaWUW1SkHSlpB0ltZA0U9JqSd8qRHBmZlZYudwpDImIj0iGpigF/gMYl8+gzMysOHJJCuUvuB0D3BMRH+YxHjMzK6Jc3mj+q6QXgU+AsyS1B9bnNywzMyuGau8UImICcChQEhGfA/8mmRTHzMwamVw6mtsA3wduTov2AEryGZSZmRVHLn0Kvwc+I7lbgGSeg1/mLSIzMyuaXJLCPhFxJfA5QDoPgvIalZmZFUUuSeEzSa2BAJC0D/BpXqMyM7OiyOXpo4uBh4G9JN0JDABG5zMoMzMrjmqTQkQ8Kmkx0J+k2ejciHg375GZmVnB5fL00fFAWUQ8GBF/BcokfSP/oZmZWaHl0qdwcfZbzBHxAUmTkpmZNTK5JIWK9snb3M5mZlY8uSSFhZKulrRP+nU1sCjfgZmZWeHlkhR+QPLy2tT061Pg7OoOknSrpHckPZ9V1k7So5KWp993Tssl6XpJKyQ9K6lPzS7HzMxqI5exj9ZFxISIKEm/LoyIdTmc+zZg6GZlE4CZEbEvMDNdBzga2Df9GsMXQ2qYmVkBVds3IKkrcD7QOXv/iDiiquMiYrakzpsVHwccni5PAWYB49Py2yMigLmSdpK0e0SsyuUizMysbuTSYXwPMAm4BdhQy/o6ZH3QvwV0SJf3BF7P2m9lWrZFUpA0huRugk6dOtUyHDMzy5ZLUiiLiDpvzomIkBQ1OG4yMBmgpKRkq483M7PK5dLR/ICk70vaPe0obiepXQ3re1vS7gDp93fS8jeAvbL265iWmZlZAeWSFEaRzMn8FMmjqIuAhTWs7/70fOXnvS+r/LT0KaT+wIfuTzAzK7xcxj7qUpMTS/ojSafyrpJWkrwFfTlwt6TTgdeAk9LdpwNfB1aQzOz2nZrUaWZmtZPL00dtgPOAThExRtK+wH7pOEiViohTKtk0uIJ9gxzefTAzs/zyzGtmZpbhmdfMzCzDM6+ZmVmGZ14zM7OMKpOCpG2AnYEReOY1M7NGr8qkEBEbJV0QEXcDDxYoJjMzK5Jc+hQek3S+pL3q4I1mMzOrx3LpU/hm+j37PYIAvlz34ZiZWTHl0qcwISKmFigeMzMroiqbjyJiI8m4R2Zm1gS4T8HMzDLcp2BmZhl5GyXVzMwanlxGST2tovKIuL3uwzEzs2LKpfmob9ZyK5KhrxcDTgpmZo1MLs1HP8hel7QTcFfeIjIzs6LJ5emjza0D3M9gZtYI5dKn8ADpsNkkSaQbcHc+gzIzs+LIpU9hYtZyGfBaRKzMUzxmZlZEuSSFfwGrImI9gKTWkjpHRGleIzMzs4LLpU/hHmBj1vqGtKzGJP1Y0guSnpf0R0mtJHWRNE/SCklTJbWsTR1mZrb1ckkKzSPis/KVdLnGH9iS9gR+CJRERHegGXAycAVwTUT8B/A+cHpN6zAzs5rJJSmsljS8fEXScUBtZ15rDrSW1BxoA6wCjgCmpdunAN+oZR1mZraVculTGAvcKenGdH0lUOFbzrmIiDckTSTpq/gEmAEsAj6IiLKsOvas6HhJY4AxAJ06dappGGZmVoFcXl57Gegvaft0fW1tKpS0M3AcybsOH5D0TwzN9fiImAxMBigpKYlqdjczs61QbfORpP+RtFNErI2ItZJ2lvTLWtR5JPBqRKyOiM+BPwEDgJ3S5iSAjsAbtajDzMxqIJc+haMj4oPylYh4H/h6Ler8F8mdRxtJIhlLaSnwODAy3WcUcF8t6jAzsxrIJSk0k7Rt+Yqk1sC2VexfpYiYR9KhvBh4Lo1hMjAeOE/SCmAX4Hc1rcPMzGoml47mO4GZkn6frn+H5OmgGouIi4GLNyt+BehXm/OamVnt5NLRfIWkZ0j6AgB+ERGP5DcsMzMrhlzuFACeBlqQDIz3dP7CMTOzYsrl6aOTgPkkncAnAfMkjaz6KDMza4hyuVP4GdA3It4BkNQeeIwv3j42M7NGIpenj7YpTwipNTkeZ2ZmDUwudwoPS3oE+GO6/k1gev5CMjOzYsnl6aNxkkYA/5kWTY6IP+c3LDMzK4acnj6KiD+RDEdhZmaNmPsGzMwsw0nBzMwyKk0Kkmam368oXDhmZlZMVfUp7C7pUGC4pLsAZW+MiMV5jczMzAquqqRwEfDfJHMbXL3ZtiCZPtPMzBqRSpNCREwDpkn674j4RQFjMjOzIsnlPYVfSBoODEyLZkXEX/MblpmZFUMuA+L9CjiXZHa0pcC5kv4n34GZmVnh5fLy2jFA74jYCCBpCsnw2T/NZ2BmZlZ4ub6nsFPWctt8BGJmZsWXy53Cr4CnJT1O8ljqQGBCXqMyM7OiyKWj+Y+SZgF906LxEfFWXqMyM7OiyHVAvFXA/XmOxczMiqwoYx9J2knSNEkvSlom6RBJ7SQ9Kml5+n3nYsRmZtaUFWtAvOuAhyNif6AXsIykn2JmROwLzMT9FmZmBVdlUpDUTNKLdVmhpLYkndW/A4iIzyLiA+A4YEq62xTgG3VZr5mZVa/KpBARG4CXJHWqwzq7AKuB30t6WtItkrYDOqR9FwBvAR0qOljSGEkLJS1cvXp1HYZlZma5dDTvDLwgaT6wrrwwIobXos4+wA8iYp6k69isqSgiQlJUdHBETAYmA5SUlFS4j5lZsXWe8GBez196+TF5OW8uSeG/67jOlcDKiJiXrk8jSQpvS9o9IlZJ2h14p47rNTOzalTb0RwRTwClQIt0eQFQ47kU0nccXpe0X1o0mGRMpfuBUWnZKOC+mtZhZmY1U+2dgqTvAWOAdsA+wJ7AJJIP85r6AXCnpJbAK8B3SBLU3ZJOB14DTqrF+c3MrAZyaT46G+gHzAOIiOWSdqtNpRGxBCipYFNtEo2ZmdVSLu8pfBoRn5WvSGpOMvOamZk1MrkkhSck/RRoLelrwD3AA/kNy8zMiiGXpDCB5L2C54AzgenAz/MZlJmZFUcuo6RuTCfWmUfSbPRSRLj5yMysEcrl6aNjSJ42eplkPoUuks6MiIfyHZyZmRVWLk8f/Rr4akSsAJC0D/Ag4KRgZtbI5NKn8HF5Qki9Anycp3jMzKyIKr1TkDQiXVwoaTpwN0mfwokkbzWbmVkjU1Xz0bFZy28Dg9Ll1UDrvEVkZmZFU2lSiIjvFDIQMzMrvlyePupCMlZR5+z9azF0tpmZ1VO5PH30F5JZ0h4ANuY3HDMzK6ZcksL6iLg+75GYmVnR5ZIUrpN0MTAD+LS8MCJqPKeCmZnVT7kkhR7At4Ej+KL5KNJ1MzNrRHJJCicCX84ePtvMzBqnXJLC88BOeM5ka+LyORF7viZhN9tauSSFnYAXJS1g0z4FP5JqZtbI5JIULs57FGZmVi/kMp/CE4UIxMzMiq/aUVIlfSzpo/RrvaQNkj6qbcWSmkl6WtJf0/UukuZJWiFpqqSWta3DzMy2TrVJISJ2iIgdI2JHkoHwTgD+tw7qPhdYlrV+BXBNRPwH8D5weh3UYWZmWyGX+RQyIvEX4KjaVCqpI3AMcEu6LpL3Hqalu0wBvlGbOszMbOvlMiDeiKzVbYASYH0t670WuADYIV3fBfggIsrS9ZXAnpXEMwYYA9CpU6dahmFmZtlyefooe16FMqAUOK6mFUoaBrwTEYskHb61x0fEZGAyQElJSdQ0DjMz21IuTx/V9bwKA4Dhkr4OtAJ2BK4DdpLUPL1b6Ai8Ucf1mplZNaqajvOiKo6LiPhFTSqMiAuBC9M6DgfOj4hTJd0DjATuAkYB99Xk/GZmVnNVdTSvq+ALkqeCxuchlvHAeZJWkPQx/C4PdZiZWRWqmo7z1+XLknYgeYT0OyT/k/91ZcdtjYiYBcxKl18B+tXFec3MrGaq7FOQ1A44DziV5DHRPhHxfiECMzOzwquqT+EqYATJkz49ImJtwaIyM7OiqKpP4SfAHsDPgTezhrr4uC6GuTAzs/qnqj6FrXrb2czMGr5cXl4zszzL5wQ+4El8LHe+GzAzswwnBTMzy3BSMDOzDCcFMzPLcFIwM7MMJwUzM8twUjAzswwnBTMzy3BSMDOzDCcFMzPLcFIwM7MMj31k1gTkc2wlj6vUuPhOwczMMnynYAXl/7Ga1W9OCtZo5Hv4abOmoOBJQdJewO1AByCAyRFxXTof9FSgM1AKnOT5oK2pKG31X7U6vvP6/6ujSKypK0afQhnwk4joBvQHzpbUDZgAzIyIfYGZ6bqZmRVQwZNCRKyKiMXp8sfAMmBP4DhgSrrbFOAbhY7NzKypK+rTR5I6AwcB84AOEbEq3fQWSfNSRceMkbRQ0sLVq1cXJE4zs6aiaElB0vbAvcCPIuKj7G0REST9DVuIiMkRURIRJe3bty9ApGZmTUdRkoKkFiQJ4c6I+FNa/Lak3dPtuwPvFCM2M7OmrOBJQZKA3wHLIuLqrE33A6PS5VHAfYWOzcysqSvGewoDgG8Dz0lakpb9FLgcuFvS6cBrwElFiM3MrEkreFKIiL8DqmTz4ELGYmZNm1943JLHPjIzswwnBTMzy3BSMDOzDA+IZ2b1mtv9C8t3CmZmluE7BTOzIqjtyLjwYZ3EsTknBTOrFTfvNC5uPjIzswwnBTMzy3BSMDOzDCcFMzPLcFIwM7MMJwUzM8twUjAzswy/p2BmtXqRqvP6/6vDSKzYnBRsC34ZyazpcvORmZllOCmYmVmGk4KZmWW4T8GsjtR+1Euz4vOdgpmZZdS7OwVJQ4HrgGbALRFxeZFDMrM8asiPwzbGu8N6lRQkNQNuAr4GrAQWSLo/IpYWN7L6pTE/MlrMP7Jif8CY1Qf1rfmoH7AiIl6JiM+Au4DjihyTmVmToYgodgwZkkYCQyPijHT928DBEXFO1j5jgDHp6n7ASzWsblfg3VqEW5/4WuqnxnItjeU6wNdSbu+IaF/RhnrVfJSLiJgMTK7teSQtjIiSOgip6Hwt9VNjuZbGch3ga8lFfWs+egPYK2u9Y1pmZmYFUN+SwgJgX0ldJLUETgbuL3JMZmZNRr1qPoqIMknnAI+QPJJ6a0S8kKfqat0EVY/4WuqnxnItjeU6wNdSrXrV0WxmZsVV35qPzMysiJwUzMwso8klBUknSnpB0kZJJVnlX5O0SNJz6fcjihlnLiq7lnTbhZJWSHpJ0lHFirEmJPWWNFfSEkkLJfUrdkw1JekHkl5Mf09XFjue2pL0E0khaddix1JTkq5KfyfPSvqzpJ2KHdPWkDQ0/bteIWlCXZ+/ySUF4HlgBDB7s/J3gWMjogcwCvhDoQOrgQqvRVI3kie3DgSGAv+bDiHSUFwJXBoRvYGL0vUGR9JXSd7I7xURBwITixxSrUjaCxgC/KvYsdTSo0D3iOgJ/BO4sMjx5CxrKKCjgW7AKenfe51pckkhIpZFxBZvQUfE0xHxZrr6AtBa0raFjW7rVHYtJB9Ed0XEpxHxKrCCZAiRhiKAHdPltsCbVexbn50FXB4RnwJExDtFjqe2rgEuIPn9NFgRMSMiytLVuSTvQzUUeR8KqMklhRydACwu/2NugPYEXs9aX5mWNRQ/Aq6S9DrJ/64bzP/kNtMVOEzSPElPSOpb7IBqStJxwBsR8UyxY6lj3wUeKnYQWyHvf9v16j2FuiLpMeBLFWz6WUTcV82xBwJXkNwmF11trqU+q+q6gMHAjyPiXkknAb8DjixkfLmq5jqaA+2A/kBf4G5JX456+hx4NdfyU+rJ30Qucvm7kfQzoAy4s5Cx1XeNMilERI0+QCR1BP4MnBYRL9dtVDVTw2up98OFVHVdkm4Hzk1X7wFuKUhQNVDNdZwF/ClNAvMlbSQZxGx1oeLbGpVdi6QeQBfgGUmQ/HtaLKlfRLxVwBBzVt3fjaTRwDBgcH1N0pXI+9+2m49S6RMIDwITIuLJYsdTS/cDJ0vaVlIXYF9gfpFj2hpvAoPS5SOA5UWMpTb+AnwVQFJXoCUNcITOiHguInaLiM4R0ZmkyaJPfU0I1Ukn8roAGB4R/y52PFsp70MBNbk3miUdD9wAtAc+AJZExFGSfk7Sdp39ATSkPncOVnYt6bafkbSXlgE/iogG024q6T9JZt9rDqwHvh8Ri4ob1dZL/2hvBXoDnwHnR8TfihtV7UkqBUoiosElOABJK4BtgTVp0dyIGFvEkLaKpK8D1/LFUECX1en5m1pSMDOzyrn5yMzMMpwUzMwsw0nBzMwynBTMzCzDScHMzDKcFKwgJK3dbH20pBuLFU9dkrR/OqLr05L2KXY8dUHSrM1H3rWmwUnBLAeSqnr7/xvAtIg4KNc34RvYqLXWhDgpWNFJuk3SyKz1ten3w9OB5O6T9IqkyyWdKml+Ou/FPul+x6aDzj0t6TFJHdLySyTdmv6v9xVJP6yk/rWSrknnPJgpqX1aPkvStZIWAudK+koazyJJj0jaPX2R6EfAWZIeT4/7VhrjEkm/KU8AaT2/lvQMcEg1+10m6Rkl80qUX0+HdPz/Z9KvQ6uqL+v6hkq6J2v9cEl/TZdvVjJnxQuSLq3s55O1PFLSbelye0n3SlqQfg3I8Vdu9ZiTghVK6/RDa4mkJcD/y/G4XsBY4ADg20DXiOhHMh7SD9J9/g70j4iDSIYSviDr+P2Bo0iGHL5YUosK6tgOWJjOefAEcHHWtpYRUQJcT/L2+MiI+DRLwwoAAAK5SURBVArJm8qXRcR0YBJwTUR8VdIBwDeBAel8EBuAU7PqmRcRvUjepq1qv7npfrOB76Xl1wNPpOV9gBeqqa/cY8DBkrZL17+Z/pwgGSCuBOgJDJLUs4KfT2WuS6+7L8nIwvV2jCrLXaMcEM/qpU/SDy0gMyBZLm3WCyJiVXrMy8CMtPw50nGFSAYFmyppd5LxhV7NOv7BdAj0TyW9A3QgGbsn20Zgarp8B/CnrG3l5fsB3YFH00HhmgGrKoh3MPAVYEG6X2ugfKiUDcC9Oez3GfDXdHkR8LV0+QjgNICI2AB8KOnbVZyHdN8ySQ8Dx0qaBhzDF4nzJEljSD4LdieZuOXZCq6rIkcC3dJ6AXaUtH1ErK3iGKvnnBSsPigjvWuVtA3JB3u57DktNmatb+SLf783AFdHxP2SDgcuqeT4DeT2bz577Jd16XcBL0TEIdUcK2BKRFQ0B8T69MO8uv0+zxq5s7qYqzpPtruAc4D3SO6KPlYyWOL5QN+IeD9tFmpVwbHZP4/s7duQ3KGtr6Zua0DcfGT1QSnJ/3YBhgMVNfFUpS1fDB88qgb1bwOU92n8F0lz1OZeAtpLOgRAUgslc29sbiYwUtJu6X7tJO1di/02P+asdP9mktpuxXmeIGly+h5fNB3tSJL0Pkz7LY6upN63JR2QJuzjs8pn8EUTHpJ6b3GkNThOClYf/JakPfsZ4BC++N95ri4B7pG0iJoNTb0O6CfpeZImmi36O9KpD0cCV6RxLgEOrWC/pcDPgRmSniWZD3j3mu63mXOBr0p6jqRZqdtW1LeBpEnq6PQ76SxqTwMvAv8HVDZk/IT0mKfYtMnsh0CJpGclLSXp+7EGzqOkWpMnaW1EbF/sOMzqA98pmJlZhu8UzMwsw3cKZmaW4aRgZmYZTgpmZpbhpGBmZhlOCmZmlvH/AbnxBw121oE4AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Model: <function cnn at 0x7fcdf56a3c80>\n",
            "Shape of input: (23750, 12, 14, 80)\n",
            "Hyperparameters: {'reg_amount': 0.0, 'drop_rate': 0.0, 'xs_shape': (23750, 12, 14, 80)}\n",
            "Number of training data: 50\n",
            "Number of validation data: 400\n",
            "Number of epochs: 400\n",
            "Doing training iteration no. 1\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 11, 13, 32)        10272     \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 11, 13, 16)        528       \n",
            "_________________________________________________________________\n",
            "average_pooling2d (AveragePo (None, 5, 6, 16)          0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 480)               0         \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 480)               0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 1)                 481       \n",
            "=================================================================\n",
            "Total params: 11,281\n",
            "Trainable params: 11,281\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Best train accuracy: 1.0\n",
            "Number of epochs: 33\n",
            "Doing training iteration no. 2\n",
            "Doing training iteration no. 3\n",
            "Doing training iteration no. 4\n",
            "Doing training iteration no. 5\n",
            "Doing training iteration no. 6\n",
            "Doing training iteration no. 7\n",
            "Doing training iteration no. 8\n",
            "Doing training iteration no. 9\n",
            "Doing training iteration no. 10\n",
            "Results ('mean_val_auc', 0.8871894717216492, 'mean_val_accuracy', 0.8284999907016755, 'metric', 0.8578447312116624, 'val_auc_std', 0.02655174341386464, 'val_accuracy_std', 0.028332845354556074)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEICAYAAACwDehOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZyWdb3/8ddbBMElFB1JRYQ8bsgmDYhyFBNDDdRENM0STEM8Vpa5YHkUf+XJlNRcjkRpYnoSl3JJVJBESFM2cQMNlzFRVMQVXAc+vz+ua25vYJaLmbnve5b38/GYx1z793PNMPeH7/d7Xd+vIgIzMzOAjUodgJmZNR1OCmZmluOkYGZmOU4KZmaW46RgZmY5TgpmZpZTsKQg6XpJb0l6ppp9P5UUkrZJ1yXpSkkvSHpKUr9CxWVmZjXbuIDXvgG4Grgxf6OkHYGhwL/zNh8K7JJ+7Q1cm36v1TbbbBPdunVrnGjNzFqJ+fPnvx0RZdXtK1hSiIhZkrpVs+ty4GzgrrxtRwA3RvIm3WOStpS0XUQsq62Mbt26MW/evMYK2cysVZD0Sk37itqnIOkI4LWIeHKdXTsAr+atL023mZlZERWy+WgtkjYFfkbSdNSQ64wBxgB07dq1ESIzM7Mqxawp7Ax0B56UVAF0ARZI+jLwGrBj3rFd0m3riYhJEVEeEeVlZdU2iZmZWT0VraYQEU8D21atp4mhPCLelnQ38ANJt5B0ML9fV3+CmTWuzz//nKVLl/LJJ5+UOhRrJO3bt6dLly60bds28zkFSwqS/gwcAGwjaSlwQURcV8PhU4FvAC8AHwEnFiouM6ve0qVL2WKLLejWrRuSSh2ONVBEsGLFCpYuXUr37t0zn1fIp4+Oq2N/t7zlAE4rVCxmVrdPPvnECaEFkcTWW2/N8uXLN+g8v9FsZjlOCC1LfX6fTgpmZpZTtI5mM2teuo27t1GvV3HxsFr3r1ixgiFDhgDwxhtv0KZNG6qeMJwzZw7t2rVr1HiyOOCAA5gwYQLl5eVFL7tUWm1SaOx/8Ouq6w/AzNa29dZbs3DhQgDGjx/P5ptvzplnnpnbX1lZycYbt9qPrKJx85GZNVmjR49m7Nix7L333px99tmMHz+eCRMm5Pb37NmTiooKAG666SYGDBhA3759OeWUU1i9evVa17r//vs5+uijc+szZ85k+PDhAJx66qmUl5ez5557csEFF1Qby+abb55bvv322xk9ejQAy5cv56ijjqJ///7079+fRx55BICHH36Yvn370rdvX/baay8+/PDDBv88isFJwcyatKVLl/Loo49y2WWX1XjM4sWLmTJlCo888ggLFy6kTZs23HzzzWsdc9BBB/H444+zatUqAKZMmcKxxx4LwEUXXcS8efN46qmnePjhh3nqqacyx3f66afzk5/8hLlz53LHHXdw8sknAzBhwgSuueYaFi5cyOzZs+nQocOG3npJuC5mZk3a0UcfTZs2bWo9ZsaMGcyfP5/+/fsD8PHHH7PtttuudczGG2/MIYccwj333MPIkSO59957ueSSSwC49dZbmTRpEpWVlSxbtoxFixbRu3fvTPE9+OCDLFq0KLf+wQcfsHLlSgYNGsQZZ5zB8ccfz4gRI+jSpcuG3HbJOCmYWZO22Wab5ZY33nhj1qxZk1uvevs6Ihg1ahS/+tWvar3Wsccey9VXX02nTp0oLy9niy224OWXX2bChAnMnTuXrbbaitGjR1f7Vnf+4535+9esWcNjjz1G+/bt1zp+3LhxDBs2jKlTpzJo0CAeeOABdt999w27+RJw85GZNRvdunVjwYIFACxYsICXX34ZgCFDhnD77bfz1ltvAfDOO+/wyivrjw49ePBgFixYwO9///tc09EHH3zAZpttRseOHXnzzTe57777qi27c+fOLF68mDVr1vDXv/41t33o0KFcddVVufWqzvIXX3yRXr16cc4559C/f3+ee+65RvgJFJ5rCmZWrab4BN1RRx3FjTfeyJ577snee+/NrrvuCkCPHj345S9/ydChQ1mzZg1t27blmmuuYaeddlrr/DZt2jB8+HBuuOEGJk+eDECfPn3Ya6+92H333dlxxx0ZNGhQtWVffPHFDB8+nLKyMsrLy1m5ciUAV155Jaeddhq9e/emsrKS/fffn4kTJ3LFFVfw0EMPsdFGG7Hnnnty6KGHFvAn03iUjDDRPJWXl0d9J9nxI6lma1u8eDF77LFHqcOwRlbd71XS/Iio9uULNx+ZmVmOk4KZmeU4KZiZWY6TgpmZ5TgpmJlZjpOCmZnl+D0FM6ve+I6NfL336zykTZs29OrVi8rKSvbYYw8mT57MpptuWq/iRo8ezfDhwxk5ciQnn3wyZ5xxBj169Kj22JkzZ9KuXTv23XdfACZOnMimm27KCSecUK+ymzPXFMysyejQoQMLFy7kmWeeoV27dkycOHGt/ZWVlfW67h/+8IcaEwIkSeHRRx/NrY8dO7ZoCSEi1hq6Y931mtT3Z1EXJwUza5L2228/XnjhBWbOnMl+++3H4YcfTo8ePVi9ejVnnXUW/fv3p3fv3vzud78Dkg/TH/zgB+y2224cdNBBuSEvIJksp+pF1/vvv59+/frRp08fhgwZQkVFBRMnTuTyyy+nb9++zJ49e60huhcuXMjAgQPp3bs3Rx55JO+++27umueccw4DBgxg1113Zfbs2dXex6WXXpqLtWpY7oqKCnbbbTdOOOEEevbsyezZs9daf/XVVznrrLPo2bMnvXr1YsqUKQDr/SxWrVrFsGHD6NOnDz179swd1xBuPjKzJqeyspL77ruPQw45BEjGOXrmmWfo3r07kyZNomPHjsydO5dPP/2UQYMGMXToUJ544gmef/55Fi1axJtvvkmPHj343ve+t9Z1ly9fzve//31mzZpF9+7deeedd+jUqRNjx45da1KfGTNm5M454YQTuOqqqxg8eDDnn38+F154IVdccUUuzjlz5jB16lQuvPBCHnzwwbXKmzZtGkuWLGHOnDlEBIcffjizZs2ia9euLFmyhMmTJzNw4EAqKirWWr/jjjtYuHAhTz75JG+//Tb9+/dn//33X+9ncccdd7D99ttz773JCA3vv193E11dXFMwsybj448/pm/fvpSXl9O1a1dOOukkAAYMGED37t2B5IP2xhtvpG/fvuy9996sWLGCJUuWMGvWLI477jjatGnD9ttvz4EHHrje9R977DH233//3LU6depUazzvv/8+7733HoMHDwZg1KhRzJo1K7d/xIgRAHz1q1/NTfaTb9q0aUybNo299tqLfv368dxzz7FkyRIAdtppJwYOHJg7Nn/9H//4R+5eOnfuzODBg5k7d+56P4tevXoxffp0zjnnHGbPnk3Hjg3vBypYTUHS9cBw4K2I6JluuxQ4DPgMeBE4MSLeS/edC5wErAZ+FBEPFCo2M2uaqvoU1pU/fHZEcNVVV3HwwQevdczUqVMLHt+6NtlkEyDpIK+ujT8iOPfccznllFPW2l5RUbHWPQHrrdck/7hdd92VBQsWMHXqVM477zyGDBnC+eefv6G3sZZC1hRuAA5ZZ9t0oGdE9Ab+BZwLIKkHcCywZ3rO/0qqfVYNM2uVDj74YK699lo+//xzAP71r3+xatUq9t9/f6ZMmcLq1atZtmwZDz300HrnDhw4kFmzZuWG3H7nnXcA2GKLLaqdLrNjx45stdVWuf6CP/3pT7laQ9ZYr7/++tyIqq+99tpafR012W+//XL3snz5cmbNmsWAAQPWO+71119n00035Tvf+Q5nnXVWbljxhihYTSEiZknqts62aXmrjwEj0+UjgFsi4lPgZUkvAAOAfxYqPjOrQ4ZHSEvh5JNPpqKign79+hERlJWVceedd3LkkUfy97//nR49etC1a1f22Wef9c4tKytj0qRJjBgxgjVr1rDtttsyffp0DjvsMEaOHMldd9211twIAJMnT2bs2LF89NFHfOUrX+GPf/xj5liHDh3K4sWLc7Fsvvnm3HTTTXXOJHfkkUfyz3/+kz59+iCJSy65hC9/+cvrzcnw9NNPc9ZZZ7HRRhvRtm1brr322syx1aSgQ2enSeFvVc1H6+y7B5gSETdJuhp4LCJuSvddB9wXEbfXdn0PnW3WeDx0dsvULIbOlvRzoBK4ua5jqzl3jKR5kuYtX7688YMzM2vFip4UJI0m6YA+Pr6oprwG7Jh3WJd023oiYlJElEdEeVlZWUFjNTNrbYqaFCQdApwNHB4RH+Xtuhs4VtImkroDuwBzihmbmSVPy1jLUZ/fZ8GSgqQ/k3QU7yZpqaSTgKuBLYDpkhZKmggQEc8CtwKLgPuB0yJidaFiM7P1tW/fnhUrVjgxtBARwYoVK2jfvv0GnVfIp4+Oq2bzdbUcfxFwUaHiMbPadenShaVLl+K+upajffv2dOnSZYPO8TAXZgZA27Ztc2/KWuvlYS7MzCzHScHMzHKcFMzMLMdJwczMcpwUzMwsx0nBzMxynBTMzCzHScHMzHKcFMzMLMdJwczMcpwUzMwsx0nBzMxy6kwKki6R9CVJbSXNkLRc0neKEZyZmRVXlprC0Ij4gGS2tArgP4CzChmUmZmVRpakUDW89jDgtoh4v4DxmJlZCWWZT+Fvkp4DPgZOlVQGfFLYsMzMrBTqrClExDhgX6A8Ij4HPgKOKHRgZmZWfFk6mjcF/gu4Nt20PVBeyKDMzKw0svQp/BH4jKS2APAa8MuCRWRmZiWTJSnsHBGXAJ8DRMRHgAoalZmZlUSWpPCZpA5AAEjaGfi0oFGZmVlJZEkKFwD3AztKuhmYAZxd10mSrpf0lqRn8rZ1kjRd0pL0+1bpdkm6UtILkp6S1K+e92NmZg2Q5emj6cAIYDTwZ5KnkGZmuPYNwCHrbBsHzIiIXUiSy7h0+6HALunXGL7o1DYzsyLK8vTRkUBlRNwbEX8DKiV9s67zImIW8M46m48AJqfLk4Fv5m2/MRKPAVtK2i7rTZiZWePI1HyU/xZzRLxH0qRUH50jYlm6/AbQOV3eAXg177il6bb1SBojaZ6kecuXL69nGGZmVp0sSaG6Y7K8CV2riAjSzusNPG9SRJRHRHlZWVlDwzAzszxZksI8SZdJ2jn9ugyYX8/y3qxqFkq/v5Vufw3YMe+4Luk2MzMroixJ4YckL69NSb8+BU6rZ3l3A6PS5VHAXXnbT0ifQhoIvJ/XzGRmZkVSZzNQRKzii6eEMpP0Z+AAYBtJS0n6IS4GbpV0EvAKcEx6+FTgG8ALJGMrnbih5ZmZWcPVmRQk7QqcCXTLPz4iDqztvIg4roZdQ6o5Nqh/7cPMzBpJlg7j24CJwB+A1YUNx8zMSilLUqiMCL9MZmbWCmTpaL5H0n9J2i4dpqKTpE4Fj8zMzIouS02h6mmh/HmZA/hK44djZmallOXpo+7FCMTMzEov08xrks6TNCld30XS8MKHZmZmxeaZ18zMLMczr5mZWY5nXjMzs5wsTx+tO/PaIJIJd8zMrIWpNSlI2gjYimTmtYEkzUanR8TbRYjNzMyKrNakEBFrJJ0dEbcC9xYpJjMzK5EsfQoPSjpT0o5+o9nMrGXL0qfwrfR7/iimfqPZzKwFytKnMC4iphQpHjMzK6Fam48iYg1rj3lkZmYtmPsUzMwsx30KZmaW41FSzcwsJ8sczSdUtz0ibmz8cMzMrJSyNB/1z1tuDwwBFgBOCmZmLUyW5qMf5q9L2hK4pSGFSvoJcDJJ38TTwInAdul1twbmA9+NiM8aUo6ZmW2YLE8frWsVUO9+Bkk7AD8CyiOiJ9AGOBb4NXB5RPwH8C5wUn3LMDOz+snSp3AP6bDZJEmkB3BrI5TbQdLnwKbAMuBA4Nvp/snAeODaBpZjZmYbIEufwoS85UrglYhYWt8CI+I1SROAfwMfA9NImovei4jK9LClwA71LcPMzOonS1L4N7AsIj4BkNRBUreIqKhPgZK2Ao4gaYJ6D7gNOGQDzh8DjAHo2rVrfUIwM7MaZOlTuA1Yk7e+Ot1WXwcBL0fE8oj4HPgLycQ9W0qqSlJdSOaCXk9ETIqI8ogoLysra0AYZma2rixJYeP8p4DS5XYNKPPfwEBJm0oSySOui4CHgJHpMaOAuxpQhpmZ1UOWpLBc0uFVK5KOAOo981pEPA7cTvKuw9NpDJOAc4AzJL1A8ljqdfUtw8zM6idLn8JY4GZJV6frS4Fq33LOKiIuIJn7Od9LwICGXNfMzBomy8trL5I092yerq8seFRmZlYSdTYfSfofSVtGxMqIWClpK0m/LEZwZmZWXFn6FA6NiPeqViLiXeAbhQvJzMxKJUtSaCNpk6oVSR2ATWo53szMmqksHc03AzMk/TFdP5FkGAozM2thsnQ0/1rSkyQvnQH8IiIeKGxYZmZWCllqCgBPAG1JBsZ7onDhmJlZKWV5+ugYYA7J28bHAI9LGln7WWZm1hxlqSn8HOgfEW8BSCoDHiR5K9nMzFqQLE8fbVSVEFIrMp5nZmbNTJaawv2SHgD+nK5/C5hauJDMzKxUsjx9dJakEcB/ppsmRcRfCxuWmZmVQqanjyLiLyTzHpiZWQvmvgEzM8txUjAzs5wak4KkGen3XxcvHDMzK6Xa+hS2k7QvcLikWwDl74yIBQWNzMzMiq62pHA+8N9AF+CydfYFcGChgjIzs9KoMSlExO3A7ZL+OyJ+UcSYzMysRLK8p/ALSYcD+6ebZkbE3woblpmZlUKWAfF+BZwOLEq/Tpf0P4UOzMzMii/Ly2vDgL4RsQZA0mSS4bN/VsjAzMys+LK+p7Bl3nLHhhYqaUtJt0t6TtJiSftI6iRpuqQl6fetGlqOmZltmCxJ4VfAE5JuSGsJ84GLGljub4H7I2J3oA+wGBgHzIiIXYAZ6bqZmRVRlo7mP0uaCfRPN50TEW/Ut0BJHUk6rUen1/8M+EzSEcAB6WGTgZnAOfUtx8zMNlzWAfGWAXc3UpndgeXAHyX1Ial5nA50TssBeAPo3EjlmZlZRqUY+2hjoB9wbUTsBaxinaaiiAiSF+TWI2mMpHmS5i1fvrzgwZqZtSalSApLgaUR8Xi6fjtJknhT0nYA6fe3qjs5IiZFRHlElJeVlRUlYDOz1qLWpCCpjaTnGrPAtD/iVUm7pZuGkLz/cDcwKt02CrirMcs1M7O61dqnEBGrJT0vqWtE/LsRy/0hcLOkdsBLwIkkCepWSScBrwDHNGJ5ZmZF1W3cvQW9fsXFwwpy3SwdzVsBz0qaQ9L+D0BEHF7fQiNiIVBeza4h9b2mmZk1XJak8N8Fj8LMzJqELO8pPCxpJ2CXiHhQ0qZAm8KHZmZmxZZlQLzvkzwh9Lt00w7AnYUMyszMSiPLI6mnAYOADwAiYgmwbSGDMjOz0siSFD5Nh6IAQNLG1PBimZmZNW9ZksLDkn4GdJD0deA24J7ChmVmZqWQJSmMIxmr6GngFGAqcF4hgzIzs9LI8vTRmnTI7MdJmo2eT8cmMjOzFqbOpCBpGDAReBEQ0F3SKRFxX6GDMzOz4sry8tpvgK9FxAsAknYG7gWcFMzMWpgsfQofViWE1EvAhwWKx8zMSqjGmoKkEeniPElTgVtJ+hSOBuYWITYzMyuy2pqPDstbfhMYnC4vBzoULCIzMyuZGpNCRJxYzEDMzKz0sjx91J1k/oNu+cc3ZOhsMzNrmrI8fXQncB3JW8xrChuOmZmVUpak8ElEXFnwSMzMrOSyJIXfSroAmAZ8WrUxIhYULCozMyuJLEmhF/Bd4EC+aD6KdN3MzFqQLEnhaOAr+cNnm5lZy5TljeZngC0LHYiZmZVelprClsBzkuaydp+CH0k1M2thsiSFCwpRsKQ2wDzgtYgYnr4PcQuwNTAf+K6brMzMiivLfAoPF6js04HFwJfS9V8Dl0fELZImAicB1xaobDMzq0aWN5o/5Is5mdsBbYFVEfGlms+q85pdgGHARcAZkkTyNNO300MmA+NxUrAN0G3cvQW9fsXFwwp6fbOmIEtNYYuq5fTD+whgYAPLvQI4G6i69tbAexFRma4vBXao7kRJY4AxAF27dm1gGGZmli/L00c5kbgTOLi+BUoaDrwVEfPrc35ETIqI8ogoLysrq28YZmZWjSzNRyPyVjcCyoFPGlDmIOBwSd8A2pP0KfwW2FLSxmltoQvwWgPKMGt0hWyectOUNRVZnj7Kn1ehEqggaUKql4g4FzgXQNIBwJkRcbyk24CRJE8gjQLuqm8ZZmZWP1n6FIo1r8I5wC2Sfgk8QTIyq5mZFVFt03GeX8t5ERG/aGjhETETmJkuvwQMaOg1zcys/mqrKayqZttmJO8PbA00OCmYmVnTUtt0nL+pWpa0BcnLZieStPn/pqbzzMys+aq1T0FSJ+AM4HiSF8r6RcS7xQjMzMyKr7Y+hUuBEcAkoFdErCxaVGZmVhK1vbz2U2B74DzgdUkfpF8fSvqgOOGZmVkx1dansEFvO5uZWfPnD34zM8txUjAzsxwnBTMzy3FSMDOzHCcFMzPLcVIwM7McJwUzM8txUjAzsxwnBTMzy3FSMDOznCzTcZo1mkLOc2xmDeeagpmZ5TgpmJlZjpOCmZnlOCmYmVlO0ZOCpB0lPSRpkaRnJZ2ebu8kabqkJen3rYodm5lZa1eKp48qgZ9GxAJJWwDzJU0HRgMzIuJiSeOAccA5JYjPrOgK/VRWxcXDCnp9azmKXlOIiGURsSBd/hBYDOwAHAFMTg+bDHyz2LGZmbV2Je1TkNQN2At4HOgcEcvSXW8AnUsUlplZq1Wyl9ckbQ7cAfw4Ij6QlNsXESEpajhvDDAGoGvXrsUI1cxKqJBNa25WW19JagqS2pIkhJsj4i/p5jclbZfu3w54q7pzI2JSRJRHRHlZWVlxAjYzayWKXlNQUiW4DlgcEZfl7bobGAVcnH6/q9ixmdmG89AlLUspmo8GAd8Fnpa0MN32M5JkcKukk4BXgGNKEJuZWatW9KQQEf8AVMPuIcWMxczM1uZRUs1aATfxVM8/l/V5mAszM8txUjAzsxw3H1mzVNH+241+zW6f/F+jX9OsuXFNwczMcpwUzMwsx0nBzMxy3KdglipEPwW4r8KaF9cUzMwsx0nBzMxy3Hxk1gz5kVwrFNcUzMwsxzUFM7MCKtQDDPB+Qa7qmoKZmeW4pmCFNb7jWqsV7UsUh5ll4qRgZoDf07CEm4/MzCzHScHMzHLcfGTraczZqNyHYNa8uKZgZmY5TgpmZpbj5iP7Qvr4qJt8rKnzk1KF0+RqCpIOkfS8pBckjSt1PGZmrUmTqilIagNcA3wdWArMlXR3RCwqbWRmVl+FG+bBCqFJJQVgAPBCRLwEIOkW4Aig2SWFqid4CvIHMb4wY56YtXZOYE2v+WgH4NW89aXpNjMzK4KmVlOok6QxwJh0daWk5+t5qW2AtxsnqpqpEBe9cL2rFuVeiqQF3svwRr9wQf5d1awF/k5agAvVkHvZqaYdTS0pvAbsmLfeJd2WExGTgEkNLUjSvIgob+h1mgLfS9PUUu6lpdwH+F6yaGrNR3OBXSR1l9QOOBa4u8QxmZm1Gk2qphARlZJ+ADwAtAGuj4hnSxyWmVmr0aSSAkBETAWmFqGoBjdBNSG+l6appdxLS7kP8L3USRFRiOuamVkz1NT6FMzMrIRaXVKQdLSkZyWtkVSet/3rkuZLejr9fmAp48yipntJ952bDhXyvKSDSxVjfUjqK+kxSQslzZM0oNQx1ZekH0p6Lv09XVLqeBpK0k8lhaRtSh1LfUm6NP2dPCXpr5K2LHVMG6LQQwG1uqQAPAOMAGats/1t4LCI6AWMAv5U7MDqodp7kdSD5MmtPYFDgP9NhxBpLi4BLoyIvsD56XqzI+lrJG/k94mIPYEJJQ6pQSTtCAwF/l3qWBpoOtAzInoD/wLOLXE8meUNBXQo0AM4Lv17bzStLilExOKIWO+Ft4h4IiJeT1efBTpI2qS40W2Ymu6F5IPoloj4NCJeBl4gGUKkuQjgS+lyR+D1Wo5tyk4FLo6ITwEi4q0Sx9NQlwNnk/x+mq2ImBYRlenqYyTvQzUXuaGAIuIzoGoooEbT6pJCRkcBC6r+mJuh5j5cyI+BSyW9SvK/62bzP7l17ArsJ+lxSQ9L6l/qgOpL0hHAaxHxZKljaWTfA+4rdRAboOB/203ukdTGIOlB4MvV7Pp5RNxVx7l7Ar8mqSaXXEPupSmr7b6AIcBPIuIOSccA1wEHFTO+rOq4j42BTsBAoD9wq6SvRBN95K+Oe/kZTeRvIossfzeSfg5UAjcXM7amrkUmhYio1weIpC7AX4ETIuLFxo2qfup5L3UOF1Jqtd2XpBuB09PV24A/FCWoeqjjPk4F/pImgTmS1pCMvbO8WPFtiJruRVIvoDvwpCRI/j0tkDQgIt4oYoiZ1fV3I2k0yaBUQ5pqkq5Bwf+23XyUSp9AuBcYFxGPlDqeBrobOFbSJpK6A7sAc0oc04Z4HRicLh8ILClhLA1xJ/A1AEm7Au1ohoOxRcTTEbFtRHSLiG4kTRb9mmpCqIukQ0j6Rg6PiI9KHc8GKvhQQK3u5TVJRwJXAWXAe8DCiDhY0nkkbdf5H0BDm3LnYE33ku77OUl7aSXw44hoNu2mkv4T+C1JTfYT4L8iYn5po9pw6R/t9UBf4DPgzIj4e2mjajhJFUB5RDS7BAcg6QVgE2BFuumxiBhbwpA2iKRvAFfwxVBAFzXq9VtbUjAzs5q5+cjMzHKcFMzMLMdJwczMcpwUzMwsx0nBzMxynBSsKCStXGd9tKSrSxVPY5K0ezqi6xOSdi51PI1B0sx1R9611sFJwSwDSbW9/f9N4PaI2Cvrm/DNbNRaa0WcFKzkJN0gaWTe+sr0+wHpQHJ3SXpJ0sWSjpc0J533Yuf0uMPSQeeekPSgpM7p9vGSrk//1/uSpB/VUP5KSZencx7MkFSWbp8p6QpJ84DTJX01jWe+pAckbZe+SPRj4FRJD6XnfSeNcaGk31UlgLSc30h6EtinjuMukvSkknklqu6nczr+/5Pp1761lZd3f4dIui1v/QBJf0uXr1UyZ8Wzki6s6eeTtzxS0g3pcpmkOyTNTb8GZfyVWxPmpGDF0iH90D+Q3tcAAAMJSURBVFooaSHw/zKe1wcYC+wBfBfYNSIGkIyH9MP0mH8AAyNiL5KhhM/OO3934GCSIYcvkNS2mjI2A+alcx48DFyQt69dRJQDV5K8PT4yIr5K8qbyRemc4hOByyPia5L2AL4FDErng1gNHJ9XzuMR0YfkbdrajnssPW4W8P10+5XAw+n2fsCzdZRX5UFgb0mbpevfSn9OkAwQVw70BgZL6l3Nz6cmv03vuz/JyMJNdowqy65FDohnTdLH6YcWkBuQLEub9dyIWJae8yIwLd3+NOm4QiSDgk2RtB3J+EIv551/bzoE+qeS3gI6k4zdk28NMCVdvgn4S96+qu27AT2B6emgcG2AZdXEOwT4KjA3Pa4DUDVUymrgjgzHfQb8LV2eD3w9XT4QOAEgIlYD70v6bi3XIT22UtL9wGGSbgeG8UXiPEbSGJLPgu1IJm55qpr7qs5BQI+0XIAvSdo8IlbWco41cU4K1hRUktZaJW1E8sFeJX9OizV562v44t/vVcBlEXG3pAOA8TWcv5ps/+bzx35ZlX4X8GxE7FPHuQImR0R1c0B8kn6Y13Xc53kjd9YVc23XyXcL8APgHZJa0YdKBks8E+gfEe+mzULtqzk3/+eRv38jkhraJ3WUbc2Im4+sKagg+d8uwOFAdU08tenIF8MHj6pH+RsBVX0a3yZpjlrX80CZpH0AJLVVMvfGumYAIyVtmx7XSdJODThu3XNOTY9vI6njBlznYZImp+/zRdPRl0iS3vtpv8WhNZT7pqQ90oR9ZN72aXzRhIekvuudac2Ok4I1Bb8nac9+EtiHL/53ntV44DZJ86nf0NSrgAGSniFpolmvvyOd+nAk8Os0zoXAvtUctwg4D5gm6SmS+YC3q+9x6zgd+Jqkp0malXpsQHmrSZqkDk2/k86i9gTwHPB/QE1Dxo9Lz3mUtZvMfgSUS3pK0iKSvh9r5jxKqrV6klZGxOaljsOsKXBNwczMclxTMDOzHNcUzMwsx0nBzMxynBTMzCzHScHMzHKcFMzMLMdJwczMcv4/dfv4yRmzwgkAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TnB4SBJ8_GJg",
        "colab_type": "text"
      },
      "source": [
        "### Summary of results of Experiment 2\n",
        "\n",
        "| Xs                       | AUC   |\n",
        "|--------------------------|-------|\n",
        "| dens_acts                | 0.871 |\n",
        "| all_acts_flat            | 0.838 |\n",
        "| conv_acts                | 0.811 |\n",
        "| conv_dens_act_conc_on_x1 | 0.879 |\n",
        "| conv_dens_act_conc_on_x2 | 0.893 |\n",
        "| conv_dens_act_conc_on_x3 | 0.868 |\n",
        "| conv_dens_act_conc_on_ch | 0.887 |"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YiZ3CaQDCDmU",
        "colab_type": "text"
      },
      "source": [
        "## Experiment 3 - Looking for Systematic Errors\n",
        "\n",
        "1. Take the best model of experiment 2 (`conv_dens_act_conc_on_ch`)\n",
        "2. Train the model 10 times with 50 different samples of 50 datests\n",
        "3. Measure the mean deviation between the proportion of values in the validation dataset and the proportion of values for which the model makes errors (e.g. validation dataset has 10% of 6 but 30% of error are for 6s, 20% difference would signal systematic error)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "47Jsua7-_KRy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        },
        "outputId": "6c4a9b9b-6d7a-4031-da46-7eaeaec3d70b"
      },
      "source": [
        "hparam = {\"reg_amount\": 0.0, \"drop_rate\": 0.0, \"xs_shape\": xs.shape}\n",
        "ys_val_hat, ys_val, pref_val, pct_diffs = train_eval(hparam, cnn, conv_dens_act_conc_on_ch, ys, prefs, num_train=50, num_val=400, epochs=400)"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: <function cnn at 0x7fcdf56a3c80>\n",
            "Shape of input: (23750, 12, 14, 80)\n",
            "Hyperparameters: {'reg_amount': 0.0, 'drop_rate': 0.0, 'xs_shape': (23750, 12, 14, 80)}\n",
            "Number of training data: 50\n",
            "Number of validation data: 400\n",
            "Number of epochs: 400\n",
            "Doing training iteration no. 1\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 11, 13, 32)        10272     \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 11, 13, 16)        528       \n",
            "_________________________________________________________________\n",
            "average_pooling2d (AveragePo (None, 5, 6, 16)          0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 480)               0         \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 480)               0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 1)                 481       \n",
            "=================================================================\n",
            "Total params: 11,281\n",
            "Trainable params: 11,281\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Best train accuracy: 1.0\n",
            "Number of epochs: 40\n",
            "Doing training iteration no. 2\n",
            "Doing training iteration no. 3\n",
            "Doing training iteration no. 4\n",
            "Doing training iteration no. 5\n",
            "Doing training iteration no. 6\n",
            "Doing training iteration no. 7\n",
            "Doing training iteration no. 8\n",
            "Doing training iteration no. 9\n",
            "Doing training iteration no. 10\n",
            "Results ('mean_val_auc', 0.8927126169204712, 'mean_val_accuracy', 0.8322499930858612, 'metric', 0.8624813050031662, 'val_auc_std', 0.024345455638997714, 'val_accuracy_std', 0.02739638713499936)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6MEZ1dBDBhhM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        },
        "outputId": "6dc71b62-e8fa-4ba4-aa60-ce88d8e417e8"
      },
      "source": [
        "# Some model have differences as big as 20% for certain preference values\n",
        "pd.concat(pct_diffs, axis=1).fillna(0).max().sort_values(ascending=False).round(4)"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8    0.1929\n",
              "2    0.1893\n",
              "5    0.1815\n",
              "4    0.1787\n",
              "3    0.1722\n",
              "7    0.1428\n",
              "6    0.1363\n",
              "0    0.1299\n",
              "9    0.1261\n",
              "1    0.0867\n",
              "dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "deGYGz00_4XE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        },
        "outputId": "d7f8e505-53f6-4c92-b926-99d68dec8371"
      },
      "source": [
        "# For no preference value all model training make sysetmatic mistakes\n",
        "pd.concat(pct_diffs, axis=1).fillna(0).mean().sort_values(ascending=False).round(4)"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8    0.0415\n",
              "4    0.0331\n",
              "9    0.0319\n",
              "6    0.0252\n",
              "3    0.0060\n",
              "2    0.0035\n",
              "7    0.0012\n",
              "1    0.0012\n",
              "0    0.0008\n",
              "5    0.0000\n",
              "dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    }
  ]
}