analysis_correlation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_csv(epoch):\n",
    "    return pd.read_csv(f'outputs++/batches_permutations_model=BasicConv(in_channels=1, out_classes=2)_epoch={epoch}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_EPOCHS = 13\n",
    "dfs = [get_csv(e) for e in range(N_EPOCHS)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in range(N_EPOCHS):\n",
    "    dfs[e] = dfs[e].rename(columns={\n",
    "        'FinalBatches': f'Epoch{e}Batches',\n",
    "        'FinalBatchesIndexes': f'FinalBatchesIndexes_{e}',\n",
    "        'FinalPermutationIndex': f'FinalPermutationIndex_{e}',\n",
    "        'TestLoss': f'TestLoss_{e}',\n",
    "        'TestCorrect': f'TestCorrect_{e}',\n",
    "        'TestTotal': f'TestTotal_{e}',\n",
    "        'TestAcc': f'TestAcc_{e}',\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged = [dfs[0]]\n",
    "for e in range(1, N_EPOCHS):\n",
    "    new_merged = merged[e - 1].merge(dfs[e], how='inner', on=[f'Epoch{prev_epoch}Batches' for prev_epoch in range(e)])\n",
    "    merged.append(new_merged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "def correlations(from_column, to_column, method='pearson', fixed_diff=None):\n",
    "#     print(f'Pearson correlation coefficients from column {from_column} to {to_column}:')\n",
    "    corrs = {'FromEpoch': [], 'ToEpoch': [], 'Correlation': []}\n",
    "    for epoch in range(1, N_EPOCHS):\n",
    "        if fixed_diff is not None:\n",
    "            prev_range = range(max(0, epoch - fixed_diff), epoch - fixed_diff + 1)\n",
    "        else:\n",
    "            also_check_against_same_epoch = 1 if from_column != to_column else 0\n",
    "            prev_range = range(0, epoch + also_check_against_same_epoch)\n",
    "            \n",
    "        for prev_epoch in prev_range:\n",
    "            m = merged[epoch]\n",
    "            corr = m[f'{from_column}_{prev_epoch}'].corr(m[f'{to_column}_{epoch}'])\n",
    "#             print(f'{prev_epoch} -> {epoch}: {corr}')\n",
    "            corrs['FromEpoch'].append(prev_epoch)\n",
    "            corrs['ToEpoch'].append(epoch)\n",
    "            corrs['Correlation'].append(corr)\n",
    "    return pd.DataFrame.from_dict(corrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_corr_all_deltas(corrs):\n",
    "    for d in range(1, 4):\n",
    "        c = corrs[corrs['FromEpoch'] == corrs['ToEpoch'] - d]\n",
    "        n = c.shape[0]\n",
    "        mid = (n + 1) // 2\n",
    "        first_half = c.iloc[:mid]\n",
    "        second_half = c.iloc[mid:]\n",
    "        print(first_half.shape[0], second_half.shape[0])\n",
    "\n",
    "        print(f'd={d} all     : {c[\"Correlation\"].mean()}')\n",
    "        print(f'd={d} 1st half: {first_half[\"Correlation\"].mean()}')\n",
    "        print(f'd={d} 2nd half: {second_half[\"Correlation\"].mean()}')\n",
    "\n",
    "        \n",
    "\n",
    "#         print(f'd={d} all     : {c[\"Correlation\"].mean()}')\n",
    "#         print(f'd={d} 1st half: {first_half[\"Correlation\"].mean()}')\n",
    "#         print(f'd={d} 2nd half: {second_half[\"Correlation\"].mean()}')\n",
    "        \n",
    "        \n",
    "# os.makedirs('csvs', exist_ok=True)\n",
    "# out_csv = 'csvs/corr_loss_loss.csv'\n",
    "# assert not os.path.isfile(out_csv)\n",
    "# corrs.to_csv(out_csv, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrs = correlations('TestLoss', 'TestLoss', fixed_diff=None)\n",
    "# print_corr_all_deltas(corrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{table}[H]\n",
      "\\centering\n",
      "\\begin{tabular}{l c c c c}\n",
      "\\cmidrule[2pt]{1-4}\n",
      "\\multirow{1}{*}{\\textbf{DeltaEpoch}} & \\multicolumn{1}{c}{AvgCorrEpochs[1-6]} & \\multicolumn{1}{c}{AvgCorrEpochs[7-13]} & \\multicolumn{1}{c}{AvgCorrEpochs[1-13]} \\\\\n",
      "\\cmidrule{1-4}\n",
      "NaNs: 0\n",
      "\t\\hfill 1 & -0.275 & 0.041 & -0.117 \\\\\n",
      "NaNs: 0\n",
      "\t\\hfill 2 & 0.017 & 0.164 & 0.083 \\\\\n",
      "NaNs: 1\n",
      "\t\\hfill 3 & -0.056 & 0.010 & -0.027 \\\\\n",
      "\\cmidrule{1-4}\n",
      "\\end{tabular}\n",
      "\\caption{Pearson correlations between test losses\\label{tab:correlations_test_losses}}\n",
      "\\end{table}\n"
     ]
    }
   ],
   "source": [
    "corrs = correlations('TestLoss', 'TestLoss', fixed_diff=None)\n",
    "\n",
    "print(r'\\begin{table}[H]')\n",
    "print(r'\\centering')\n",
    "print(r'\\begin{tabular}{l c c c c}')\n",
    "print(r'\\cmidrule[2pt]{1-4}')\n",
    "head = ' & '.join([\n",
    "    r'\\multirow{1}{*}{\\textbf{DeltaEpoch}}',\n",
    "    r'\\multicolumn{1}{c}{AvgCorrEpochs[1-6]}',\n",
    "    r'\\multicolumn{1}{c}{AvgCorrEpochs[7-13]}',\n",
    "    r'\\multicolumn{1}{c}{AvgCorrEpochs[1-13]}',\n",
    "]) + r' \\\\'\n",
    "print(head)\n",
    "print(r'\\cmidrule{1-4}')\n",
    "\n",
    "for d in range(1, 4):\n",
    "    c = corrs[corrs['FromEpoch'] == corrs['ToEpoch'] - d]\n",
    "    print('NaNs:', c['Correlation'].isnull().values.sum()) \n",
    "    n = c.shape[0]\n",
    "    mid = (n + 1) // 2\n",
    "    first_half = c.iloc[:mid]\n",
    "    second_half = c.iloc[mid:]\n",
    "    s = '\\t' + r'\\hfill ' + '{} & {:.3f} & {:.3f} & {:.3f} \\\\\\\\'.format(d, first_half[\"Correlation\"].mean(), second_half[\"Correlation\"].mean(), c[\"Correlation\"].mean())\n",
    "    print(s)\n",
    "#         print(first_half.shape[0], second_half.shape[0])\n",
    "print(r'\\cmidrule{1-4}')\n",
    "print(r'\\end{tabular}')\n",
    "print(r'\\caption{Pearson correlations between test losses\\label{tab:correlations_test_losses}}')\n",
    "print(r'\\end{table}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 6\n",
      "d=1 all     : -0.1079790826382764\n",
      "d=1 1st half: -0.2748973080768131\n",
      "d=1 2nd half: 0.05893914280026033\n",
      "6 5\n",
      "d=2 all     : 0.11139868684314283\n",
      "d=2 1st half: 0.016689350940818647\n",
      "d=2 2nd half: 0.2534626906966291\n",
      "5 5\n",
      "d=3 all     : -0.016342751976741926\n",
      "d=3 1st half: -0.05626907347738348\n",
      "d=3 2nd half: 0.05020111719099401\n"
     ]
    }
   ],
   "source": [
    "corrs = correlations('TestAcc', 'TestAcc')\n",
    "print_corr_all_deltas(corrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "corrs = correlations('TestAcc', 'TestAcc')\n",
    "os.makedirs('csvs', exist_ok=True)\n",
    "out_csv = 'csvs/corr_test_acc_test_acc.csv'\n",
    "assert not os.path.isfile(out_csv)\n",
    "corrs.to_csv(out_csv, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs('plots', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3df5xVVb3/8ddbIKCUhBgJgYKUuoG36EqoZV3LStJbWF8tvJV2+0H51VJvZph9b/r9RlFplpV2Tb3gj69GpYmpJVra11JpMBTxx5UCZYRg1EzUIsHP94+1JjfjmZkzMnvOnsP7+Xicx+yz9lprr3XOmf05e+119lZEYGZmVjU7NboBZmZmtThAmZlZJTlAmZlZJTlAmZlZJTlAmZlZJTlAmZlZJTlAmXVD0o2SPpaXPyDpuj6se6WkA/LyqZIu7sO6Py/pvL6qrxfbfZWk30naJOnT/b39gUBSSNqz0e0YCBygBiBJTxQez0j6S+H5B55HfX/fCVvXIuKSiHhHT/kkLZD0pTrqmxoRN25vuyQdIKmtU91fjohGvKcnATdGxC4RcZakNZLe1pcbkHSspFZJmyUt6LRuSl73p/y4XtKUvty+9R8HqAEoInbueAAPAu8qpF3S6Pb1NUmDG92GvtRs/enk5cDKvqhISa191DrgS8AFXaw7DBgFjAYWA5f1RXus/zlANRFJO0maK+n3kh6RtEjSqLxumKSLc/pjkn4raYykecCbgO/kI7DvdFH3DyX9UdKfJf1K0tTCuuGSzpD0QF5/s6Thed3+kn6Tt7lW0odz+jZHbZI+LOnmwvOQdIyk+4H7c9q3ch2PS1om6U2F/IPysNbv8/DSMkkTJH1X0hmd+nKVpOO76OfbJd2b+/EdQLXamHeeZ0ramPPeKWkvSXOADwAn5dfzqpx/jaTPSboTeFLS4BpHF8Mk/SC3/3ZJr+30euxZeL5A0pckvQi4Fti9cBS9uzoNGUp6t9KQ4mP5tX91Yd0aSSfmPvw5t2FYF6/PHpJ+kT9HD0u6RNKued0vgLfw7GfpUuBlwFX5+Uk5376Fz8QdysOced2NkuZJ+jXwFPCKzm2IiMsj4ifAIzXWPRYRayJdIkfAVqDL4TRJL5Z0vqT1kh7Kr+mgvO7Dkn4t6dv5dblX0oGFsrtLWizpUUmrJH28sK7m57Gw6bdJul/pKO+7kpTL7Snppry9hyX9oKu27xAiwo8B/ADWAG/Ly8cDtwLjgaHAfwKX5nWfAK4CXggMAvYGRuR1NwIf62E7HwF2yfV+E1heWPfdXMe4XPcbcr6XAZuAI4AhwEuAabW2CXwYuLnwPIAlpG/Cw3PaB3Mdg4HPAH8EhuV1nwVWAK8i7Zhem/POIH2r3innG03a8Y2p0cfRwOOkb+BDgBOALR3tLLYROAhYBuyat/dqYGxetwD4Uo33aTkwodCf4nt3KvB0YdsnAquBIYXXY89CfX/fBnAA0NZpe6cCF+flVwJPAm/PdZ8ErAJeUGjHUmD3/HrfA3yyi8/BnrmeoUAL8Cvgm4X1nd/Xv/cxPx9HCiwHk74gvz0/bymUfxCYmt/nId18Jr8ELOhi3WP5vXsG+EI3dfyE9H/yImC3/Dp8ovB+b8mfgyHA+4E/A6Py+puAs4FhwDSgHTiwu89j4b38Kemz87JcbmZedylwSn5thgH7N3of08iHj6CayyeAUyKiLSI2k3ZShykNKT1N2mHvGRFbI2JZRDxeb8URcUFEbCrU+9r87XMnUvA6LiIeynX/Juf7AHB9RFwaEU9HxCMRsbwX/flKRDwaEX/Jbbg417ElIs4g7SRflfN+jLQjui+SO3LepaSdSsc339mkcyQbamzvYODuiPhRRDxNCsR/7KJtT5MC9j8Aioh7ImJ9D/05KyLWdvSnhmWFbX+DtIPat4c66/F+4OqIWJLrPh0YTvoiUWzbuoh4lPRFZlqtiiJiVa5nc0S053b+cy/a8kHgmoi4JiKeiYglQCvpte+wICJW5vf56V7UXWznrsCLgWOB39XKI2kM8E7g+Ih4MiI2AmeSPiMdNpIC8NMR8QPgPuCQfDS0P/C5iPhr/lyfB3wol6v5eSzUOz/S0d6DwC959vV+mjRMunuu92Z2YA5QzeXlwBV56OQx0jfhrcAY4CLg58BlktZJ+pqkIfVUmocr5ufhisdJ34ohHXGMJu1If1+j6IQu0uu1tlM7PiPpnjz88RhpBzS6jm0tJO0YyX8v6iLf7sVtRkR0bkNh3S+A75COHjdIOlfSiN70p7v1EfEM0JbbtL12Bx7oVPda0tFMh2IgfgrYuVZFknaTdFkeDnscuJhn34N6vBw4vOMzmt/H/YGxhTw9vU51iYgnge8BF0rarYu2DAHWF9ryn6QjqQ4P5c9BhwdIr+fuwKMRsanTuo7XtKfPflev90mkI66leUj2I931sdk5QDWXtcA7I2LXwmNYPrJ5OiJOi4gppG/O/wIcmcv1dEn7fwVmAW8jBYWJOV3Aw8BfgT26aE+tdEhDTi8sPH9pjTx/b5fS+abPAe8DRuZvyH/m2XNE3W3rYmBWPqfzatKwTi3rSTuWjm2q+Pw5jYs4KyL2Jg1HvZI0rLNNu7vqTxeK296JNFS7Lic9RdevV0/1riPtjDvq7ujXQz2Uq+UreXuviYgRpICvbvJ3btta4KJOn9EXRcT8bspsj51Ir9u4GuvWApuB0YW2jIiIqYU84zrOD2UvI72e64BRknbptK7jNe3u89iliPhjRHw8InYnjYicrR14SroDVHP5HjBP0ssBJLVImpWX3yLpH/MJ4MdJQwlbc7kN1DgZXbAL6R/5EdI/+5c7VuRv4xcA38gnjQdJ2k/SUOAS0sng9ylNCniJpI6hjOXAeyW9MP8DfrSHvu1COh/QDgyW9B9A8YjlPOD/SJqs5DWSXpLb2Ab8lnTk9ONuhtiuBqZKem8eFv00tQMnkl4vaZ98FPokKUjX+3p2Ze/Cto8nvea35nXLgX/Nr+9Mth1W2wC8RNKLu6h3EWlY6sDc3s/kun/zPNq4C/AE8JikcTwblLvS+bW4GHiXpINyX4YpTZMfX28D8mdpGOl8Z0cdg/O6t0t6Xa57BGkI8k+k0YRt5CHZ64AzJI1QmmS0h6Tia7sb8GlJQyQdTvqCc01ErCW9fl/J238N6TPcMYu2y89jD307vPBa/IkUrLd2U6SpOUA1l2+RptVeJ2kTaee2T173UuBHpOB0D+kE78WFcoflGUVn1aj3QtLwxUPA3Ty70+xwIumE8G+BR4GvkiYlPEg6t/CZnL6cdLIY0lj/30g7sIU8+4/dlZ+TZqv9d27LX9l2KOgbpB3xdbmP55POs3RYCPwjXQ/vEREPA4cD80nBeDLw6y6yjwC+T9qJPJDzn57XnQ9MycNGXR2t1XIl6XzRn0jnMt5bOAdzHPAu0sn/D1A4CoyIe0kn1/+Qt7nNsGBE3Ec60vk26Yj3XaSfJvytF23rcBrwT6Sj16uBy3vI/xXgC7ldJ+Yd+yzg86QvG2tJQa43+6IvAH8B5pL69ZecBmniwaW5fb8nTeqYGRF/7aKuI4EXkD7XfyL9jxSHG28jfQ4eBuYBhxXOJR1BGk1YB1wBfDGfU4OeP49deT1wm6QnSP/Lx0XE6jrKNSVtO7xq1pwkvZkUkCfmoz6zbin9JOJjEbF/o9uyo/IRlDW9PKx1HHCeg5PZwOEAZU1N6Qepj5GGbb7Z4OaYWS94iM/MzCrJR1BmZlZJTXvRytGjR8fEiRMb3QwzM+vBsmXLHo6Ils7pTRugJk6cSGtra6ObYWZmPZD0QK10D/GZmVkllRag8q+rlypdTn+lpNNy+qn5Ol7L8+PgQpmTlS5bf5+kgwrpe0takded1enSI2Zm1oTKHOLbDLw1Ip7Iv0O5WdK1ed2ZEXF6MbPSXS9nk65rtjtwvaRXRsRW4BxgDukKBtcAM0lXFTAzsyZV2hFUvsT8E/npkPzobk77LOCyfBn/1aT71cyQNJZ036Jb8lWFLwQOLavdZmZWDaWeg8oXbFxOuqfKkoi4La86VununRdIGpnTxrHttdXactq4vNw53czMmlipASrSzeumkW4bMEPSXqThuj1IN+haD3TcjrvWeaXoJv05JM2R1Cqptb29fbvbb2ZmjdMvs/gi4jHSrZxnRsSGHLieIV0NekbO1sa2997puBdOW17unF5rO+dGxPSImN7S8pwp9WZmNoCUOYuvRdKueXk46WZ39+ZzSh3eA9yVlxcDsyUNlTSJdIn7pfmeLZsk7Ztn7x1Jui2BmZk1sTJn8Y0FFuYb5O0ELIqIn0q6KN+0Lki3Dv8EQESslLSIdF+WLcAxeQYfwNHAAtL9VK7FM/jMzJpe014sdvr06eErSZjZQDNx7tV1510z/5ASW9J/JC2LiOmd030lCTMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzq6TSApSkYZKWSrpD0kpJp+X0UZKWSLo//x1ZKHOypFWS7pN0UCF9b0kr8rqzJKmsdpuZWTWUeQS1GXhrRLwWmAbMlLQvMBe4ISImAzfk50iaAswGpgIzgbMlDcp1nQPMASbnx8wS221mZhVQWoCK5In8dEh+BDALWJjTFwKH5uVZwGURsTkiVgOrgBmSxgIjIuKWiAjgwkIZMzNrUqWeg5I0SNJyYCOwJCJuA8ZExHqA/He3nH0csLZQvC2njcvLndNrbW+OpFZJre3t7X3bGTMz61elBqiI2BoR04DxpKOhvbrJXuu8UnSTXmt750bE9IiY3tLS0vsGm5lZZfTLLL6IeAy4kXTuaEMetiP/3ZiztQETCsXGA+ty+vga6WZm1sTKnMXXImnXvDwceBtwL7AYOCpnOwq4Mi8vBmZLGippEmkyxNI8DLhJ0r559t6RhTJmZtakBpdY91hgYZ6JtxOwKCJ+KukWYJGkjwIPAocDRMRKSYuAu4EtwDERsTXXdTSwABgOXJsfZmbWxEoLUBFxJ/C6GumPAAd2UWYeMK9GeivQ3fkrMzNrMr6ShJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVZIDlJmZVVJpAUrSBEm/lHSPpJWSjsvpp0p6SNLy/Di4UOZkSask3SfpoEL63pJW5HVnSVJZ7TYzs2oYXGLdW4DPRMTtknYBlklaktedGRGnFzNLmgLMBqYCuwPXS3plRGwFzgHmALcC1wAzgWtLbLuZmTVYaUdQEbE+Im7Py5uAe4Bx3RSZBVwWEZsjYjWwCpghaSwwIiJuiYgALgQOLavdZmZWDf1yDkrSROB1wG056VhJd0q6QNLInDYOWFso1pbTxuXlzum1tjNHUquk1vb29j7sgZmZ9bfSA5SknYEfA8dHxOOk4bo9gGnAeuCMjqw1ikc36c9NjDg3IqZHxPSWlpbtbruZmTVOqQFK0hBScLokIi4HiIgNEbE1Ip4Bvg/MyNnbgAmF4uOBdTl9fI10MzNrYmXO4hNwPnBPRHyjkD62kO09wF15eTEwW9JQSZOAycDSiFgPbJK0b67zSODKstptZmbVUOYsvjcCHwJWSFqe0z4PHCFpGmmYbg3wCYCIWClpEXA3aQbgMXkGH8DRwAJgOGn2nmfwmZk1udICVETcTO3zR9d0U2YeMK9GeiuwV9+1zszMqs5XkjAzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0pygDIzs0oq85bvZmZWoolzr64775r5h5TYknL4CMrMzCrJAcrMzCrJAcrMzCrJAcrMzCqptAAlaYKkX0q6R9JKScfl9FGSlki6P/8dWShzsqRVku6TdFAhfW9JK/K6sySprHabmVk1lHkEtQX4TES8GtgXOEbSFGAucENETAZuyM/J62YDU4GZwNmSBuW6zgHmAJPzY2aJ7TYzswooLUBFxPqIuD0vbwLuAcYBs4CFOdtC4NC8PAu4LCI2R8RqYBUwQ9JYYERE3BIRAVxYKGNmZk2qrgAl6Y31pHVTfiLwOuA2YExErIcUxIDdcrZxwNpCsbacNi4vd06vtZ05kloltba3t9fbPDMzq6B6j6C+XWfac0jaGfgxcHxEPN5d1hpp0U36cxMjzo2I6RExvaWlpZ7mmZlZRXV7JQlJ+wFvAFok/Xth1QhgUO1S25QfQgpOl0TE5Tl5g6SxEbE+D99tzOltwIRC8fHAupw+vka6mZk1sZ6OoF4A7EwKZLsUHo8Dh3VXMM+0Ox+4JyK+UVi1GDgqLx8FXFlIny1pqKRJpMkQS/Mw4CZJ++Y6jyyUMTOzJtXtEVRE3ATcJGlBRDzQy7rfCHwIWCFpeU77PDAfWCTpo8CDwOF5WyslLQLuJs0APCYituZyRwMLgOHAtflhZmZNrN6LxQ6VdC4wsVgmIt7aVYGIuJna548ADuyizDxgXo30VmCvOttqZmZNoN4A9UPge8B5wNYe8pqZmW23egPUlog4p9SWmJmZFdQ7zfwqSf9T0th8qaJRkkaV2jIzM9uh1XsE1THr7rOFtABe0bfNMTMzS+oKUBExqeyGmJmZFdUVoCQdWSs9Ii7s2+aYmZkl9Q7xvb6wPIw0Tfx20oVbzczM+ly9Q3yfKj6X9GLgolJaZGZmxvO/3cZTpEsRmZmZlaLec1BX8ewVxAcBrwYWldUoMzOzes9BnV5Y3gI8EBFtXWU2MzPbXnUN8eWLxt5LupL5SOBvZTbKzMys3jvqvg9YSrry+PuA2yR1e7sNMzOz7VHvEN8pwOsjYiOApBbgeuBHZTXMzKzKJs69uu68a+YfUmJLmle9s/h26ghO2SO9KGtmZtZr9R5B/UzSz4FL8/P3A9eU0yQzM7MeApSkPYExEfFZSe8F9ifdhPAW4JJ+aJ+Zme2gehqm+yawCSAiLo+If4+IE0hHT98su3FmZrbj6ilATYyIOzsn5luwTyylRWZmZvQcoIZ1s254XzbEzMysqKcA9VtJH++cKOmjwLJymmRmZtZzgDoe+DdJN0o6Iz9uAj4GHNddQUkXSNoo6a5C2qmSHpK0PD8OLqw7WdIqSfdJOqiQvrekFXndWZL0/LpqZmYDSbcBKiI2RMQbgNOANflxWkTsFxF/7KHuBcDMGulnRsS0/LgGQNIUYDYwNZc5W9KgnP8cYA7p6umTu6jTzMyaTL33g/ol8MveVBwRv5I0sc7ss4DLImIzsFrSKmCGpDXAiIi4BUDShcChwLW9aYuZmQ08jbgaxLGS7sxDgCNz2jhgbSFPW04bl5c7p9ckaY6kVkmt7e3tfd1uMzPrR/0doM4B9gCmAeuBM3J6rfNK0U16TRFxbkRMj4jpLS0t29tWMzNroH4NUPmc1taIeAb4PjAjr2oDJhSyjgfW5fTxNdLNzKzJ9WuAkjS28PQ9QMcMv8XAbElDJU0iTYZYGhHrgU2S9s2z944EruzPNpuZWWPUe7HYXpN0KXAAMFpSG/BF4ABJ00jDdGuATwBExEpJi4C7SXfsPSYituaqjibNCBxOmhzhCRJmZjuA0gJURBxRI/n8bvLPA+bVSG8F9urDppmZ2QDgezqZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklOUCZmVklDW50A8zMrHwT515dd9418w8psSX18xGUmZlVkgOUmZlVUmkBStIFkjZKuquQNkrSEkn3578jC+tOlrRK0n2SDiqk7y1pRV53liSV1WYzM6uOMo+gFgAzO6XNBW6IiMnADfk5kqYAs4GpuczZkgblMucAc4DJ+dG5TjMza0KlBaiI+BXwaKfkWcDCvLwQOLSQfllEbI6I1cAqYIakscCIiLglIgK4sFDGzMyaWH+fgxoTEesB8t/dcvo4YG0hX1tOG5eXO6fXJGmOpFZJre3t7X3acDMz619VmSRR67xSdJNeU0ScGxHTI2J6S0tLnzXOzMz6X38HqA152I78d2NObwMmFPKNB9bl9PE10s3MrMn1d4BaDByVl48Criykz5Y0VNIk0mSIpXkYcJOkffPsvSMLZczMrImVdiUJSZcCBwCjJbUBXwTmA4skfRR4EDgcICJWSloE3A1sAY6JiK25qqNJMwKHA9fmh5mZNbnSAlREHNHFqgO7yD8PmFcjvRXYqw+bZmZmA0BVJkmYmZltwwHKzMwqyQHKzMwqyQHKzMwqyQHKzMwqyTcsNDPLenNTPyufj6DMzKySHKDMzKySHKDMzKySHKDMzKySHKDMzKySHKDMzKySHKDMzKySHKDMzKySHKDMzKySHKDMzKySHKDMzKySHKDMzKySHKDMzKySHKDMzKySHKDMzKySGhKgJK2RtELSckmtOW2UpCWS7s9/RxbynyxplaT7JB3UiDabmVn/auQR1FsiYlpETM/P5wI3RMRk4Ib8HElTgNnAVGAmcLakQY1osJmZ9Z8qDfHNAhbm5YXAoYX0yyJic0SsBlYBMxrQPjMz60eNClABXCdpmaQ5OW1MRKwHyH93y+njgLWFsm057TkkzZHUKqm1vb29pKabmVl/GNyg7b4xItZJ2g1YIunebvKqRlrUyhgR5wLnAkyfPr1mHjMzGxgacgQVEevy343AFaQhuw2SxgLkvxtz9jZgQqH4eGBd/7XWzMwaod8DlKQXSdqlYxl4B3AXsBg4Kmc7CrgyLy8GZksaKmkSMBlY2r+tNjOz/taIIb4xwBWSOrb/fyPiZ5J+CyyS9FHgQeBwgIhYKWkRcDewBTgmIrY2oN1mZtaP+j1ARcQfgNfWSH8EOLCLMvOAeSU3zczMKqRK08zNzMz+zgHKzMwqyQHKzMwqyQHKzMwqyQHKzMwqyQHKzMwqyQHKzMwqyQHKzMwqyQHKzMwqqVFXMzcz6xcT517d6CbY8+QAZWZm2+hNUF8z/5DS2uEhPjMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyQHKDMzqyT/DsrMKqEqv72x6vARlJmZVZIDlJmZVZKH+MyaVFlDZr62nfWXAROgJM0EvgUMAs6LiPkNbpJZv3NwsB3JgBjikzQI+C7wTmAKcISkKY1tlZmZlWmgHEHNAFZFxB8AJF0GzALubmirbBtVmYVV1lGGh8HM+pciotFt6JGkw4CZEfGx/PxDwD4RcWynfHOAOfnpq4D7+rAZo4GH+7C+qtpR+gnua7NyXweel0dES+fEgXIEpRppz4msEXEucG4pDZBaI2J6GXVXyY7ST3Bfm5X72jwGxDkooA2YUHg+HljXoLaYmVk/GCgB6rfAZEmTJL0AmA0sbnCbzMysRANiiC8itkg6Fvg5aZr5BRGxsp+bUcrQYQXtKP0E97VZua9NYkBMkjAzsx3PQBniMzOzHYwDlJmZVdIOGaAkzZR0n6RVkubWWD9S0hWS7pS0VNJehXUnSFop6S5Jl0oa1qnsiZJC0uj+6EtPyuqrpE/leldK+lp/9ac7ZfRV0jRJt0paLqlV0oz+7FNXtrOvx+V+rpR0fCF9lKQlku7Pf0f2V3+6UlI/vy7p3lzmCkm79ld/ulNGXwvrK7VfqltE7FAP0iSL3wOvAF4A3AFM6ZTn68AX8/I/ADfk5XHAamB4fr4I+HCh3ATSRI4HgNHN2lfgLcD1wND8fLcm7ut1wDvz8sHAjQO8r3sBdwEvJE2Suh6YnNd9DZibl+cCX23Sfr4DGJyXv9rofpbZ17y+Uvul3jx2xCOov182KSL+BnRcNqloCnADQETcC0yUNCavGwwMlzSY9IEo/h7rTOAkavyIuEHK6uvRwPyI2JzLbSy3G3Upq68BjMjLL6Yav7/bnr6+Grg1Ip6KiC3ATcB7cplZwMK8vBA4tNxu9KiUfkbEdTkN4FbS7yobraz3FKq3X6rbjhigxgFrC8/bclrRHcB7AfKQzsuB8RHxEHA68CCwHvhzRFyX870beCgi7ii3+b1SSl+BVwJvknSbpJskvb7EPtSrrL4eD3xd0tqc5+TSelC/591X0jftN0t6iaQXko4KO34EPyYi1gPkv7uV1oP6lNXPoo8A1/Zxu5+PUvpa0f1S3XbEAFXPZZPmAyMlLQc+BfwO2JLH5GcBk4DdgRdJ+mD+UJwC/Ed5zX5e+ryvucxgYCSwL/BZYJGkWtvqT2X19WjghIiYAJwAnF9G43vpefc1Iu4hDWstAX5G2ultoZpK7aekU3LaJX3c7uejz/ta4f1S3QbED3X7WI+XTYqIx4F/A8g73tX5cRCwOiLa87rLgTeQPhCTgDvyfno8cLukGRHxx1J7070y+npxrvfySAPcSyU9Q7poZXupveleWX09CjguV/FD4LzyulC37ekrEXE+OdBK+nKuD2CDpLERsV7SWKDRQ7dl9RNJRwH/AhyYP8eNVkZf96Ca+6X6NfokWH8/SEH5D6Q3ruNk5NROeXYFXpCXPw5cmJf3AVaSzlGINE7/qRrbWEMFTkaW1Vfgk8D/zsuvJA1NqEn7eg9wQF4+EFg2kN/X/Hy3/PdlwL3AyPz862w7SeJrTdrPmaRb9bQ0+r0su6+dyldiv9Sr16XRDWjQh+Fg4L9Js2ZOyWmfBD6Zl/cD7s9v9OXFNxs4LaffBVxEnslW1Q9CGX3N/0AX5/Tbgbc2up8l9nV/YFneYdwG7N3ofvZBX/9f3kHfQTqC6Eh/Cekk/P3576gm7ecq0peq5fnxvUb3s6y+dqq/Mvuleh++1JGZmVXSjjhJwszMBtNPt1gAAANfSURBVAAHKDMzqyQHKDMzqyQHKDMzqyQHKDMzq6Qd8Ye6Zr0iqWP6NcBLga08+6PkGZGundZd+QOAv0XEb7rJcyXptyz7bX+LzZqDA5RZDyLiEWAagKRTgSci4vReVHEA8ARQM0Dl2z38E/CEpEkRsXq7GtwFSYPj2YukmlWeh/jMngdJe+cL5S6T9PN8aSAkfVrS3fmePZdJmkj6seUJSveUelON6v4HcBXpCtazC9vYU9L1ku6QdLukPXL6SZJW5PT5Oe1GSdPz8mhJa/LyhyX9UNJVwHWSdpZ0Q65vhaRZhe0dmdt9h6SLJO0iabWkIXn9CElrOp6blc1HUGa9J+DbwKyIaJf0fmAe6crYc4FJEbFZ0q4R8Zik79H9UdcRpCtZbAB+BHwlp19Cuq3JFUo3UNxJ0jtJt8HYJyKekjSqjvbuB7wmIh7NtxN5T0Q8rnTzulslLSbdyuEU4I0R8bCkURGxSdKNwCHAT0jB88cR8XSvXi2z58kByqz3hpJuErckX4RzEOk2HQB3ApdI+glpp96tfD+fPYGbIyIkbVG6U+oDwLiIuAIgIv6a878N+K+IeCqnP1pHe5cU8gn4sqQ3A8+QbukwBngr8KOIeLhTveeR7iX0E9KFSj9ex/bM+oQDlFnvCVjZxYSGQ4A3A+8G/pekqT3U9X7SrUtW52A3gnSk8rVutl3r+mRbeHbIflindU8Wlj8AtJCuKfh0Hgoc1lW9EfFrSRMl/TMwKCLu6qE/Zn3G56DMem8z0CJpPwBJQyRNlbQTMCEifkk66tgV2BnYBOzSRV1HADMjYmJETAT2BmZHurVCm6RD8zaG5vv7XAd8JC9TGOJbk8sCHNZN218MbMzB6S2km95BmqX4vjxjkU5DhxcClwL/1cPrYtanHKDMeu8ZUhD4qqQ7SFfEfgNpqO9iSStIN5M7MyIeI02AeE/nSRJ5AsXLSLcdByDP4Htc0j7Ah4BPS7qTNAPwpRHxM2Ax0JpvXHdiLno6cLSk35DuzdWVS4DpklpJR1P35u2uJJ1Huyn36RudyowkBSmzfuOrmZtZtyQdRpoQ8qFGt8V2LD4HZWZdkvRt4J2kexWZ9SsfQZmZWSX5HJSZmVWSA5SZmVWSA5SZmVWSA5SZmVWSA5SZmVXS/wfQvQKw/sgsdwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(N_EPOCHS):\n",
    "    plt.clf()\n",
    "    plt.title(f'Test accuracy distribution after {i+1} {\"epoch\" if i == 0 else \"epochs\"}')\n",
    "    plt.hist(merged[i][f'TestAcc_{i}'], bins=30)\n",
    "    plt.xlabel('Test Accuracy')\n",
    "    plt.ylabel('Count')\n",
    "    plt.tight_layout()\n",
    "    out_png = f'plots/test_acc_dist_after_{i+1}_epochs.png'\n",
    "    assert not os.path.isfile(out_png)\n",
    "    plt.savefig(out_png)\n",
    "#     plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "means = [dfs[i][f'TestAcc_{i}'].mean() for i in range(N_EPOCHS)]\n",
    "meds  = [dfs[i][f'TestAcc_{i}'].median() for i in range(N_EPOCHS)]\n",
    "maxs  = [dfs[i][f'TestAcc_{i}'].max() for i in range(N_EPOCHS)]\n",
    "mins  = [dfs[i][f'TestAcc_{i}'].min() for i in range(N_EPOCHS)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeXxddZn48c+T5N4kNzdrm6b7ApQudAFS2cVWEEtVEAeGMsoIyKDjMD/UUSjOICqOCvx+jg4u4CDggnQURXZQsLUte8vSNmlLS5u2IVubtrnZc5fn98c5SW+z3iT3ZOvzfr3O696zPt9zKefJOd/v+X5FVTHGGGNGmpThLoAxxhjTHUtQxhhjRiRLUMYYY0YkS1DGGGNGJEtQxhhjRqS04S5AMo0fP15nzpw54P0bGxvJyspKXoEs3pDFG8vnZvFGd7yxfG7Jirdp06aDqlrYZYWqjpmpuLhYB2PNmjWD2t/iDV+8sXxuFm90xxvL55aseMBG7eaabo/4jDHGjEiWoIwxxoxIlqCMMcaMSJ4lKBF5QERqRGRrD+tFRP5bRHaJyGYROT1u3XIR2eGuW+VVGY0xxoxcXt5BPQQs72X9xcBsd7oB+BmAiKQCP3HXzweuEpH5HpbTGGPMCORZglLVdcChXja5FPiV24jjVSBPRCYBZwC7VHW3qrYBq91tjTHGHEdEPezNXERmAk+p6oJu1j0FfF9VN7jzLwK3ADOB5ap6vbv8auBMVb2xhxg34NyBUVRUVLx69eoBl7ehoYFgMDjg/S3e8MUby+dm8UZ3vLF8bsmKt2zZsk2quqTLiu7anidrwkk2W3tY9zRwXtz8i0AxcAVwf9zyq4F7Eoln70Edv/HG8rlZvNEdbyyfW7Li0cN7UMPZk0Q5MC1ufipQAfh7WG6MMaYXqkpUozRHmgnHwoSjYeezfYqGicQixyyLxCLdbnfMfA/HCsfCZIQyWMpST85nOBPUE8CNIrIaOBOoU9VKETkAzBaRWcD7wErgH4axnMaY45iqEtFjL+LHXOQ7X/Q7Xcg3N2ym9t3arkmhl4v+MdtGu0koveyrKDyc/N/BRwo+ScGH4ENIA3zAYp2Q/GAuzxKUiDwCLAXGi0g5cDvO+aCq9wLPACuAXUATcK27LiIiNwLPA6nAA6pa4lU5jTGjTzgapinSRHOkmaZwE02Rpi6f3a1rjjTTFGmi5mANDzz7QM8X+07LBu2V7hf7UnzulIZP0vClpOKTVCcRSAo+UkhzE0Imgg/wKc4y9eEjDR/p+CSGLyWKjyiRxgZy/D58sQi+aIS0aBhftA1fNIwv0oYvGsGH4tP2ibh58KmSpuouc+cBAUj1Q1om+DIgzZn2Z4zCBKWqV/WxXoF/6WHdMzgJzBgzRsQ0Rn1bPaHWEKG2EHWtddS11VHXWseWui1seWvLMUmlOdx8bOKJ+x6JRRKO60vxEfAFCKQ5U2ZaJlGi+FP8BHyBuCThc5KDmxB87QkBSIu5F+xYDJ/G8MUUn0bxxaJOIojFnAQQjbifYdIibfiibTQfPsi4TL8zH2nG19aKL9JMWqQVibQM/AdNSYtLFpmQlg6+DOqahNzMcZAet+6YT3fyZfb+mZbRdb+U1C7FeG/t2mPqZJJpTPVmbozxXjgapq6tjlBrqCPBdCSc9sldH5+IQq0h5/FTD+SIkJmWeTSZuJ+5GblMSpvUkWACKT4yJZWApBIghQAQUCUQUwKxKJnRCIFohECkjcxwK75wM7Q1OlNjA7TV0HDkIMGa/RBpgXCz8xlpAY0N8FeRHi/0R2Jp5AUmJJYUekwO3Xymdn/5fmvtWpYuXTrA8xhZLEEZc5yJxqI0RhppaGugvq2exnAjDWHne0NbA/XhekoOl7D25bXHJJ72702Rph6PLQjZ/mxy03PJ9eeSm57L1Oyp5PhznGW+ILmaQm4sSk64jdxwCzkt9Rzas4uTiwqRcJOTSJoaoa0B2qqOJpc2d1kvSa4LXxb4s8AfAH/Q+Z6eTUtGKsFJU+PuPDITSwo93YGk+kGk2yK8PYYSxlCzBGXMKBKOhWlsa6Q+7CSTjsQSbuiYb08y8Z+N4caO7RrDjX3GSSONvHBeR5KZFJzEHP+cYxJPbnquk3h82eSqkhNuJbulnpTGg9BQ7Uz1B6ByBzTUOPNNh+guwYyTNKQmeDSJtE85U7omGH9W1+18WV3X+QKQ0n1fBFvHSNKobwnzbnU9O6oa2FEVYkd1PeU1TUzc9jLZGWlkZ/g6fbpT+rHLczJ8BDPSSE3pPskOF0tQxgyR1mhrx11KY/hokolPMPXh+qPJJC4BHWo8RNtv2miJ9l1n4U/xE/QHyfZnE/QFCfqDFGYWEvQHO+aDvmPXZ/uyO/bJ8mXx6rpXWHZO8dHE0lDjTHXV0LDXWdZYc3S5RrsWJC0DgkUQnAAFJ8D0s5z5rEJ3ubsuOIF1L702JhKGV9oiMd470MCOqnp2VNc7n1X1vH+kuWObLH8qJ0/MpjCQgi81hQMNrew+2Eh9S4T6ljDhaN93ngF/areJLaf9e3oawU4Jr6pxoI9F+2YJypheqCot0ZZjWoM1R5rZ1ryNlrIWGtsaj7mLiU8s8Xc19W31CbUGy0zLPCZpZPuzmZQ1ifpYPbOnzz42mbifQX+Q7NQAQVIIKvhjEffxWNxjsfbvrQ1Qfwja9nfa5tjtzm88COvauhYwJQ2yJriJZSJMXHhsssmacPR7enaPj71M92IxZf/hpo4E1J6M9hxsJBJzEkxainBiYZDiGfn8w5nTmVOUzZyJ2UzJyyQlRVi7di1Ll57V5dgt4WhHsnI+3e+tkU7Lj66vaw5TfripY3lLuGsyWlKUysqPefN7WIIyY0Y4Fu69FVgvzY87b9/eHLkp3NRzxX7N0a+CkJUWIOgLEEwLkJ2WSUFqOjMCkwimphNMSSc7xUcwxU9Q0twplWxSCSJkk0KWQlosCtE2dwpDWxs0t1FVU83EcHsSaeqaeCLN3ZexO5La9RGZP8tJKu7jsvdrG5k+rzjuLsdNQhl5PT42M/1zoL41LgmF2FHdwM7qeprajt6NTivIZE5RDhedUsSciTnMKcpm1vgs/Gn9/2+Q4Uslw5dKYXb6gMscjsa6JLF3S94e8PH6YgnKjAqN4UaqGquobKyksrGSqsaqjvmqxiqqGqoI/zrx91V8Kb5jW4y5rcYmBiaS6cskkJpJpiqBSCuBtmYCrQ0EmusINB4i0HiQjIaD5ETDBGMxgrEYWckenjo13al4T/VBqp/cSAy0sFMyCbp1M53rZDp993XaJi29zzub3WvXMv2cpck8o+NWc0R5c9/ho3dFVfW8W11PbePRO9RxWX7mTMzmyg9M67gjml2UTTB9ZF2ifakpFGT5Kcjydyxr3d+16XmyjKyzN8elcCzMgaYDPSagysZK6tvqj9knRVKYEJjApKxJLBi3gJPlZOadMK8j4XTXXDngc5enBfClpEHjATiyDw6XwZG9cHgvVO9zvh/ZD50fyWVPgrwZMPls9h0OM/2E2R0JJD6ZJPQ9xUcrqRxugdpm5UCTUtMco6YxRk1DhAMNbRyob+VgQysHQq20RqKMj2VQkOVnnPgp8PvJT/MzLtNPQdDPuCw/BVnpFGT5KMhKJy/TR8owV3irKi3hGEea2zjSFOZIU5g693tdc5gjzc6yUHOYQ7Ut/LVua4/1H8H0o3UeWf60IT+37s/l6Pm0n0tdc5tzbnHbNLRG4IWXAaeOZ3ZRNhfOK2LOxOyOaXxw4Hc1Y5klKOMpVaWuta7HO5/KxkoONB8g1un9kxx/DpOyJjE5azKnTzidiVkTmZQ1iUnBSUzKmsT4zPGkpRz957t27VqWLl4aHxhajjhJp+Y9NxHtdZPPPmcKd2ouHRgPedNh0mKYdwnkz3Dm82ZC7lSnibFr99q1TO+mUj8aU2obW93k4iSZ9ulgg/u9IcSB+lbqmru/48vN9FGYnU5hMJ2FU/MoDKZTWVFOsGA8hxrbqG1sY29tE4cb26hv7f6F1RSB/IDzl25+VnsCO/rpLEt3lgX95Af8PT42isWU+pYIR+Ivvs1h6pqOXpzrmo9NQM76MG3RnivQfalCbqafnMw0Qg0xSg9XUN8SJtZHXb4IBNOdlmftSSuY3kvFfqf1AX8qNU0xNpcf6XIu8YnzmHNpDtMW6ftc8gI+8jJ9TMzJYM7EbPIy/YRqyvno2YuZU5TN1PzMYf/DYTSxBGUGLBqLUttSS01TzTFTdVM1NU017D6wm5t/ezPNnepHfCk+JmVNYmLWRM6cdKaTeNz59s+AL9BNwDC01kNLHYRKoCUErSForWdK+Rvw3PPHJqLW0LH7p+dC/nQYdxKceIGbgNqT0HRI7zpkQDgacy5ah8PUNR/q+Mv4jT1hXmneFpd8nGR0qLG12wtslj+V8W7SmT0hyDknjmN8ML0jERVmO9O4oJ/0tK6PTNaurWHp0sVdlrdGohxuDFPb2MqhxrZjptrGNg41tHGoqY2dNQ0camzjcFMbPY2wk52eRkHQSWB1dc188401HRfn3kblyfKnkpvpIzfgJy/Tx0kTguQFfORk+siLu2jnBpz5XHc+4E9F3EeNa91m36pKc1xlfqilawV+Q0uky/KBtFhj3UtdFgX8qW5ZB34u3f63m1/Ud3lMF5agTBeqSmO48Zhkc6D5ANWN1UcTUXMNtc21RDs1L06VVMZnjmdCYAKT/ZNZOHOhk4AyxjHJl01RSiYFCiltDUcTTEsI6nZCyyY34YSOXecmoS53PHFmA5RlHk0608/q+K5502nOmsoRzer6l35VmCO7w9Q17zm6rP0v6aY2Gtu6aT7t8u8qozA7nfFBP1PyMjh1Wm5Hshkfl3TGB9PJ8qguIT0tlYm5qUzMzeh7Y5w7vCNNTqKqbYhLZJ2msE84YUpe3AXZuWDnZvqcZQEfuZl+cjN9A6qw74mIEPCnEfCnUZST2Dl1pqq0RrpW5rd/b2qLUF62i7NOW+TpuZjBswR1nAnHwhxsOkhN87F3PAeaDhwz3/muByDbn01RoIgJgQmclH8ShZmFFAUmMEHSmRBuZUJziIJQFamHy6BiL02H3iew52UnwUS7abLcmc95y5+MHEjPgYxcyJ12dD49J25dDlF/NpUtft4LpfDilnJ8k+ZxpNlpGlu3v40jO9of31TSFn2/57CdHs9Mzstg3qScThdjH3lxF+mSt15nxYVLe/yreaRKTRHGBdMZF0znpF76+HTuaE4buoIlkYj02WJtbXiv3dWMApagxrCGtgbePvA2m6o38Wb1m+w8uJOGXzd0aTbtS/ExITCBCYEJnJx/MudNOY+iQBGFgUImBCZQlFFIYSRMZqgCDu+BQ7uhfDccWuN8j++ZQFKcpJI/k/rskwhMO+nYhJOe0ykJxSWfHvoWU1WqQ60dzXG373NaQe2sbqA10h47k6zy/eQF/B2Jpf3xTHzyyY17PNOefDJ9PT+e6UmZT0ZdcjJmtLEENYbUtdbxZvWbbKzeyKbqTWw7tI2YxkiTNOaPn8/CwEIWn7i4Ixm1J6H89HwkFnHqbw61J6AdcOhZ5/vhvRBtPRoo1Q/5MyF/Fsw8DwpmOT0FFJzgJKc0pwnqtrVrKepn7wB1zU7XLdur6nk37mXF+AYFE7LTmTMxm6vPmsGcidnMnZhDxY43+egFy5LwKxpjRgpLUKPYweaDbKrexKbqTWys3sjOwzsBp6ubRYWL+KeF/8SSiUtYNH4RAV+AdS8+z/mTpztJaN9mN/m4CenI/mO7q/EFnIRTOAdOXn40ARXMcvpH66bb/f5oCUePdt0S98Z8Zd3Rrnyy09M4eWI2H1s0qePdkDlF2eTHvYPRrnaX3c0YM9ZYghpFqhqr2Fi9kY1Vzh1SWagMcLrHObXwVD566kdZMnEJC8YvID017tn7+2/C0//G+RVvwvq4A2bkOklnSjEsvMK5I2pPRMEJSemmJhpT9h2K77olxI6qespqm4i6zd38qSmcOCHImbMKnLflJwaZMzGHybkZ9hjNmOOYJagRSlUpry93EpL7yO79BqeiP9uXzWlFp/Gp2Z+iuKiYeePm4UvxdT1IuBnWfBde+TEEi9gz8ypmFV/o3AXlz4JAQVLKGo0p1aEW9h9qovxwM/sPN7HvUBNv7mqm6sXnOvrvEoHpBQHmFGWzYuEk9/FcNjPGZeFLtdZTxphjWYIaIVSV3XW7Ox7XbaraRE2z09lbfno+xUXFfGbeZ1gycQmz82aT2tcjtrIN8MS/Oo/vTv8sXHQHe199i1kLlw6obAcaWp3k4yah8sNN7D/kJKOKI83HvHciAkXZGYzzwacXzYjruiVIwG//5IwxibGrxTCJaYzytnIe3vZwxyO7w62HASjMLGRJ0RKKi4pZMnEJJ+SekPijrpYQvHA7bHzAacjwj0/ACR/qdRdVpa453JFw4pNPezLq3IvxuCw/UwsCLJySy8ULJjGtIJNp+QGm5mcyJT+T9LRUt6ny/IH8PMYYYwlqOMQ0xk1/vYm1lWuhEqYEp/DBqR/sSErTsqcNrO7l3T/DU1+C+ko4+0ZY9u9OZ6I4jRL218f4S2k1+w81dSSf9juihk5d5uRkpDGtIMCJhVksPbmQaQVO8mn/tDshY4zX7CozDH5Z8kvWlq9lee5yvnLhV5gUnDS4AzbWwvO3wub/hcK58Pe/gqlLjq5ujfCJezaw+2AzvLQRgExfasddz5mzCtzEE2BaQSZT8wPkZnZTp2WMMUPIEtQQ23JgC//95n/zkRkfYYWuGFxyUoWSx+CZrzkdo35oFXzwK85wCnH+7593sPtgI/8438+nln2AafmZFGT5rYWcMWZEswQ1hBraGrh53c0UBgq5/ezbeeuVtwZ+sFAlPP1vsONpmHwaXPoEFJ3SZbO39x/hoZfL+MxZ0/lwXi2nTssbxBkYY8zQsba9Q0RV+far36aysZI7z7+T3PTcgR4I3vwV/ORMeO9FuOg78LkXuk1O4WiMVX/YzITsdG5ePneQZ2CMMUPL0zsoEVkO/AhIBe5X1e93Wp8PPACcCLQA16nqVnddGVAPRIGIqi5hFHv8vcd5ds+z3HjqjZw2YYCdcB7aA0/+H9izDmacB5f8N4w7scfNf75uN9ur6vn51cXkZFidkjFmdPEsQYlIKvAT4CNAOfCGiDyhqqVxm30deFtVLxORue72F8StX6aqB70q41DZU7eH7772XT4w8QNcv/D6/h8gFoXX7oO/3gGSCh//Lzj9Gkjp+QZ494EGfvTiTi5eMJGLTpk48MIbY8ww8fIO6gxgl6ruBhCR1cClQHyCmg98D0BVt4vITBEpUtVqD8s1pNqibdy87mbSU9P53nnf6/sF285qtsHjN8L7G2H2R53klDul111iMeXWP24hPS2Fb13S9dGfMcaMBqK9DZU5mAOLXA4sV9Xr3fmrgTNV9ca4bb4LZKjqV0TkDOBld5tNIrIHOAwocJ+q/ryHODcANwAUFRUVr169esBlbmhoIBjsOqrqYDx66FH+Vv83Pl/4eRYEFiQcT2Jhpu/7IzP2/o5oaiY7Z/8TNRPOT6h/vL/tD/NgSRvXnOJn6bSjj/a8OL/eDGW8sXxuFm90xxvL55aseMuWLdvUbTWOqnoyAVfg1Du1z18N3NNpmxzgQeBt4NfAG8Bid91k93MC8A5wfl8xi4uLdTDWrFkzqP27HG/fGl3w0AL9/mvf71+88k2qPz1H9fYc1d9fq1pfk3DM6rpmXXj7c3rFvS9rNBpLLJ5HhjLeWD43ize6443lc0tWPGCjdnNN9/IRXzkwLW5+KlARv4GqhoBrAcR5KWePO6GqFe5njYg8hvPIcJ2H5U2q6sZqbnvpNuYWzOXLxV9ObKdOnbuy8hGYu6Jfcb/5ZAktkRjf+9RCUlLsPSdjzOjlZYJ6A5gtIrOA94GVwD/EbyAieUCTqrYB1wPrVDUkIllAiqrWu98vAr7tYVmTKhqLsmr9Klqjrdx1/l34U7uOX9RF585dP/JtyOzfO0t/Ka3mmS1VfPWikzmxcOhu8Y0xxgueJShVjYjIjcDzOM3MH1DVEhH5grv+XmAe8CsRieI0nvicu3sR8Jjb00Ea8FtVfc6rsibb/2z5HzZWb+SOc+9gVu6s3jceQOeu3alvCXPbn7YypyibG87vuem5McaMFp6+B6WqzwDPdFp2b9z3V4DZ3ey3G1jsZdm88lbNW/zsnZ+xYtYKLj3x0l63LajdCD/9Ylznrl8Hf9aA4t713A6q61v42WdOx59m718bY0Y/6+ooiepa67hl3S1MCU7htrNu672vu/U/YNGWO7rt3LW/Nu09xG9e28tnz57JadPzB3wcY4wZSSxBJYmq8s2Xv8mBpgP8esWvCfr7qAN685cczltA/uf/2qVz1/5ojUS55Q9bmJSTwVc/OmfAxzHGmJHGngUlye/f/T0v7HuBm06/iQXjF/S+8ZH9cLiMg+PPGlRyAvjZ2vfYVdPAdy5bQDDd/t4wxowdlqCSYOfhndz1xl2cO/lc/vGUf+x7h7INABzJWziouLtq6vnpmvf4xOLJfHhu0aCOZYwxI40lqEFqjjTztb99jaAvyHfO+w4pksBPWrYBMvNpzJo+4LixmLLqD1vI9KfyjY/bsOrGmLHHngkN0l1v3MV7de9x30fuY3zm+MR2KlsPM86FRJJZDx5+fR8b9x7m7ssXUZg9uMeExhgzEtkd1CA8X/Y8j777KNctuI5zJp+T2E5H9sGRvTDr/AHHrapr4c5nt3PuSeO4vHjqgI9jjDEjmSWoAXq/4X2+9fK3WDh+ITeedmPfO7Rz65+Yed6A4qoqtz2+lXA0xn9+cqEN226MGbMsQQ1AJBbhlnW3oCh3nn8nvpR+DAZYtgEyC6Bw3oBiP7e1ir+UVvPlj5zMzPEDe6nXGGNGA6uDGoCfvv1T3jnwDnedfxfTsqf1vUO8svXO3VMvgw32pK45zDeeKGH+pByuP6+PLpSMMWaUszuofnqt8jXu33I/l510GRfPurh/Ox/e69RBzfzggGJ//9lt1Da0cuffLSIt1f7TGWPGNrvK9cOhlkPcuv5WZubOZNUZq/p/gEHUP726u5ZHXt/P586bxcKpuf2PbYwxo4w94kuQqnLbS7dR11rHzy78GQFfoP8HKdsAgXFO/3v90BKO8vU/bmFqfiZf/sjJ/Y9rjDGjkN1BJeg3237DuvJ1/NuSf2NOwQD6vFMdcP3Tj/+6i90HG/nuZQsJ+O1vCmPM8cESVAJKa0v5waYfsGzaMq6ae9XADnJkL9Tt73f90/aqEPf+7T0+ddoUzj+5cGCxjTFmFLIE1YfGcCM3r7uZcRnj+PY53x74e0cDqH+Kut0Z5WT6+A/rzsgYc5yx50V9+O5r32V//X7uv+h+8jL6NwT7Mfash8D4ftU//eqVMt7ef4QfXnkqBVkJDBtvjDFjiN1B9eLJ957kifee4IZFN/CBiR8Y+IFUnTuomedBgndg7x9p5u7nd3D+yYVceurkgcc2xphRyhJUD/aF9vGdV7/D6RNO5/OLPj+4gx0ug1B5wo/3VJX/eGwLqvCfn1xg3RkZY45LlqC6EY6GuXndzaSlpHHn+XeSljLIJ6Ed9U+JNZB4cnMla3Yc4N8uOplpBQNozm6MMWOA1UF140dv/oiS2hJ+uOyHTMyaOPgDlq2HrEIo7Lt5+pGmNr79ZAmLpuZy7bnWnZEx5vhlCaqT9eXr+WXpL7lyzpVcMP2CwR+wn/VP//n0Ng43hfnVdWeSmmKP9owxxy9LUHHqInX84KUfMDt/Nl9d8tXkHPTwHgi9n1D900u7DvL7TeX889ITmT85JznxjTFmlLIE5YppjF/X/pqmcBMPfvRBMtIyknPgPeudz5m9D1DYEo7y9ce2MHNcgJsumJ2c2MYYM4p52khCRJaLyA4R2SUiXXpXFZF8EXlMRDaLyOsisiDRfZPtwa0PsqNlB6vOWMUJeSck78BlGyBrAozvPen88IWd7K1t4ruXLSTDl5q8+MYYM0p5lqBEJBX4CXAxMB+4SkQ6d4fwdeBtVV0E/CPwo37sm1TnTjmXC3Mu5FOzP5W8gyZY/7T1/Tr+Z/1u/n7JVM45aXzy4htjzCjm5R3UGcAuVd2tqm3AauDSTtvMB14EUNXtwEwRKUpw36SaWzCXS/MvTe47R4d2Q30FzOq5eXkkGuPWP24hP+Dj6ysGNsquMcaMRaKq3hxY5HJguape785fDZypqjfGbfNdIENVvyIiZwAvA2cCs/raN+4YNwA3ABQVFRWvXr16wGVuaGggGAwOeP/OJlX8mTnv/oTXzvgJzYGp3cZbfyCd/93RxhcXp3PGJG+rBJN9fiMp3lg+N4s3uuON5XNLVrxly5ZtUtUlXVaoqicTcAVwf9z81cA9nbbJAR4E3gZ+DbwBLE5k3+6m4uJiHYw1a9YMav8uHv2c6t2zVWOxblf/7ukXde5/PKvXPfi6xnrYJpmSfn4jKN5YPjeLN7rjjeVzS1Y8YKN2c0338k/2cmBa3PxUoCJ+A1UNAdcCiPNsbY87Bfrad8Tro/5JVfllSRspAndYd0bGGNOFl3VQbwCzRWSWiPiBlcAT8RuISJ67DuB6YJ2btPrcd8SrfQ/qK3vs3uhPb7/P1tooNy+fy+S8zCEunDHGjHye3UGpakREbgSeB1KBB1S1RES+4K6/F5gH/EpEokAp8Lne9vWqrJ4oa3//qfsE9cjr+5kSFD5z1owhLJQxxowentbKq+ozwDOdlt0b9/0VoNsXhLrbd1Qp2wDBiTDuxC6rYjFlW0WID0xIte6MjDGmB9abuRdUnTuoWR/stv6p/HAz9a0RpufYz2+MMT2xK6QXandBQ3WP/e+VVNQBWIIyxphe2BXSC33UP5VWhkhNEaYG7ec3xpie2BXSC2UbIHsyFHTfp19JRYgTC7Pwp1r9kzHG9MQSVLKpOj2Y99L/XmlFiPmTbDgNY4zpTZ8Jyu241STq4E5orOmx/qm2oZWqUAunTM4d4oIZY8zoksgd1C4Rudvr3sTHjI76p+4TVGllCMAGJDTGmD4kkqAWAe8C94vIqyJyg4jY1bUnZeshZ0qP9U+lFW6Cskd8xhjTqz4TlKrWq+d5gBQAACAASURBVOr/qOo5wM3A7UCliPxSRE7yvISjSQLjP5VUhJicm0F+lr/b9cYYYxwJ1UGJyCUi8hjOgIL/DzgBeJLR3NODFw6+C40Heny8B84jPnu8Z4wxfUukq6OdwBrgblV9OW75oyJyvjfFGqX2rHM+e3j/qbktyu4DDaxYOGkIC2WMMaNTIglqkao2dLdCVf9PksszupVtgJypkD+z29Xbq0LE1OqfjDEmEYk0kviJiOS1z4hIvog84GGZRqcE658ATrFHfMYY06eEWvGp6pH2GVU9DJzmXZFGqQM7oOmg00FsD0orQ+RkpDE138Z/MsaYviSSoFJEJL99RkQK8HiYjlGpj/efwLmDmj85x0bPNcaYBCSSaP4f8LKIPOrOXwH8p3dFGqXK1kPuNMjrfgDCSDTG9soQnz7TBig0xphE9JmgVPVXIrIJWAYI8ClVLfW8ZKNJe/3T7It6rH/ac7CR1kjM6p+MMSZBCT2qc4dqPwBkAIjIdFXd52nJRpOabdBU22PzcrAujowxpr8SeVH3EhHZCewB/gaUAc96XK7RpWyD89nbC7oVIfypKZw0IThEhTLGmNEtkUYSdwBnAe+q6izgAuAlT0s12pSth9zpkN9z/VJJRYiTJwbxpdoIJ8YYk4hErpZhVa3Fac2XoqprgFM9LtfoEYs5d1C9NC9XVUorQ5wyyYbYMMaYRCVSB3VERILAOuBhEakBIt4WaxQ5sA2aD/X6eK8q1MKhxjarfzLGmH5I5A7qUqAJ+DLwHPAe8AkvCzWqtNc/zTi3x01KrQcJY4zpt17voNzRdB9X1QuBGPDLISnVaFK2HvL6rn8CmGt98BljTMJ6vYNS1SjQJCIDqjwRkeUiskNEdonIqm7W54rIkyLyjoiUiMi1cevKRGSLiLwtIhsHEt9z7fVPM3vv1L20IsTMcQGC6dYBhzHGJCqRK2YLsEVE/gI0ti/sqydz9+7rJ8BHgHLgDRF5otNLvv8ClKrqJ0SkENghIg+rapu7fpmqHuzH+QytmlJoPtxr/RNASWUdi6bk9bqNMcaYYyWSoJ52p/46A9ilqrsBRGQ1Tn1WfIJSIFuczumCwCFGUwOMBN5/qmsOs/9QMys/MH2ICmWMMWODqKo3Bxa5HFiuqte781cDZ6rqjXHbZANPAHOBbOBKVX3aXbcHOIyTxO5T1Z/3EOcG4AaAoqKi4tWrVw+4zA0NDQSDib9Ie8rW7xJs2MtrZ93X4zbbD0X5/ustfKU4nUWFx/490N94gzWW443lc7N4ozveWD63ZMVbtmzZJlVd0mWFqvY64fQgsbvzlMB+VwD3x81fDdzTaZvLgf/C6ePvJDdWjrtusvs5AXgHOL+vmMXFxToYa9asSXzjaFT1e9NV//TFXjf7xfrdOuOWp7S6rnlw8ZJgLMcby+dm8UZ3vLF8bsmKB2zUbq7piTzii89qGW7iKUhgv3JgWtz8VKCi0zbXAt93C7jLvWuaC7yuqhUAqlojIo/hPDJcl0DcoVFTAi1Heu1/D5wWfOOD6UzIyRiighljzNjQ53tQqlobN72vqj8EPpzAsd8AZovILBHxAytxHufF24fTdRIiUgTMAXaLSJb7+A8RyQIuArYmfFZDIYH6J3A6ibUXdI0xpv/6vIMSkdPjZlNw7qiy+9pPVSMiciPwPJAKPKBOr+hfcNffi9PP30MisgXnMd8tqnpQRE4AHnMH9ksDfquqz/Xv1Dy2Zz3kz4LcqT1u0haJsaumnqVzCoewYMYYMzYkOmBhuwhOPdHfJ3JwVX0GeKbTsnvjvlfg3B113m83sDiRGMMiFoO9L8G83jvUeLe6nnBUmW8v6BpjTL8lMmDhsqEoyKhSvTWh+qf2MaCsiyNjjOm/RMaD+q6I5MXN54vId7wt1ghXtt757Kv+qSJEwJ/KzHFZQ1AoY4wZWxLpLPZiVT3SPqOqh4EV3hVpFCjbAAUnQO6UXjcrrQgxb1IOKSndDwNvjDGmZ4kkqFQRSW+fEZFMIL2X7ce2WNSpf+rj7ikWc8aAsvonY4wZmEQaSfwGeFFEHsTp1eE6judezau2QEtdnx3E7j/cRENrxOqfjDFmgBJpJHGXiGwGLsRpCn6Hqj7veclGqo73n3oe/wmODrFh70AZY8zAJPIe1Cxgbft7SCKSKSIzVbXM68KNSGUboOBEyJnc62alFSFSU4STi/p8ZcwYY0w3EqmD+j3OYIXtou6y408sCntfhlm9Ny8HKKmo46TCIBm+1CEomDHGjD2JJKg0PTo+E+53v3dFGsGqNkNrXZ/vP4F1cWSMMYOVSII6ICKXtM+IyKXAyB1E0Evt9U8zeq9/OtjQSnWo1RpIGGPMICTSiu8LwMMi8mOcRhL7cYbOOP6UbYBxJ0HOpF43K21vIGFNzI0xZsASacX3HnCWiARxBjisF5EPAO95XrqRJBpx6p8W/F2fm7Z3cWSP+IwxZuASuYNqNx1YKSIrgRDHjhM19lVthtZQny/ogtPEfEpeJnmB47OqzhhjkqHXBCUiM4Cr3CkCzACWHJdNzBMc/wmgtKLO7p6MMWaQemwkISIv4wyV4QMuV9VioP64TE7gdBA7/mTIntjrZk1tEXYfbLT6J2OMGaTeWvEdwBmYsAhoH3FPPS/RSBSNwN5XErp72l5Vj6oNsWGMMYPVY4JS1UuBhcCbwLdEZA+QLyJnDFXhRoyqd6CtPuH6J7AGEsYYM1i91kGpah3wAPCAiEwArgR+KCLTVHXaUBRwROh4/ymR+qcQuZk+puRlelwoY4wZ2xJ5URcAVa1R1XtU9Ryg7yv1WLJnPYyfA9lFfW5aWlHH/Ek5iNgYUMYYMxgJJ6h4qro32QUZsaIR2JdY/VMkGmN7Vb3VPxljTBIMKEEdVyrfgbaGhBLU7oONtEZiVv9kjDFJ0GeCEpEuHc91t2zMKlvnfCbSQazbQOKUyblelsgYY44LidxB3ZPgsrGpbAMUzoVgYZ+bllTU4U9L4YTCrCEomDHGjG09tuITkbOBc4BCEflK3Koc4PgY5Cgahn2vwuKVCW1eWhliTlE2vlR7cmqMMYPV25XUDwRxklh23BQCLk/k4CKyXER2iMguEVnVzfpcEXlSRN4RkRIRuTbRfYdER/1T34/3VJXSipA1kDDGmCTp8Q5KVf8G/E1EHmpvtSciKUBQVUN9HVhEUoGfAB8ByoE3ROQJVS2N2+xfgFJV/YSIFAI7RORhnFF7+9rXe3vc+qc+xn8CqKxr4XBT2BpIGGNMkiTyLOp7IpIjIllAKU4S+VoC+50B7FLV3e4ovKuBSztto0C2OC8NBYFDOJ3SJrKv98o2QOG8hOqfjjaQsARljDHJIKq9d68nIm+r6qki8mmgGLgF2KSqi/rY73Jguape785fDZypqjfGbZMNPAHMxXl8eKWqPp3IvnHHuAG4AaCoqKh49erVCZ56Vw0NDQSDQee4sQjnbfg0VRM/zM6TP9/nvo/vauNPu8L87MIAGWmJvaQbH28ojOV4Y/ncLN7ojjeWzy1Z8ZYtW7ZJVbsO4aSqvU5ACU6P5r8HPuQueyeB/a4A7o+bvxq4p9M2lwP/hTNS70nAHpxGGH3u291UXFysg7FmzZqjM/teU709R7XkTwnte8Ov3tBld6/pc7se4w2BsRxvLJ+bxRvd8cbyuSUrHrBRu7mmJ/KI7z6gDMgC1rljRPVZB4VTdxTfX99UoKLTNtcCf3TLuMtNUHMT3NdbZeudzwTqn8DpJHaePd4zxpik6TNBqep/q+oUVV3hJpK9wLIEjv0GMFtEZomIH1iJ8zgv3j7gAgARKQLmALsT3NdbZRtgwnzIGt/npnXNYcoPN1v9kzHGJFEiPUkUicgvRORZd34+8Nm+9lPVCHAj8DywDfidqpaIyBdE5AvuZncA54jIFuBF4BZVPdjTvgM4v4GJtDnvPyXQvByONpCwQQqNMSZ5eh1uw/UQ8CDw7+78u8D/Ar/oa0dVfQZnVN74ZffGfa8ALkp03yFT8RaEmxLqfw+cF3TBujgyxphk6m3I9/bkNV5VfwfEoOPOKDoEZRs+/a5/qqMwO53C7HQPC2WMMceX3h7xve5+NorIONzh3kXkLKDO64INq7INULQAssYltLn1IGGMMcnXW4Jqf5nnKzgNFE4UkZeAXwH/6nXBhk2kDfa/lvDjvdZIlF01DVb/ZIwxSdZbHVR8J7GP4dQHCdAKXAhs9rhsw6PizX7VP+2sbiASU6t/MsaYJOstQaXidD/UuVuEgHfFGQHK1gOScP1TRws+e8RnjDFJ1VuCqlTVbw9ZSUaKPeud+qdAQUKbl1TUkeVPZUbB2M7bxhgz1BKpgzpuSCwM+19P+PEeOE3M503KISXluPu5jDHGU70lqAuGrBQjRE5oJ0SaE05QsZgzBpQ93jPGmOTrMUGp6qGhLMhIkHdkC0790zkJbb/vUBONbVFrYm6MMR6wscnj5B3ZChP7U//U3sWRteAzxphkswTVLtJKTmh7wv3vAZRW1pGWIswuGrqxV4wx5nhhCard+5tIjbX1K0GVVIQ4aUKQDF+qhwUzxpjjkyWodnvWowjMODvhXayBhDHGeCeR3syPDws+xfaqRuZl5ie0+YH6VmrqW62LI2OM8YjdQbUbP5vqiR9OeHMbYsMYY7xlCWqASiqcDt3tDsoYY7xhCWqASitCTM3PJDfgG+6iGGPMmGQJaoBKK0N292SMMR6yBDUAja0R9hxstPonY4zxkCWoAdheVY+qDbFhjDFesgQ1AKVuAwnrg88YY7xjCWoASitD5AV8TMrNGO6iGGPMmGUJagBKKpwGEiI2BpQxxnjFElQ/RaIxtlfV2+M9Y4zxmKcJSkSWi8gOEdklIqu6Wf81EXnbnbaKSFRECtx1ZSKyxV230cty9sd7Bxppi8SsgYQxxnjMs774RCQV+AnwEaAceENEnlDV0vZtVPVu4G53+08AX+40UOIyVT3oVRkHorSyvYGENTE3xhgveXkHdQawS1V3q2obsBq4tJftrwIe8bA8SVHyfoj0tBROGJ813EUxxpgxTVTVmwOLXA4sV9Xr3fmrgTNV9cZutg3g3GWd1H4HJSJ7gMOAAvep6s97iHMDcANAUVFR8erVqwdc5oaGBoLB3gcfvPP1ZlqicPvZmQOO0594yTSW443lc7N4ozveWD63ZMVbtmzZJlVd0mWFqnoyAVcA98fNXw3c08O2VwJPdlo22f2cALwDnN9XzOLiYh2MNWvW9Lo+Fovpom8+r6v+8M6g4iQaL9nGcryxfG4Wb3THG8vnlqx4wEbt5pru5SO+cmBa3PxUoKKHbVfS6fGeqla4nzXAYziPDIdVRV0Ldc1h5lv9kzHGeM7LBPUGMFtEZomIHycJPdF5IxHJBT4EPB63LEtEstu/AxcBWz0sa0JKK5wxoKyTWGOM8Z5nrfhUNSIiNwLPA6nAA6paIiJfcNff6256GfBnVW2M270IeMx9ETYN+K2qPudVWRNVUlGHCMyblD3cRTHGmDHP0yHfVfUZ4JlOy+7tNP8Q8FCnZbuBxV6WbSBKK0LMGp9FwO/pz2aMMQbrSaJfSipC9v6TMcYMEUtQCaprCvP+kWarfzLGmCFiCSpBJZU2xIYxxgwlS1AJam/BN8/uoIwxZkhYbX+CSitCTMhOpzA7fbiLYowZocLhMOXl5bS0tHQsy83NZdu2bUNWhpEcLyMjg6lTp+Lz+RLa3hJUgkorQ/Z4zxjTq/LycrKzs5k5c2bHeHH19fVkZw/dqykjNZ6qUltbS3l5ObNmzUro2PaILwEt4Sg7axpsiA1jTK9aWloYN26cDWbaDRFh3Lhxx9xd9sUSVAJ2VjcQjak1MTfG9MmSU8/6+9tYgkpASYXTgs+amBtjzNCxBJWA0soQwfQ0phcEhrsoxhjTq5ycHK6++uqO+UgkQmFhIR//+McBeOKJJ/j+978/XMXrF2skkYDSihDzJmWTkmK37saYkS0rK4utW7fS3NxMZmYmf/nLX5gyZUrH+ksuuYRLLrlkGEuYOEtQfYjFlG2VIa5YMq3vjY0xxvWtJ0sorQgRjUZJTU1NyjHnT87h9k+c0ud2F198MU8//TSXX345jzzyCFdddRXr168H4KGHHmLjxo38+Mc/5pprriEnJ4eNGzdSVVXFXXfdxeWXX56UsiaDPeLrw95DTTS2Ra3+yRgzaqxcuZLVq1fT0tLC5s2bOfPMM3vctrKykg0bNvDUU0+xatWqISxl3+wOqg8dDSSsibkxph/a73SG+r0kgEWLFlFWVsYjjzzCihUret32k5/8JCkpKcyfP5/q6uohKmFiLEH1obQiRFqKMLsoONxFMcaYhF1yySV89atfZe3atdTW1va4XXr60d5xnNHXRw5LUH0oqQgxuyib9LTkPEM2xpihcN1115Gbm8vChQtZu3btcBdnQKwOqg+llSGrfzLGjDpTp07lpptuGu5iDIrdQfWipr6FA/Wt1gefMWbUqKys7LJs6dKlLF26FIBrrrmGa665BnBa9MVraGjwuHT9Y3dQvWgfYsMaSBhjzNCzBNWLEktQxhgzbCxB9aK0MsS0gkxyMhIbu8QYY0zyWILqRWmFNZAwxpjhYgmqBw2tEcpqG22IDWOMGSaWoHqwoyqEqg2xYYwxw8XTBCUiy0Vkh4jsEpEunTyJyNdE5G132ioiUREpSGRfr7U3kDhliiUoY8zo0ddwG4launQpGzduBGDFihUcOXIkqeVMhGcJSkRSgZ8AFwPzgatEZH78Nqp6t6qeqqqnArcCf1PVQ4ns67XSihD5AR8TczKGMqwxxgxK/HAbQJfhNgbimWeeIS8vLxnF6xcvX9Q9A9ilqrsBRGQ1cClQ2sP2VwGPDHDfpCupCHHK5FwbvtkYMzDProKqLWRGI5CapEvtxIVwcd+DDfY23EZjYyP/+q//ypYtW4hEInzzm9/k0ksvpbm5mWuvvZbS0lLmzZvXkeAAZs6cycaNGxk/fjyf/OQn2b9/Py0tLdx0001cddVVAASDQW666SaeeuopMjMzefzxxykqKhrU6XqZoKYA++Pmy4Fu+3wXkQCwHLhxAPveANwAUFRUNKg+pxoaGli7di2RmLKtsokLp/s87cOqPd5QGcvxxvK5WbzREy83N5f6+noA0sNtpEQjoBCJRpJy/Fi4jVb3+L35xCc+wZ133smHPvQh3n77bVauXMmaNWuor6/nW9/6FmeffTY/+tGPOHLkCMuWLePMM8/kwQcfxOfz8dJLL7F161Y++MEP0tjYSH19PapKQ0MD6enp/OhHP6KgoIDm5maWLl3KhRdeCDiJb/HixaxatYrbbruNH//4x9x8881dytbS0pL4b6+qnkzAFcD9cfNXA/f0sO2VwJMD2Td+Ki4u1sFYs2aNqqpuq6zTGbc8pX96q3xQx0s03lAZy/HG8rlZvNETr7S0tMuyUCjkSayeZGVlqapqcXGxPvDAA3rrrbfqmjVr9GMf+1jH8lNOOUUXL16sixcv1mnTpmlpaaleeuml+uKLL3Yc57TTTtM33nhDVVVnzJihBw4cUFXV22+/XRctWqSLFi3SnJwcfeGFF1RV1e/3aywWU1XV1atX6+c+97luy9fdbwRs1G6u6V7eQZUD8cPQTgUqeth2JUcf7/V336Tr6OLIWvAZY0apnobbUFX+8Ic/MGfOnC779FWlsXbtWl544QVeeeUVAoEAS5cupbW1FQCfz9exf2pqKpHI4O8avWzF9wYwW0RmiYgfJwk90XkjEckFPgQ83t99vVJSESLDl8IJhTYGlDFmdLruuuv4xje+wcKFC49Z/tGPfpR77rmnY+ynt956C4Dzzz+fhx9+GICtW7eyefPmLsesq6sjPz+fQCDA9u3befXVVz09B88SlKpGcOqUnge2Ab9T1RIR+YKIfCFu08uAP6tqY1/7elXWzkorQsyZmENqijWQMMaMTj0Nt3HbbbcRDodZtGgRCxYs4LbbbgPgn//5n2loaGDRokXcddddnHHGGV32Xb58OZFIhEWLFnHbbbdx1llneXoOng63oarPAM90WnZvp/mHgIcS2XcoqColFXV8fPHkoQ5tjDGD1tdwG5mZmdx3331dtsnMzGT16tXdHrOsrKzj+7PPPnvMuvZGIfFDdVx++eVcfvnl/S16F9aTRCfvH2km1BKx+idjjBlmlqA66ehBwobYMMaYYWUJqpPSihApAnMnWoIyxpjhZAmqk9LKELPGZ5HpTx3uohhjzHHNElQnpW4XR8YYY4aXJag4DW3K+0eabYh3Y4wZASxBxdlXHwOsgYQxZvRK1nAbI4ElqDj7Qk6CsibmxpjRyovhNoaLpy/qjjZ766NMzMlgXDB9uItijBnl7nz9TrYf2k40GiU1NTmNruYWzOWWM27pc7vehtt4/fXX+dKXvkRzczOZmZk8+OCDzJkzhx/84Ads3bqVBx54gC1btnDVVVfx+uuvEwgEklL2gbA7qDj7QzGrfzLGjHorV65k9erVtLS0sHnzZs488+hoRXPnzmXdunW89dZbfPvb3+brX/86AF/60pfYtWsXjz32GNdeey333XffsCYnsDuoDi3hKBWNymWWoIwxSdB+p1NfX092dvaQxl60aBFlZWU88sgjrFix4ph1dXV1fPazn2Xnzp2ICOFwGICUlBQeeughFi1axOc//3nOPffcIS1zd+wOyvVudT0xtfonY8zY0D7cRvuIt+1uu+02li1bxtatW3nyySdpaWnpWLdz506CwSAVFUM2ulGvLEG5jnZxZO9AGWNGv56G26irq+toNPHQQw8ds/ymm25i3bp11NbW8uijjw5lcbtlCcrVFolRFBCm5mcOd1GMMWbQehpu4+abb+bWW2/l3HPPJRqNdiz/8pe/zBe/+EVOPvlkfvGLX7Bq1SpqamqGsshdWB2U67PnzGRGWxkpNgaUMWYU62u4jbPPPpt33323Y90dd9wBwAMPPNCxbNq0aezatcvbgibA7qCMMcaMSJagjDHGjEiWoIwxJolUdbiLMGL197exBGWMMUmSkZFBbW2tJaluqCq1tbVkZGQkvI81kjDGmCSZOnUq5eXlHDhwoGNZS0tLvy7KgzWS42VkZDB16tSEj20JyhhjksTn8zFr1qxjlq1du5bTTjttyMowluLZIz5jjDEjkiUoY4wxI5IlKGOMMSOSjKXWJiJyANg7iEOMBw4mqTgWb2jjjeVzs3ijO95YPrdkxZuhqoWdF46pBDVYIrJRVZdYvNEXbyyfm8Ub3fHG8rl5Hc8e8RljjBmRLEEZY4wZkSxBHevnFm/UxhvL52bxRne8sXxunsazOihjjDEjkt1BGWOMGZEsQRljjBmRLEEBIvKAiNSIyNYhijdNRNaIyDYRKRGRruMyJy9Whoi8LiLvuLG+5VWsTnFTReQtEXlqCGKVicgWEXlbRDYOQbw8EXlURLa7/w3P9jDWHPe82qeQiHzJw3hfdv+dbBWRR0TE015HReQmN1aJF+fV3f/bIlIgIn8RkZ3uZ77H8a5wzy8mIkltjt1DvLvdf5ubReQxEcnzON4dbqy3ReTPIjI5WfFQ1eN+As4HTge2DlG8ScDp7vds4F1gvkexBAi6333Aa8BZQ3COXwF+Czw1BLHKgPFD+O/ll8D17nc/kDdEcVOBKpyXGr04/hRgD5Dpzv8OuMbD81kAbAUCOB1XvwDMTnKMLv9vA3cBq9zvq4A7PY43D5gDrAWWDMH5XQSkud/vHILzy4n7/n+Ae5MVz+6gAFVdBxwawniVqvqm+70e2IZzcfAilqpqgzvrcydPW8aIyFTgY8D9XsYZDiKSg/M/6S8AVLVNVY8MUfgLgPdUdTC9pfQlDcgUkTScxFHhYax5wKuq2qSqEeBvwGXJDNDD/9uX4vyRgfv5SS/jqeo2Vd2RrBgJxPuz+3sCvAokPr7FwOKF4mazSOL1xRLUMBORmcBpOHc2XsVIFZG3gRrgL6rqWSzXD4GbgZjHcdop8GcR2SQiN3gc6wTgAPCg+wjzfhHJ8jhmu5XAI14dXFXfB/4vsA+oBOpU9c9excO5ezpfRMaJSABYAUzzMF67IlWtBOePRWDCEMQcLtcBz3odRET+U0T2A58GvpGs41qCGkYiEgT+AHyp018hSaWqUVU9FecvqTNEZIFXsUTk40CNqm7yKkY3zlXV04GLgX8RkfM9jJWG84jjZ6p6GtCI85jIUyLiBy4Bfu9hjHycu4tZwGQgS0Q+41U8Vd2G8wjqL8BzwDtApNedTMJE5N9xfs+HvY6lqv+uqtPcWDcm67iWoIaJiPhwktPDqvrHoYjpPopaCyz3MMy5wCUiUgasBj4sIr/xMB6qWuF+1gCPAWd4GK4cKI+7C30UJ2F57WLgTVWt9jDGhcAeVT2gqmHgj8A5HsZDVX+hqqer6vk4j452ehnPVS0ikwDcz5ohiDmkROSzwMeBT6tbOTREfgv8XbIOZglqGIiI4NRhbFPVH3gcq7C9FY+IZOJchLZ7FU9Vb1XVqao6E+eR1F9V1bO/wkUkS0Sy27/jVBB71hpTVauA/SIyx110AVDqVbw4V+Hh4z3XPuAsEQm4/0YvwKkf9YyITHA/pwOfwvtzBHgC+Kz7/bPA40MQc8iIyHLgFuASVW0agniz42YvIZnXl2S2KBmtE87/FJVAGOcv5M95HO88nHqTzcDb7rTCo1iLgLfcWFuBbwzh77oUj1vx4dQJveNOJcC/D8F5nQpsdH/TPwH5HscLALVA7hCc27fcC8xW4NdAusfx1uMk+HeACzw4fpf/t4FxwIs4d2svAgUex7vM/d4KVAPPexxvF7A/7tqStFZ1PcT7g/vvZTPwJDAlWfGsqyNjjDEjkj3iM8YYMyJZgjLGGDMiWYIyxhgzIlmCMsYYMyJZgjLGGDMiWYIypgcicpmIqIjMjVtWKCKvud0cfVBEvpiEOGvje2EXkSUiXhb3bgAAAy9JREFUsnawxx1AOa4RkR8PdVxjemIJypieXQVswHnhuN0FwHZ1ujnaD/QrQYmju//vJojIxQMuqTFjkCUoY7rh9pN4Ls6LiCvdZafiDNWwwu18907gRHccnLvdbb4mIm+44+N8y1020x036qfAm3TfIerdwH8kUK6ejr9dRH7pLn/U7XwVEbnAvdvb4o7lk+4u/4CIvCzOOGGvt/fGAUwWkefcsZLucrdNFZGHxBm3aYuIfHlgv6ox/WMJypjufRJ4TlXfBQ6JyOmq+jZOT83/q07nu7fgDH9xqqp+TUQuAmbj9AV4KlAc13HtHOBXqnqadj9cxitAq4gs66lACRz/56q6CAgBXxRnsMGHgCtVdSFOR7f/7HY8+7/ATaq6GKf7q2b3OKcCVwILgStFZJq7bIqqLnCP82DiP6MxA2cJypjuXYXT2S3u51UJ7HORO72Fc6c0FyehAOxV1Vf72P879H4X1dvx96vqS+733+B0pzUHp/PXd93lv8QZy2oOUKmqb4Azno8eHT/oRVWtU9UWnC6IZgC7gRNE5B63nzfPet43Jl7acBfAmJFGRMYBHwYWiIjijGSrInJzX7sC31PV+zodbybOsBy9UtW/isgdwFkDOH7nPsvU3b6n4/TUx1lr3Pcozsish0Vk8f9v745RGgiiMI7/P/t0dgFNYeMBLDyBFxBFUURbT2BhKdhbiGeQdKJoLxa2guk8gFgpKDbP4s1C0CREJDrC96t2Z2Fgq7dvZpkPWAJ2gRUyZ8hsotxBmX21TC7HzUZEJzLn5oHsSvo9A62++0tgp+xfIandnNb9DQdk2OMgo+afkbRYrpufO3pAR9JcGd8kU2t75F7TQpmnpUzQHUjSNDAVEV1gn9+JFzFzB2U2wBpw+GmsC6zTl3wcEU+SriXdARdlH2oeuMm0Cl6ADbITGUtEnEt6HPLsasT898CWpBPylO7jiHiTtA2clgJ0S55s/S5pFTgqESyv5D7UMG0yQbj5oN0b933MfsKnmZv9c2WJ7ywiJpaUbPYXvMRnZmZVcgdlZmZVcgdlZmZVcoEyM7MquUCZmVmVXKDMzKxKLlBmZlalD7udGN3nSDqtAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = range(1, N_EPOCHS + 1)\n",
    "plt.plot(x, mins, label='Min')\n",
    "# plt.plot(means, label='Mean')\n",
    "plt.plot(x, meds, label='Median')\n",
    "plt.plot(x, maxs, label='Max')\n",
    "\n",
    "plt.xticks(x)\n",
    "plt.xlabel('After N epochs')\n",
    "plt.ylabel('Test Accuracy')\n",
    "plt.grid()\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "out_png = 'plots/test_acc_all_epochs.png'\n",
    "assert not os.path.isfile(out_png)\n",
    "plt.savefig(out_png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.67138671875 0.77197265625 0.7755398220486112 0.9033203125\n",
      "0.8935546875 0.939453125 0.937208726671007 0.9609375\n",
      "0.95947265625 0.966796875 0.9663064139229911 0.97119140625\n",
      "0.96484375 0.97265625 0.9726129441034226 0.9755859375\n",
      "0.97216796875 0.97509765625 0.9762707907056052 0.98046875\n",
      "0.97509765625 0.98046875 0.9798473539806548 0.982421875\n",
      "0.97900390625 0.9814453125 0.98154296875 0.98486328125\n",
      "0.9755859375 0.9833984375 0.9834171355716765 0.9873046875\n",
      "0.978515625 0.9873046875 0.9870466928633432 0.990234375\n",
      "0.97607421875 0.9892578125 0.9888883560422868 0.9921875\n",
      "0.97900390625 0.99072265625 0.9895095582992312 0.9931640625\n",
      "0.986328125 0.9921875 0.9912748790922619 0.994140625\n",
      "0.9912109375 0.99365234375 0.9933253046291978 0.99462890625\n"
     ]
    }
   ],
   "source": [
    "for mn, md, mean, mx in zip(mins, meds, means, maxs):\n",
    "    print(mn, md, mean, mx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_prev_vs_med(4) = 1\n",
      "best_prev_vs_med(5) = 1\n",
      "best_prev_vs_med(6) = 1\n",
      "best_prev_vs_med(7) = 1\n",
      "best_prev_vs_med(8) = 1\n",
      "best_prev_vs_med(9) = 1\n",
      "best_prev_vs_med(10) = 1\n",
      "best_prev_vs_med(11) = 2\n",
      "best_prev_vs_med(12) = 1\n"
     ]
    }
   ],
   "source": [
    "for i in range(N_EPOCHS):\n",
    "    best_prev_vs_med = i\n",
    "    best_prev_vs_min = i\n",
    "    best_prev_vs_avg = i\n",
    "    for prev in range(i - 1, 0, -1):\n",
    "        if maxs[prev] >= mins[i]:\n",
    "            best_prev_vs_min = prev\n",
    "        if maxs[prev] >= meds[i]:\n",
    "            best_prev_vs_med = prev\n",
    "        if maxs[prev] >= means[i]:\n",
    "            best_prev_vs_avg = prev\n",
    "    if best_prev_vs_med != i:\n",
    "        print(f'best_prev_vs_med({i}) = {i - best_prev_vs_med}')\n",
    "#     if best_prev_vs_avg != i:\n",
    "#         print(f'best_prev_vs_avg({i}) = {i - best_prev_vs_avg}')\n",
    "#     if best_prev_vs_min != i:\n",
    "#         print(f'best_prev_vs_min({i}) = {i - best_prev_vs_min}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{table}[H]\n",
      "\\centering\n",
      "\\begin{tabular}{l c c c c c c}\n",
      "\\cmidrule[2pt]{1-6}\n",
      "\\multirow{1}{*}{\\textbf{Epoch}} & \\multicolumn{1}{c}{Min Accuracy} & \\multicolumn{1}{c}{Mean Accuracy} & \\multicolumn{1}{c}{Max Accuracy} & \\multicolumn{1}{c}{Min Loss} & \\multicolumn{1}{c}{Max Loss} \\\\\n",
      "\\cmidrule{1-6}\n",
      "\t\\hfill 1 & 0.671 & 0.776 & 0.903 & 2.393 & 2.449 \\\\\n",
      "\t\\hfill 2 & 0.894 & 0.937 & 0.961 & 1.643 & 1.714 \\\\\n",
      "\t\\hfill 3 & 0.959 & 0.966 & 0.971 & 0.983 & 1.023 \\\\\n",
      "\t\\hfill 4 & 0.965 & 0.973 & 0.976 & 0.626 & 0.767 \\\\\n",
      "\t\\hfill 5 & 0.972 & 0.976 & 0.980 & 0.431 & 0.582 \\\\\n",
      "\t\\hfill 6 & 0.975 & 0.980 & 0.982 & 0.322 & 0.429 \\\\\n",
      "\t\\hfill 7 & 0.979 & 0.982 & 0.985 & 0.254 & 0.294 \\\\\n",
      "\t\\hfill 8 & 0.976 & 0.983 & 0.987 & 0.217 & 0.324 \\\\\n",
      "\t\\hfill 9 & 0.979 & 0.987 & 0.990 & 0.190 & 0.294 \\\\\n",
      "\t\\hfill 10 & 0.976 & 0.989 & 0.992 & 0.178 & 0.345 \\\\\n",
      "\t\\hfill 11 & 0.979 & 0.990 & 0.993 & 0.162 & 0.320 \\\\\n",
      "\t\\hfill 12 & 0.986 & 0.991 & 0.994 & 0.144 & 0.237 \\\\\n",
      "\t\\hfill 13 & 0.984 & 0.994 & 0.995 & 0.126 & 0.269 \\\\\n",
      "\\cmidrule{1-6}\n",
      "\\end{tabular}\n",
      "\\caption{Summary of results on test data after each epoch\\label{tab:summary}}\n",
      "\\end{table}\n"
     ]
    }
   ],
   "source": [
    "# \\begin{tabular}{l c c c c c}\n",
    "#     \\cmidrule[2pt]{1-5} %\\toprule\n",
    "#     \\multirow{1}{*}{\\textbf{Epochs}} & \\multicolumn{1}{c}{Max Accuracy} & \\multicolumn{1}{c}{Min Accuracy} & \\multicolumn{1}{c}{Max Loss} & \\multicolumn{1}{c}{Min Loss} \\\\ \n",
    "#     \\cmidrule{1-5}\n",
    "#     \\hfill 1 & 0.9033203125 & 0.6713867188 & 2.449115098 & 2.393164754 \\\\\n",
    "\n",
    "#     \\hfill 2 & 0.9609375 & 0.8935546875 & 1.71363318 & 1.643359601 \\\\\n",
    "\n",
    "#     \\hfill 3 & 0.9711914063 & 0.9594726563 & 1.023167834 & 0.9834425896 \\\\\n",
    "\n",
    "#     \\hfill 4 & 0.9755859375 & 0.96484375 & 0.7668097764 & 0.6259695292 \\\\\n",
    "#     \\cmidrule[2pt]{1-5}\n",
    "# \\end{tabular}\n",
    "\n",
    "# means = [dfs[i][f'TestAcc_{i}'].mean() for i in range(N_EPOCHS)]\n",
    "# meds  = [dfs[i][f'TestAcc_{i}'].median() for i in range(N_EPOCHS)]\n",
    "# maxs  = [dfs[i][f'TestAcc_{i}'].max() for i in range(N_EPOCHS)]\n",
    "# mins  = [dfs[i][f'TestAcc_{i}'].min() for i in range(N_EPOCHS)]\n",
    "\n",
    "print(r'\\begin{table}[H]')\n",
    "print(r'\\centering')\n",
    "print(r'\\begin{tabular}{l c c c c c c}')\n",
    "print(r'\\cmidrule[2pt]{1-6}')\n",
    "head = ' & '.join([\n",
    "    r'\\multirow{1}{*}{\\textbf{Epoch}}',\n",
    "    r'\\multicolumn{1}{c}{Min Accuracy}',\n",
    "    r'\\multicolumn{1}{c}{Mean Accuracy}',\n",
    "    r'\\multicolumn{1}{c}{Max Accuracy}',\n",
    "    r'\\multicolumn{1}{c}{Min Loss}',\n",
    "    r'\\multicolumn{1}{c}{Max Loss}',\n",
    "]) + r' \\\\'\n",
    "print(head)\n",
    "print(r'\\cmidrule{1-6}')\n",
    "for e in range(N_EPOCHS):\n",
    "    acc = dfs[e][f'TestAcc_{e}']\n",
    "    loss = dfs[e][f'TestLoss_{e}']\n",
    "    s = '\\t' + r'\\hfill ' + '{} & {:.3f} & {:.3f} & {:.3f} & {:.3f} & {:.3f} \\\\\\\\'.format(e+1, acc.min(), acc.mean(), acc.max(), loss.min(), loss.max())\n",
    "    print(s)\n",
    "print(r'\\cmidrule{1-6}')\n",
    "print(r'\\end{tabular}')\n",
    "print(r'\\caption{Summary of results on test data after each epoch\\label{tab:summary}}')\n",
    "print(r'\\end{table}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}