examples/Benchmarking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score\n",
    "from mlaut.benchmarking.data import RAMDataset\n",
    "from sklearn import datasets\n",
    "from mlaut.highlevel.tasks import TabClassifTask, TabRegrTask\n",
    "\n",
    "from mlaut.benchmarking.results import HDDResults, RAMResults\n",
    "from mlaut.highlevel.strategies import TabClassifStrategy\n",
    "from mlaut.model_selection import SingleSplit\n",
    "from mlaut.benchmarking.orchestration import Orchestrator\n",
    "\n",
    "from sklearn import linear_model\n",
    "from mlaut.strategies.baseline import BaselineClassifierStrategy, BaselineRegressorStrategy\n",
    "from mlaut.strategies.bayes import GaussianNaiveBayesStrategy, BernoulliNaiveBayesStrategy\n",
    "from mlaut.strategies.cluster import KNeighboursStrategy\n",
    "from mlaut.strategies.decision_trees import DecisionTreeClassifierStrategy, DecisionTreeRegressorStrategy\n",
    "from mlaut.strategies.ensemble import (RandomForestClassifierStrategy,\n",
    "                                       RandomForestRegressorStrategy,\n",
    "                                       GradientBoostingClassifierStrategy,\n",
    "                                       GradientBoostingRegressorStrategy, \n",
    "                                       BaggingClassifierStrategy, \n",
    "                                       BaggingRegressorStrategy)\n",
    "from mlaut.strategies.glm import (LinearRegressonStrategy, \n",
    "                                  RidgeRegressionStrategy, \n",
    "                                  LassoStrategy, \n",
    "                                  LassoLarsStrategy, \n",
    "                                  LogisticRegressionStrategy, \n",
    "                                  BayesianRidgeStrategy, \n",
    "                                  PassiveAggressiveClassifierStrategy)\n",
    "from mlaut.strategies.svm import SVMStrategy, SVRStrategy\n",
    "from mlaut.strategies.neural_networks import KerasClassificationStrategy, KerasRegressionStrategy\n",
    "from mlaut.benchmarking.evaluation import Evaluator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Benchmark strategies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:********* Fitting BaselineClassifier\n",
      "INFO:root:********* Fitting GaussianNaiveBayes\n",
      "INFO:root:********* Fitting KNeighbours\n",
      "INFO:root:********* Fitting DecisionTreeClassifier\n",
      "INFO:root:********* Fitting KerasClassifier4Layers\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /media/Data/PhD/mlaut/venv_mlaut/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /media/Data/PhD/mlaut/venv_mlaut/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "112/112 [==============================] - 0s 2ms/sample - loss: 0.2160 - acc: 0.3661\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:********* Fitting BaselineClassifier\n",
      "INFO:root:********* Fitting GaussianNaiveBayes\n",
      "INFO:root:********* Fitting KNeighbours\n",
      "INFO:root:********* Fitting DecisionTreeClassifier\n",
      "INFO:root:********* Fitting KerasClassifier4Layers\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "133/133 [==============================] - 0s 1ms/sample - loss: 0.4597 - acc: 0.2932\n"
     ]
    }
   ],
   "source": [
    "iris = datasets.load_iris()\n",
    "wine = datasets.load_wine()\n",
    "\n",
    "iris_pd = pd.DataFrame(iris.data, columns=iris.feature_names)\n",
    "iris_pd['target'] = iris.target\n",
    "\n",
    "wine_pd = pd.DataFrame(wine.data, columns=wine.feature_names)\n",
    "wine_pd['target'] = wine.target\n",
    "\n",
    "datasets = ([RAMDataset(iris_pd, name='iris'),\n",
    "            RAMDataset(wine_pd, name='wine')])\n",
    "\n",
    "tasks = [TabClassifTask(target=\"target\") for _ in range(len(datasets))]\n",
    "\n",
    "\n",
    "strategies = [\n",
    "    BaselineClassifierStrategy, \n",
    "    GaussianNaiveBayesStrategy, \n",
    "    KNeighboursStrategy,\n",
    "    DecisionTreeClassifierStrategy, \n",
    "    KerasClassificationStrategy\n",
    "]\n",
    "\n",
    "results = HDDResults(path='results')\n",
    "\n",
    "orchestrator = Orchestrator(datasets=datasets,\n",
    "                            tasks=tasks,  \n",
    "                            strategies=strategies, \n",
    "                            cv=SingleSplit(), \n",
    "                            results=results,\n",
    "                            log_file_path=None)\n",
    "\n",
    "orchestrator.fit_predict(save_fitted_strategies=True, overwrite_predictions=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlaut.benchmarking.evaluation import Evaluator\n",
    "from mlaut.benchmarking.metrics import PairwiseMetric\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>strategy</th>\n",
       "      <th>accuracy_mean</th>\n",
       "      <th>accuracy_stderr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BaselineClassifier</td>\n",
       "      <td>0.397368</td>\n",
       "      <td>0.077106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>0.895906</td>\n",
       "      <td>0.045674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GaussianNaiveBayes</td>\n",
       "      <td>0.988889</td>\n",
       "      <td>0.011111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>KNeighbours</td>\n",
       "      <td>0.809064</td>\n",
       "      <td>0.049240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>KerasClassifier4Layers</td>\n",
       "      <td>0.280117</td>\n",
       "      <td>0.070603</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 strategy  accuracy_mean  accuracy_stderr\n",
       "0      BaselineClassifier       0.397368         0.077106\n",
       "1  DecisionTreeClassifier       0.895906         0.045674\n",
       "2      GaussianNaiveBayes       0.988889         0.011111\n",
       "3             KNeighbours       0.809064         0.049240\n",
       "4  KerasClassifier4Layers       0.280117         0.070603"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator = Evaluator(results)\n",
    "\n",
    "metric = PairwiseMetric(func=accuracy_score, name='accuracy')\n",
    "metrics_by_strategy = evaluator.evaluate(metric=metric)\n",
    "metrics_by_strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>strategy</th>\n",
       "      <th>accuracy_mean_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BaselineClassifier</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GaussianNaiveBayes</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>KNeighbours</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>KerasClassifier4Layers</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 strategy  accuracy_mean_rank\n",
       "0      BaselineClassifier                 4.0\n",
       "1  DecisionTreeClassifier                 2.5\n",
       "2      GaussianNaiveBayes                 1.0\n",
       "3             KNeighbours                 2.5\n",
       "4  KerasClassifier4Layers                 5.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.rank()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">BaselineClassifier</th>\n",
       "      <th colspan=\"2\" halign=\"left\">DecisionTreeClassifier</th>\n",
       "      <th colspan=\"2\" halign=\"left\">GaussianNaiveBayes</th>\n",
       "      <th colspan=\"2\" halign=\"left\">KNeighbours</th>\n",
       "      <th colspan=\"2\" halign=\"left\">KerasClassifier4Layers</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>t_stat</th>\n",
       "      <th>p_val</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>p_val</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>p_val</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>p_val</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>p_val</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BaselineClassifier</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-9.674859</td>\n",
       "      <td>0.010515</td>\n",
       "      <td>-51.803722</td>\n",
       "      <td>0.000372</td>\n",
       "      <td>-2.500569</td>\n",
       "      <td>0.129564</td>\n",
       "      <td>3.277978</td>\n",
       "      <td>0.081808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DecisionTreeClassifier</th>\n",
       "      <td>9.674859</td>\n",
       "      <td>0.010515</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-1.766122</td>\n",
       "      <td>0.219415</td>\n",
       "      <td>0.503502</td>\n",
       "      <td>0.664594</td>\n",
       "      <td>9.834252</td>\n",
       "      <td>0.010182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GaussianNaiveBayes</th>\n",
       "      <td>51.803722</td>\n",
       "      <td>0.000372</td>\n",
       "      <td>1.766122</td>\n",
       "      <td>0.219415</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.089883</td>\n",
       "      <td>0.389577</td>\n",
       "      <td>18.969947</td>\n",
       "      <td>0.002767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KNeighbours</th>\n",
       "      <td>2.500569</td>\n",
       "      <td>0.129564</td>\n",
       "      <td>-0.503502</td>\n",
       "      <td>0.664594</td>\n",
       "      <td>-1.089883</td>\n",
       "      <td>0.389577</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.140261</td>\n",
       "      <td>0.088198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KerasClassifier4Layers</th>\n",
       "      <td>-3.277978</td>\n",
       "      <td>0.081808</td>\n",
       "      <td>-9.834252</td>\n",
       "      <td>0.010182</td>\n",
       "      <td>-18.969947</td>\n",
       "      <td>0.002767</td>\n",
       "      <td>-3.140261</td>\n",
       "      <td>0.088198</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       BaselineClassifier           DecisionTreeClassifier  \\\n",
       "                                   t_stat     p_val                 t_stat   \n",
       "BaselineClassifier               0.000000  1.000000              -9.674859   \n",
       "DecisionTreeClassifier           9.674859  0.010515               0.000000   \n",
       "GaussianNaiveBayes              51.803722  0.000372               1.766122   \n",
       "KNeighbours                      2.500569  0.129564              -0.503502   \n",
       "KerasClassifier4Layers          -3.277978  0.081808              -9.834252   \n",
       "\n",
       "                                 GaussianNaiveBayes           KNeighbours  \\\n",
       "                           p_val             t_stat     p_val      t_stat   \n",
       "BaselineClassifier      0.010515         -51.803722  0.000372   -2.500569   \n",
       "DecisionTreeClassifier  1.000000          -1.766122  0.219415    0.503502   \n",
       "GaussianNaiveBayes      0.219415           0.000000  1.000000    1.089883   \n",
       "KNeighbours             0.664594          -1.089883  0.389577    0.000000   \n",
       "KerasClassifier4Layers  0.010182         -18.969947  0.002767   -3.140261   \n",
       "\n",
       "                                 KerasClassifier4Layers            \n",
       "                           p_val                 t_stat     p_val  \n",
       "BaselineClassifier      0.129564               3.277978  0.081808  \n",
       "DecisionTreeClassifier  0.664594               9.834252  0.010182  \n",
       "GaussianNaiveBayes      0.389577              18.969947  0.002767  \n",
       "KNeighbours             1.000000               3.140261  0.088198  \n",
       "KerasClassifier4Layers  0.088198               0.000000  1.000000  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df, df_pivot = evaluator.t_test()\n",
    "df_pivot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BaselineClassifier</th>\n",
       "      <th>DecisionTreeClassifier</th>\n",
       "      <th>GaussianNaiveBayes</th>\n",
       "      <th>KNeighbours</th>\n",
       "      <th>KerasClassifier4Layers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BaselineClassifier</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DecisionTreeClassifier</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GaussianNaiveBayes</th>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KNeighbours</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KerasClassifier4Layers</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        BaselineClassifier  DecisionTreeClassifier  \\\n",
       "BaselineClassifier                   False                   False   \n",
       "DecisionTreeClassifier               False                   False   \n",
       "GaussianNaiveBayes                    True                   False   \n",
       "KNeighbours                          False                   False   \n",
       "KerasClassifier4Layers               False                   False   \n",
       "\n",
       "                        GaussianNaiveBayes  KNeighbours  \\\n",
       "BaselineClassifier                    True        False   \n",
       "DecisionTreeClassifier               False        False   \n",
       "GaussianNaiveBayes                   False        False   \n",
       "KNeighbours                          False        False   \n",
       "KerasClassifier4Layers               False        False   \n",
       "\n",
       "                        KerasClassifier4Layers  \n",
       "BaselineClassifier                       False  \n",
       "DecisionTreeClassifier                   False  \n",
       "GaussianNaiveBayes                       False  \n",
       "KNeighbours                              False  \n",
       "KerasClassifier4Layers                   False  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.t_test_with_bonferroni_correction()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>estimator_2</th>\n",
       "      <th>BaselineClassifier</th>\n",
       "      <th>DecisionTreeClassifier</th>\n",
       "      <th>GaussianNaiveBayes</th>\n",
       "      <th>KNeighbours</th>\n",
       "      <th>KerasClassifier4Layers</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>estimator_1</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BaselineClassifier</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DecisionTreeClassifier</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GaussianNaiveBayes</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KNeighbours</th>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KerasClassifier4Layers</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "estimator_2             BaselineClassifier  DecisionTreeClassifier  \\\n",
       "estimator_1                                                          \n",
       "BaselineClassifier                     0.5                     0.5   \n",
       "DecisionTreeClassifier                 0.5                     0.5   \n",
       "GaussianNaiveBayes                     0.5                     0.5   \n",
       "KNeighbours                            0.5                     1.0   \n",
       "KerasClassifier4Layers                 0.5                     0.5   \n",
       "\n",
       "estimator_2             GaussianNaiveBayes  KNeighbours  \\\n",
       "estimator_1                                               \n",
       "BaselineClassifier                     0.5          0.5   \n",
       "DecisionTreeClassifier                 0.5          1.0   \n",
       "GaussianNaiveBayes                     0.5          0.5   \n",
       "KNeighbours                            0.5          0.5   \n",
       "KerasClassifier4Layers                 0.5          0.5   \n",
       "\n",
       "estimator_2             KerasClassifier4Layers  \n",
       "estimator_1                                     \n",
       "BaselineClassifier                         0.5  \n",
       "DecisionTreeClassifier                     0.5  \n",
       "GaussianNaiveBayes                         0.5  \n",
       "KNeighbours                                0.5  \n",
       "KerasClassifier4Layers                     0.5  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df, df_pivot = evaluator.sign_test()\n",
    "df_pivot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">BaselineClassifier</th>\n",
       "      <th colspan=\"2\" halign=\"left\">DecisionTreeClassifier</th>\n",
       "      <th colspan=\"2\" halign=\"left\">GaussianNaiveBayes</th>\n",
       "      <th colspan=\"2\" halign=\"left\">KNeighbours</th>\n",
       "      <th colspan=\"2\" halign=\"left\">KerasClassifier4Layers</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>t_stat</th>\n",
       "      <th>p_val</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>p_val</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>p_val</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>p_val</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>p_val</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BaselineClassifier</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>-1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>-1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DecisionTreeClassifier</th>\n",
       "      <td>1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GaussianNaiveBayes</th>\n",
       "      <td>1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KNeighbours</th>\n",
       "      <td>1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KerasClassifier4Layers</th>\n",
       "      <td>-1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>-1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>-1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>-1.549193</td>\n",
       "      <td>0.121335</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       BaselineClassifier           DecisionTreeClassifier  \\\n",
       "                                   t_stat     p_val                 t_stat   \n",
       "BaselineClassifier               0.000000  1.000000              -1.549193   \n",
       "DecisionTreeClassifier           1.549193  0.121335               0.000000   \n",
       "GaussianNaiveBayes               1.549193  0.121335               1.549193   \n",
       "KNeighbours                      1.549193  0.121335               0.000000   \n",
       "KerasClassifier4Layers          -1.549193  0.121335              -1.549193   \n",
       "\n",
       "                                 GaussianNaiveBayes           KNeighbours  \\\n",
       "                           p_val             t_stat     p_val      t_stat   \n",
       "BaselineClassifier      0.121335          -1.549193  0.121335   -1.549193   \n",
       "DecisionTreeClassifier  1.000000          -1.549193  0.121335    0.000000   \n",
       "GaussianNaiveBayes      0.121335           0.000000  1.000000    1.549193   \n",
       "KNeighbours             1.000000          -1.549193  0.121335    0.000000   \n",
       "KerasClassifier4Layers  0.121335          -1.549193  0.121335   -1.549193   \n",
       "\n",
       "                                 KerasClassifier4Layers            \n",
       "                           p_val                 t_stat     p_val  \n",
       "BaselineClassifier      0.121335               1.549193  0.121335  \n",
       "DecisionTreeClassifier  1.000000               1.549193  0.121335  \n",
       "GaussianNaiveBayes      0.121335               1.549193  0.121335  \n",
       "KNeighbours             1.000000               1.549193  0.121335  \n",
       "KerasClassifier4Layers  0.121335               0.000000  1.000000  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df, df_pivot = evaluator.ranksum_test()\n",
    "df_pivot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>statistic</th>\n",
       "      <th>p_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.6</td>\n",
       "      <td>0.10738</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   statistic  p_value\n",
       "0        7.6  0.10738"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df, df_pivot = evaluator.friedman_test()\n",
    "df_pivot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BaselineClassifier</th>\n",
       "      <th>DecisionTreeClassifier</th>\n",
       "      <th>GaussianNaiveBayes</th>\n",
       "      <th>KNeighbours</th>\n",
       "      <th>KerasClassifier4Layers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BaselineClassifier</th>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.912540</td>\n",
       "      <td>0.415938</td>\n",
       "      <td>0.912540</td>\n",
       "      <td>0.979393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DecisionTreeClassifier</th>\n",
       "      <td>0.912540</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.912540</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.604451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GaussianNaiveBayes</th>\n",
       "      <td>0.415938</td>\n",
       "      <td>0.912540</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.912540</td>\n",
       "      <td>0.136852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KNeighbours</th>\n",
       "      <td>0.912540</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.912540</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.604451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KerasClassifier4Layers</th>\n",
       "      <td>0.979393</td>\n",
       "      <td>0.604451</td>\n",
       "      <td>0.136852</td>\n",
       "      <td>0.604451</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        BaselineClassifier  DecisionTreeClassifier  \\\n",
       "BaselineClassifier               -1.000000                0.912540   \n",
       "DecisionTreeClassifier            0.912540               -1.000000   \n",
       "GaussianNaiveBayes                0.415938                0.912540   \n",
       "KNeighbours                       0.912540                1.000000   \n",
       "KerasClassifier4Layers            0.979393                0.604451   \n",
       "\n",
       "                        GaussianNaiveBayes  KNeighbours  \\\n",
       "BaselineClassifier                0.415938     0.912540   \n",
       "DecisionTreeClassifier            0.912540     1.000000   \n",
       "GaussianNaiveBayes               -1.000000     0.912540   \n",
       "KNeighbours                       0.912540    -1.000000   \n",
       "KerasClassifier4Layers            0.136852     0.604451   \n",
       "\n",
       "                        KerasClassifier4Layers  \n",
       "BaselineClassifier                    0.979393  \n",
       "DecisionTreeClassifier                0.604451  \n",
       "GaussianNaiveBayes                    0.136852  \n",
       "KNeighbours                           0.604451  \n",
       "KerasClassifier4Layers               -1.000000  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.nemenyi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfgAAADnCAYAAADy30SFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deVxUVf8H8M+AW4rJIqiPkSkplZQa7qAIKIuIuwkCYvi4p+bKIghomqkkKam/eEotbFFxSVR8cMHcwP1Rs7TMFFwAWVQEmYE5vz94cWNkR3Tg+nm/Xr5ezpmZe7/DzJzvPXfOPV+FEEKAiIiIZEVH2wEQERFRzWOCJyIikiEmeCIiIhligiciIpIhJngiIiIZYoInIiKSISZ4IiIiGWKCJyIikiEmeCIiIhligiciIpIhJngiIiIZYoInIiKSISZ4IiIiGaqn7QCI6PkQQmDTpk344YcfkJ+fDx0dHVhZWWHOnDmIi4vDokWL0LJlS+Tn50MIATs7O8yYMQNNmzbVduhEVAM4gieSqVWrViEmJgYbNmzAwYMHsXPnTjx58gSTJk2CEAKdO3dGbGwsDhw4gF27diEvLw+enp7Iy8vTduhEVAMUrAdPJD+ZmZno168ftm/fDjMzM6n9yZMnOHHiBDIyMhATE4ONGzdK9wkhMGLECHzwwQdwc3PTQtREVJM4gieSoYsXL6Jly5YayR0AGjVqBDs7O+jolPzqKxQK2Nra4vTp0y8qTCJ6jpjgiWTo4cOHMDQ0rPLzjI2N8ejRo+cQERG9aEzwRDJkYGCAlJSUKj8vLS0NRkZGzyEiInrRmOCJZKhz585IS0vDpUuXNNpVKhVWrVqF3NzcEs9Rq9X473//iz59+ryoMInoOWKCJ5IhPT09TJo0CYGBgUhOTgYAPH78GAsXLsTly5fxyiuvaDz+8ePHWLRoEZo0aQJHR0dthExENYyz6Ilk7Pvvv5eug1er1bC3t8fMmTOxZ88e6Tp4IQTy8/NhY2OD2bNnQ09PT9thE1ENYIInIiKSIZ6iJyIikiEmeCIiIhligiciIpIhJniqcywsLLQdAhE/h1TrcZId1TkKhQL82JK28XNItR1H8ERERDLEevBUJykUCm2HQERUqzHBU53EU6OkbTzIpNqOp+iJiIhkiAmeiIhIhjiLnoiISIY4giciIpIhJngiIiIZYoInIiKSISZ4IiIiGWKCJyIikiEmeCIiIhligiciIpIhJngiIiIZYoInIiKSISZ4IiIiGWKCpzohOTkZHTt2hJOTk/Rv/vz52g6rTPHx8TA3N0dycrK2QynTvn374OrqCkdHR7i5ueHq1avaDqmEQ4cOYfDgwXB0dMTo0aPx+++/azukMqlUKixbtgzm5ua4d++etsMhYrlYqjtatGiB2NhYbYdRodzcXISFhUFfX1/boZQpKSkJoaGhiI6ORuvWrbFp0yYEBAQgOjpa26FJUlNT4evrix9++AFvvvkmvv32WwQFBWHr1q3aDq1UU6dOxbvvvqvtMIgkHMET1bA1a9ZgyJAhaNKkibZDKVPDhg0RFhaG1q1bAwCsrKzw999/azeop+jq6iIsLAxvvvkmAKBHjx61Lsbipk6dihkzZmg7DCIJEzzVGdnZ2Zg8eTIcHR0xfvx4XL9+XdshlXD16lUkJCRg3Lhx2g6lXCYmJrCysgIA5OfnY+vWrejXr592g3qKkZER+vbtK90+cuQIOnXqpMWIytelSxdth0CkgQme6oQmTZpg4MCB8Pf3R2xsLHr37o3JkycjPz9f26FJhBAICQlBUFAQ6tWrG79+bdq0CT169MCZM2dq9ZyGkydPYuPGjfD399d2KER1BhM81QkGBgYICQlBmzZtoFAo4OPjg4yMDNy4cUPboUl++uknmJub16mRnLe3N86cOQNvb2+4u7vjyZMn2g6phAMHDmD+/PlYt24dzMzMtB0OUZ3BBE91QlZWFpKSkjTa1Go1GjRooKWISjp48CD2798PKysrWFlZ4e7duxg5ciQSEhK0HVoJ169fx8mTJwEACoUCgwcPRk5OTq06YAKAEydO4JNPPsE333xTq0/PE9VGTPBUJ1y+fBnjxo1DRkYGgMLRcsuWLfHaa69pObJ/REZG4uTJkzh+/DiOHz+OVq1aYdu2bejZs6e2QyshMzMT8+fPR2pqKgDgzJkzUCqVMDU11XJk/8jNzYWfnx9Wr16N9u3bazscojqnbvxQSC89a2truLm5wd3dHUDhJXMRERHQ1dXVcmR1U9euXTF58mR4e3tDCIFGjRohPDwcenp62g5NcvDgQWRkZJSYGxAVFYXmzZtrKarS3b9/H56entJtLy8v6OrqYtOmTWjRooUWI6OXmUIIIbQdBBEREdUsnqInIiKSISZ4IiIiGWKCJyIikiEmeCIiIhligiciIpIhJniqUxQKBRQKhbbDqFBdiLMuxAgwTqLqYoInIiKSISZ4IiIiGWKCJyIikiEmeCIiIhligicC6sTkqLoQI8A4iWoLJngiIiIZYrEZqlM46qLajl0q1RYcwRMREckQ68FTnWJjYwMAiI+P124gL5i+vj4AICsrS8uRvHj9+vUDUPvf86I4iWoLjuCJiIhkiAmeiIhIhpjgiYiIZIi/wRPVQmq1GiEhIYiKisKNGzek9pf5KoK68tqL4mzbti08PT0REhICHR2OpejFY4InqoVCQkKwePFibYdBz+DGjRvSe7ho0SItR0MvIx5WEtVCUVFR2g6BagjfS9IWJngiIiIZYoInqoU8PT21HQLVEL6XpC38DZ6oFgoJCQGAEpPsqO4oPsmOSBu4Fj3VKXVlVbOaxpXsav97XlfipJcHT9ETERHJEE/RU61V1rXgQN25JrqmvayvG+D15URVxQRPtRavBafS8Ppyosrh4S/VWrx+mMrDzwdR+ZjgiYiIZIgJnmotXj9M5eHng6h8/A2eai1eC06l4fXlRJXD6+CpTnlZrzXmdfC1/z2vK3HSy4On6ImIiGSICZ6IiEiGmOCJiIhkiAmeiIhIhpjgiYiIZIgJnoiISIaY4ImIiGSICZ6IiEiGmOCJiIhkiAmeqiQxMRE9e/aEl5cXvLy8SpRzPX/+PNzc3DBs2DCsXbtWag8PD4ebmxuGDx+OS5cuveiwieqEa9euoX///qVWyuN3i6qKa9FTlXXv3h2rV68u9T4/Pz9s3LgRLVq0wOjRozFo0CDcuXMHly9fxo8//ohr164hNDQUmzdvfsFRE9VuOTk5WLx4MXr16lXq/fxuUVVxBE81JikpCc2aNUOrVq2go6ODfv364dixY0hMTIS9vT0AoEOHDkhNTUVubq6WoyWqXRo0aIDIyEiYmJiUuI/fLaoOJniqsj///BP//ve/4e7ujuPHj0vtqampMDQ0lG4bGRnh/v37SEtL02g3NDTE/fv3X2jMRLVdvXr10KhRo1Lv43eLqoOn6KlK3njjDUyZMgUuLi64ffs2xo4di/3796NBgwaoX7++xmOFEFAoFGW2E1Hl8LtF1VHhCN7c3Bz37t2TbsfExMDBweGFHSX+9ddfmDp1Kvr06QNHR0eMHj0ahw8fBgAkJyfjnXfeqfF9zp8/H4cOHQIArFq1Cj179sT27ds12qsjKioK5ubm0u01a9ZgwYIFzxzvi9SiRQu4urpCR0cHpqamaN68OVJSUgAAJiYmSE9Plx57//59mJiYwNjYWKM9IyMDzZs3f+GxE9VVz+u7JYTAd999BxcXF/Tv3x92dnaYMGECrly5UqPxlycuLg7+/v7PtA1zc3MEBARotCUmJsLLy6vC5z5Lv56YmAgLCws4OTnByckJDg4OCAgIQF5eXrW2V9OqdIo+ISEBK1asQGRkZJU+RAUFBVUODABSUlLg5eWFPn36ID4+Hvv378e8efPg5+eHY8eOVWublbF8+XLY2dkBAPbs2YNVq1Zh+PDhGu2VUfx1p6amYsuWLTUea1VV970osmfPHqxZswZAYWeSnp6OFi1aAABatmyJ/Px83LlzBwUFBTh8+DD69u2Lvn374uDBgwCAX3/9FaampmWeiiSikp7Xd+vzzz/H7t27ERkZiQMHDiAuLg79+vWDt7c3MjIyntfL0TBgwAB8+umnz7yd06dPV+vApKr9+tNatWqF2NhYxMbGYvfu3UhKSir1KghtqHSCv3btGnx9fREREYE2bdpI7QcPHoSrqyucnJwwceJEpKWlASgcnfr7+2P48OHYtGkThBBYunQpHBwcYGdnh3nz5kGlUgEATp06hWHDhsHZ2RnOzs7Yt28fAGDjxo3o2bMn3N3doaurCwDo2rUr1q5di3bt2mnEV53tl9Xu5eWFXbt2Yc6cObh79y4CAgKwZcsWqR0Azp07hxEjRsDJyQmenp64ceMGAGD79u2YOnUqvLy8sHz5cim+JUuWYMqUKZX9c+PIkSNwdXWFo6Mjhg8fjt9++w0FBQWwsrLSuBTmu+++w9SpUwEAW7ZswcCBA+Hk5IQ5c+YgOzsbQOHs208++QQuLi6IjY3FtWvXMHr0aAwcOBAODg5V+jD269cPv//+O9zc3DBlyhQEBwcjJiYGcXFxAICAgABMnToVI0eOhKurK1q1agULCwu89dZbGDZsGIKDg+Hn51fp/RG9LC5fvgwvLy/s2LED3377Lby8vLBhw4bn9t3KyMjAt99+i+XLl+Nf//oXAEBXVxceHh44fPgwDA0NcfPmTXh4eEij05iYGAAlz54Wv/348WNMmzYNzs7OsLe3R2BgIFQqVZnt27dvx7hx4wAAmZmZmDhxIhwdHWFvb48NGzZI+7CxscGWLVswatQo9O3bF0uXLtV4PbNnzy7RVqS8/FDUr8+cORPffPON9JzffvsN1tbWUKvVZfb3T2vYsCG6dOmC5OTkcvcbHx+PQYMGaTx3+PDhOHDgAB49eoS5c+fC0dERgwYN0hgYrlq1Co6OjnB0dMTYsWOls6dlEhXo0KGDuHDhgrC1tRVHjx7VuO/evXuiW7du4vr160IIIdavXy+mT58uhBBi9erVwtraWqSnpwshhIiLixMDBw4UeXl5Ii8vTzg7O4udO3cKIYQYPny4SExMFEIIcfPmTTF79mwhhBAjRowQP//8c5mxJSUlibfffrva2y+r3dPTU3qura2tOH36tEZ7Tk6O6N27t/Tc3bt3i+HDhwshhIiOjhadO3cWf/31lxRnfHy8+Pe//y39PYusXr1aBAQElHhd+fn5onv37uLMmTNCCCHWrFkjvL29hRBCLF68WCxdulR6rJeXl4iJiREXL14U1tbWIjU1VQghRFBQkFi2bJkQQghfX1/h6uoqcnNzhRBCTJ8+XWzfvl0IIUR6erqYOnWqyMvLK/PvXJvY2NgIGxsbbYfxwjVr1kw0a9ZM22FoRV15z+tKnEUOHz4sHBwcyn3MlClTxNq1a4UQQpw6dUq89957QqlUavS9Qmj2xVFRUcLPz08IIYRKpRLBwcHiypUrZbZHR0dL/dsnn3wiFixYIIQQ4tatW6Jjx47izp07QojCvnjOnDmioKBA3Lt3T+O+on51zJgxYt++fUIIIRISEoSnp6cQovz8UNSv79mzR3h4eEiv6YsvvhCLFy8ut79PSEgQ/fv3l56TkZEhhg4dKuLj48vdr1KpFN27dxe//fabEEKI5ORkYWlpKfLy8sSiRYuEv7+/UKvVIj09XfTt21dcvXpVXLt2TTg4OAilUimEEOK7774TO3bsKPf9q9Qku7lz50KpVCIzM1Oj/ejRo+jUqZM0mnZzc4OVlRWEEACATp06STM8+/fvjz59+qBBgwYAgHfffRdJSUkAAGNjY+zcuRNGRkYwMzNDWFgYAODRo0cwMjKqTIjV2n5Z7RW5cOECmjVrhu7duwMABg0ahJCQEGmuwhtvvIG2bdsCAJ48eYIVK1Zg3bp1ldo2UHgUffToUem1dO3aFTt27AAAuLi4YMaMGfD19cWDBw9w+fJlrF+/HpGRkbC3t4exsTEAwN3dXXocAPTq1Us6dWdiYoL9+/ejffv26NixI7788ssKY/r22281jm615cKFC+jcubO2wyAq1YULF9CvXz9thwEfHx+MHTu23Mc8evRIYwb+w4cP8cEHHwAovCbfy8sLa9askfrz999/H3l5edJZ2rIYGxvj/PnzOHbsGLp3746QkBAAhZf6ldb+22+/Sc8NCAiQRtampqYwNjZGcnIyWrVqBaCwr9XR0UGLFi3QvHlz3Lt3T7qv6PkzZ86Era2tRkzl5Yci/fr1Q2BgILKysqCvr4+4uDgEBwdX2N/fvXsXTk5OEELgzp07sLGxgaWlZbn7rV+/PhwdHbFnzx689dZbiIuLg729PRo0aICDBw9i9erVUCgUMDQ0hIODAw4cOIBRo0YhMzMTu3fvhr29PTw9Pct9H4BKzqIPDAyEkZERfHx80KFDB2miWEZGBs6dOwcnJyfpsXp6etKBQLNmzaT2lJQULF26FFevXgVQOEnE29sbALBs2TJERETgww8/RKNGjTB37lw4ODjAwMCg4lMQz7D9storkpGRgdu3b2u87gYNGkiTXYq/7i+//BJDhgyBqalppV5HkY0bN2L37t1QKpVQKpXSzNguXbqgfv36OHXqFG7fvo0+ffqgcePGSE9Px/79+5GQkACg8NSQUqmUtlc8pnnz5mHt2rWYPXs2cnNzMXXqVLi7u1cpPm3p3LkzxowZo+0wiEqoa59LQ0NDpKamSrdfffVVxMbGAgAWLFiAJ0+e4MiRI/i///s/ZGVlQaFQQAgBtVpd7nYdHBzw8OFDrF69GtevX8fQoUPh6+tbZntx58+fx6pVq5CamgqFQoG0tDSN/enp6Un/19HRKTGnqGPHjujWrRs2bNiALl26SO3l5YcijRs3Ru/evREfHw9LS0s8fPgQlpaW2Lt3b7n9fdFv8ACgVqsRFRUFb29vREdHl7tfFxcX+Pv7Y/bs2Thw4AAmTJgAAEhPT8fs2bNRr15hes7Ly4OzszOMjY2xdu1aREZGYvHixejevTtCQ0PRsmXLMt+LSiV4c3NztGzZEjNnzsT06dMRHR2Npk2bonnz5ujVqxciIiIq3EZ4eDh0dXXx888/o0GDBpg7d650n76+PgIDAxEYGIiTJ09i2rRpsLa2Rrdu3RAbG4thw4ZpbOvgwYNo2LAh3njjjWfaflntFWnevDnatWsnjaqLK3ojixw6dAiZmZnYuHGj1GZlZYXvv/++zO2fPn0a33zzDbZu3QpTU1OcOHECgYGB0v0uLi7Yu3cvUlNTpb+NsbExhg4dWqnZqA0bNsSsWbMwa9YsXLlyBT4+Pujdu7fG3IqnjR07tsIRAdHLbOLEiZg4caK2w6i0Ll26ICMjA1euXCn1aiSlUomPP/4Y4eHhsLOzg0qlwrvvvgug8Cxj0cgeKPzdvbiRI0di5MiRSEtLw4wZM7Br1y6MGjWq1Pai+VVA4eDjww8/lEanffv2rfLrmjVrFoYPH47XXntNaisvPxTn6OiIuLg4ZGRkwNHREQqFotz+PjExUeO2jo4O3NzcsGTJEmRkZJS7327duiE/Px+HDx/Gn3/+id69ewMo7MsjIiLw1ltvldhf165d0bVrV+Tm5iIsLAxhYWFYsWJFmX+LKs2i9/DwgIWFBebPnw8hBKytrXHmzBncvHkTAHDx4sUyJzk8fPgQb7/9Nho0aICLFy/i3LlzyMnJgUqlgqenp3QkaW5ujnr16kGhUGDcuHG4fPkyvv32W+ko7vTp01i4cCEaNmz4TNsvKCgoc78V6dSpE+7fv49z584BKDz15Ovrq/GBL7Jnzx6cOHECx48flxaFOX78eLnJ9MGDB2jevDlat26Nx48fY8uWLcjJyZG2P2jQIBw8eBAXLlyAjY0NAMDOzg5xcXHSUeWBAwcQGRlZ6vYnTZqEP/74AwDQrl07jaNiIno5NG7cGB999BF8fX1x/fp1AIUj0L1792Lfvn1o37498vLy0KlTJ6jVavznP/9BgwYNkJOTAyMjI+jq6uLvv/8GAOzfv1/abkREBLZt2wagcDBUNIGvrPbiHj58iPfeew8ApH4vJyenSq/LxMQEHh4e0tU+RdstLT88zdbWFufOncOhQ4fg7OwMoGr9PVDY97Zs2RIGBgbl7ldHRwcDBw7E4sWLYW9vL61pYGdnhx9++AFCCOTn52Pp0qW4cuUKfvnlF4SGhkKtVuOVV17Bm2++WWYMRaq80M2iRYswcuRIrF+/HlOmTMGSJUswffp0KJVKNG7cWGOkWdyHH34IX19fbNu2Dd27d4e/vz8CAwPRpUsXjBw5EuPGjYMQAvXq1UNQUBBeeeUVvPLKK/jxxx+xcuVKREVFQaFQwMTEBOHh4ejWrZs0U7E622/atGmZ+61Io0aNsHr1aixZsgTZ2dmoX78+Pv7442otMLF//36cPXtWuv32229j2bJl+P7779G/f38YGxsjKCgIV65cwZw5c/D555/D3NwcBgYGMDc3l35X79ixI6ZMmYKxY8ciPz8fhoaGWLJkSan79PT0xJw5c6BSqaBQKODp6VnuAQcRydP48eOhr68vXXWjVqvRtm1brFmzBlZWVrh27RqGDh0KfX19TJ8+HQMGDMD06dOxY8cOzJgxA5MmTcJrr72GAQMGSIOwIUOGICAgAJGRkVAoFOjSpQuGDBmClJSUUtuLZuYDwIwZMzBt2jQ0bdoUHh4ecHd3R2hoKN58880qvS4fHx9s3bpVul1efihOT08PFhYWuHr1qjTXp6L+vug3+CJFp9IVCkW5+3VwcICLiws2bNiAgQMHSs//+OOPERoaCicnJ6jVatjY2KBDhw4wMzPD3r174ejoiPr168PY2LjMPr6IQlR0CEC10sSJE+Hh4SGN4Ene9PX1AQBZWVlajuTFK5q0Fh8fr9U4iGra/fv3MWzYMMTHx2v8VFFTuBZ9HXT+/HkkJSVVar4AERHVThEREXBzc3suyR3gWvR1TkBAABITExEWFvbcPhTlyc3NhZ+fH9LT05GTk4Np06ZJ1ayAwt+PWrZsKcW2cuVKtGjRAuHh4UhISIBSqURoaKg0WYeICn3++edITEyESqXChAkTpN+AgcKD+s8++wx5eXkYMGCAtLgVv1d10/379zF69GiYmZk98zK95Sr3Knmip8TExIivvvpKCFG4OMPTC2XY2tqK7OxsjbaTJ0+K8ePHCyGEuHr1qhgzZsyLCVZGuNCNjbbDeK5OnTolLYaVmZkp+vTpo3G/g4ODuHPnjigoKBAjR44UN2/e5PeKKsQRPFWJi4uL9P979+5J69CXp6ya1ZWZ0Ej0MujSpQvCw8MBAE2bNoVKpYJarYaOjo5GLXgAUi34tLQ0fq+oXEzwVC2jRo3C/fv38dVXX5W4LygoCHfv3sX777+PuXPnIi0tTeOazqKa1VVd/IdIrurVqyctbLJt2zbY2NhAR6dwilRpteBTU1P5vaIKMcFTtWzduhW//vorZs+ejV27dkmd0YwZM9C7d28YGRlh+vTp2LdvH2tWE1XSgQMHsGXLFo0iK6wFT9VV5Vn05ubmGDBgABwdHWFtbY0JEybgwoULzxSEk5NTufXln7Ve8OTJk6V6vUXxOzk5YeTIkdXeZmlSU1Mxf/589O3bF46Ojhg6dCi2b98u3W9ubi6tX1xTwsLC8MMPPwAAvv/+e/To0QPr1q3TaK9Jly5dwp07dwAUXnuvVqs1ahQMHToUJiYm0NXVhbW1Nf7880/WgyeqhKNHj2Lt2rX4z3/+g1dffVVqf1614Iv3hY6Ojhg/fjxu3bpVMy/mKXZ2djhz5gwuXryI8ePHP9O2hBDYuHGjVHFuwIABCAkJwaNHjwAUVs9cu3ZtTYQtKR53cnIy+vfvD1dX1xp5Pc9VVX+079Chg7h7964QQoiCggKxd+9e0aNHD3Hq1KkanRzwvBSP/2lqtVqo1epqbffx48fCwcFBhIeHS9V+rl69KmxtbcVPP/1U4b5rwtixY8WWLVuq9dz8/PxKPW7Tpk1SNbu0tDRhY2MjCgoKhBBCPHr0SHh4eIicnBwhhBCzZ88We/fuFZcuXRI+Pj5CCCEuX74sxo0bV60YX2acZGej7TCeq4cPH4qBAweKtLS0Uu8fOnSouH37tsjPzxfDhg0Td+7ceebv1dP90cqVK8XEiROr/yLKUbwq57MKCwsTI0aMELdv3xZCCJGdnS18fX2Fu7u7UKvVwtfXV3z55Zc1sq/S7NixQ7i7u1fruc+SY6rjmU7R6+jowNnZGWlpaVi5ciV++uknKJVKLF++HMeOHYOOjg5cXFwwdepUKBQKXL16FcHBwcjKyoKhoSE+++wzmJqawtzcHEeOHEHTpk0xf/58/PXXX1AqlejVqxeCg4Oxe/du/Pzzz9i4cSOysrIQHByM33//HQqFAiNGjMCECRNQUFCAd955BytWrMCGDRuQlpYGHx8f+Pj4VPg67OzsMGTIEOzZswfffPMN6tevj4ULF+Lvv/9Gw4YN8fHHH8POzg5A4Tr44eHhUKlUeP3117FkyRIYGxtj+/btMDAwwMyZM6XtdujQAV9++aVUSai4r776CtHR0SgoKICZmRlWrFiBV199FdeuXUNQUBAePXqE/Px8jB07Fp6enmW2+/n54fXXX0d2djYuXLiA69ev4969e7h9+zZef/11TJ06FdevX0dwcDDu37+PJk2aICgoCJ07d0ZiYiJWrlyJVq1aQUdHR5rkUx43Nzf4+/tjzJgxUCqVCA4Oxs6dO9G0aVMMGDAAzs7OGDNmDBo1aoR33nkHTk5OUCgUUs1qXV3dCldfInrZ7N27Fw8ePMCsWbOkth49ekij7KJa8AqFAoMHD0arVq3QqlWrGv1e9ejRA4cOHZJub9++HZGRkSgoKICxsTGWL1+O1q1bIyUlBfPnz0daWhqUSiVcXFwwa9YsCCGwdu1axMTESEuZ+/r6avyUkJiYiMDAQMTFxSE8PByPHj1Camoqfv/9d7z66qtYu3YtWrRogZSUlFL74MzMTGzatAnbt2+Xlrpt0qQJQkJCcOLEiRJLt168eBGhoaF4/PgxdHV1sWDBAvTu3Rv5+fkIDg7GmTNnUFBQgLfeegvLli1Do8c8GT4AABApSURBVEaNSm3/9ddfERgYiOXLl2PlypXIzs7G4MGDsWDBAun1lJf7ns4xxdfJf66qekRQ2ig0KSlJvPXWWyI3N1d8/fXXwsfHRyiVSvH48WMxdOhQcfjwYaFWq8XAgQOlOrkbNmyQLvEo2mZl6gUHBQWJoKAgIYQQWVlZGkeGHTp0ECtXrhRCCPG///1PWFhYCJVKVWH8tra2IjAwULo9adIksWbNGiGEEH///bfo3r27SE9PF/fu3RPdunUT169fF0IIsX79ejF9+nQhhBAzZswQ69atq9Tf7sqVK6J79+7i4cOHoqCgQHh7e0tHnGXVai+rvfjRavE69kXtarVauLq6SnWDz5w5I/r06SOUSqVISEgQ7777rjhx4kS5cZP2cQRvo+0wZKd4X6hUKoW/v79YsWKFEKKwj+nYsaM0Svbz8xMBAQFCCCGWLVsm9Y85OTli9uzZIiUlRcTGxgpXV1fx6NEjoVKpxMSJE0VUVJQQ4p8RfPH66atXrxa9e/eW9jFhwgQREREhhCi7D46Pj6+whn3xPnHw4MFi165dQojCkXfRvg8fPizGjh0rjajDw8PFL7/8UmZ78biL56Pi7WXlvqLXXzzHvCg1MsnOxMQEarUaOTk5OHjwIDw8PFC/fn3Ur18fQ4YMQVxcHNq1a4eUlBRpaVVPT88SJUrLqiNcvF7wkSNHpOp1zZo1g62tLY4dO4auXbsCAAYPHgwAsLCwgFKpREZGBkxMTCp8DUXLYapUKvzyyy/S0XCbNm1gaWmJo0ePQqVSoVOnTmjXrh2AwtGslZUVhBBVql3/9ttv45dffpEK5rz//vtSbeKyarVXp4Y7UPh7UVJSEoYMGQIAsLS0hIGBAf73v/8BKFxnuVevXhVup7bUg39ZZWdnv9RFgWpLnfW6ojL14AHAy8sLurq6SE9Pl9ZQBwpn5J89e1bqo7p27Ypdu3YB+Kcv6tmzJ7p06YKwsDAAhWc3XV1dpc/pqFGjEBUVBQ8PjzL3b2lpKY3EO3bsiJSUlHL7YB0dHY0rCiqybds2aQKwpaWl1M82b94c169fR1xcHKytraUzr5cvXy61/emqcaUpK/cVfW618fmtkQSfmpqK+vXro1mzZkhPT8fy5cuxevVqAIUlBzt16oT09HSNiSPFLwspUpl6wenp6dK63EBhki8+Qa9p06YAIL2pFdUuLr4doHCt74KCAo0PZU5ODnr16oXc3FycO3dOo7CAnp4eMjMzq1S7Pjs7G0uXLsWFCxegVquRlZUlvfll1Wqvbg339PR0KJVKjVWxsrOzkZWVhaZNm2rUiafaS09Pr1IHqnJU1+qs1yXfffedVE/87Nmz8PT0RHR0NIyMjLBq1SocO3YM+fn5yM7ORtu2bQEA3t7eUKvVCA0NRWpqKry8vPDRRx8hPT0dJ06cQHR0NACgoKCgwkFPUX8N/FPfvbw+uG3btpXuZwFg165d2Lx5M3JycqBWq6VT+BYWFggNDcWmTZvg6+sLe3t7LFy4sMz2yigr9xXRRl9bIwm+6GhOV1cXxsbGGDdunMbypQBw69YtZGVlSZdyqFQqpKSklPgtoqJ6wUZGRsjMzJSu9czMzKzRGdkGBgbQ1dXFtm3bSoyYtm/fjl69eklnEIrr1q0bNm/ejGnTpmlcqnLu3DkkJydLZxYAYOPGjUhKSsLWrVvRpEkTrFq1SvrQllervbT2ijRv3hx6enqIjY0tcV9ljkqLsB68dr3Mo9e6Vme9rrK0tETr1q1x/vx5PHnyBEeOHMHmzZthaGiIrVu34ueffwZQmIjHjx8vzbqfMGECLC0tYWxsDBsbm2fuJ8rrg7Ozs5GWloZLly5pLMurUqkQERGByZMnS23JyckICQnBli1b8M477+DmzZtwcHCQ7re3t4e9vT0ePnyIwMBAfPPNN/j4449Lba/MWc6ycp82PXOxmaK640WTQ+zt7bF161aoVCpp0sXRo0dhamqKVq1aSYlm27ZtCAoK0thWZeoF29raSpeeZWRk4NChQzXa+dWrVw82Njb4/vvvARSuve7v74979+7B2toaZ86cwc2bNwEUTuBYunQpAGDYsGEoKCjAihUroFQqAQDXrl3DvHnzSlyb+vDhQ5iZmaFJkya4efMmDh8+LNUILqtWe3VruLdu3RotW7bEnj17ABT+zebMmYPc3Nxq/42ISH5u3ryJGzduoH379njw4AFee+01GBoaIj09HT///DMeP34MoHAhq+PHjwMo7F+MjIwghIC9vT127dolPe7HH3/Ezp07qxxHeX2wnp4eJk2ahMDAQKlc+OPHj7Fw4UJcvnxZYxW/7OxsNG7cGGZmZlCpVIiKipIev23bNulnzldffRVt2rSBEKLM9sooK/dpU7VG8EW/26hUKpiamuKrr75Cx44dARSeTktOToarqysKCgrw3nvvwdvbGwqFAp9//jkWLlyI8PBwGBkZ4dNPP9XYbll1hIvXC541axaCgoLg6OgIhUKBiRMn4r333nuGP0FJISEhWLhwoVSPd8iQIdJprCVLlmD69OlQKpVo3LgxAgMDARSOvKOiorBy5UoMGjQIOjo6aNq0Kfz8/DBgwACN7Y8ePRozZsyAg4MDzM3NsWDBAsyYMQObN28us1Z7dWu4F/3dQ0JCsGbNGgCFv89xOUsiKurLgcI+LCQkBGZmZtDX10dMTAz69+8PU1NTzJs3D1OmTEFYWBhGjx6NkJAQLF68GEBhYisa4f7xxx8YMWIE1Go12rRpIw2Aqqq8Pvijjz6CoaEhpkyZgvz8fKjVatjb20tztoqYm5tLa5IYGBhgwYIFuHTpEsaPH4/169fDz88PDg4O0NXVRdu2bfHpp59CCFFq+++//15hzGXlPm1iPXiiOoA10YmoqrhULRGRlrEMMz0PTPBERFp26NAhWFhYYMKECbh9+zZ8fHxKTNaKjIxEkyZNpNsJCQm4fPkyfvzxR1y7dg2hoaHYvHnziw6dajEmeCIiLWMZZnoemOCJiGoJlmGmmsQET0RUS7AMM9WkZ74OnoiIno02yjA/Xb46JiYGDg4O0sqg5ubmCAgI0HhOYmIivLy8Kty2t7c3fv3113Ifs2bNGixYsKDU+4rKy9KzYYInItKy8+fPY9OmTQAK670/fvwYBgYGAAoXbPH09JQWpzp79izat2+Pvn374uDBgwCAX3/9FaampmjUqFG19p+QkIAVK1YgMjJS4yDh9OnTuHLlSpW3t2nTJmltlNqmssuXywETPBGRlrm5ueH+/fsYM2YMJk+eLJVhjouLg56enlSG2d3dHfr6+nBycoKFhYVULjY4OBh+fn7V2ve1a9fg6+uLiIiIEotnzZ49u8zFaoQQ+PLLL+Hs7AwnJyd88sknUKlUADRH4OvWrUO/fv0watQobN68WSo4BhSu1z5nzhzY2tpixIgRGuvMJyQkYOjQobCxsZHWdweAffv2wcXFBY6Ojhg7dixu3boFAPDz85OK5Tx928vLCytXroSjoyPOnTuHU6dOYdiwYXB2doazszP27dtXrb9dbcff4ImItKxBgwZSVbbSeHh4lFqVbd68ec+035SUFMyaNQtLliwp9Rp6Z2dnREVFITY2VqPIFgD897//xf79+7F161Y0atQI06ZNw5YtWzTi/OOPP/D1119j79690NfXx6RJkzRqi5w8eRI//fQTWrdujYkTJ2Lbtm2YNm0aAODKlSuIjo5GZmYmnJyc4OzsjCZNmmDhwoWIjo7G66+/jo0bNyIoKEg6+1Ge3377Dfv27YOOjg5GjBgBf39/dO/eHbdu3cIXX3yhUZBLLpjgieoIlkylyqpsudi5c+dCqVRq/N7/tICAAMycORO2trYa7ZUpD3vmzBl07dpVqoQ4YsQIfP7559L9RQVugH/KxRZxdXWFrq4umjdvDktLS1y4cAFAYXnt119/HUDh8uafffaZdOagPDY2NtKkRWNjY+zcuRNGRkYwMzMr9+CqLmOCJ6oDWDKVnofAwEAYGRnBx8cHHTp0gLm5eYnHdOzYEd26dcOGDRvQpUsXqb0y5WEfPHigUd776fuLF80qKhdbpGgOAlBYVvbBgwcoKCgoUS68qOR2RYqXa122bBkiIiLw4YcfolGjRpg7d65GpTm5YIInqgNYMpWeB3Nzc7Rs2RIzZ87E9OnTER0drVGjvcisWbMwfPhwjfLelSkPq6enh+zsbOn23bt3Kx3bgwcPNP5flKDPnj0rtWdlZUFXVxcGBgbQ0dHRqPxWVNWuNPr6+ggMDERgYCBOnjyJadOmwdraGo0bN650fHUBJ9kREb3kPDw8YGFhgfnz55daHtXExAQeHh5SRUoAlSoP++677+Ls2bPIzMyEUqnE3r17Kx1TTEwM1Go1UlNTce7cOVhaWqJPnz64cOECkpKSABSWHbeyskK9evXQokUL/PXXXwAKzy6UdZmdSqWCp6cnUlNTARQe5NSrV0+WawhwBE9ERFi0aBFGjhyJ9evXY8qUKSXu9/HxwdatW6Xb/fv3r7A8bKdOnTBo0CC4urrC1NQUzs7OUhIuT0FBATp16oThw4cjMzMTEyZMQLt27QAAoaGhmDx5MvLz82FqaiqVrf3ggw8wbdo0uLu741//+hdsbW1LvSSufv36GDlyJMaNGwchBOrVq4egoCBZLvHLcrFERPTcFF9hLz4+Hl988QV27Nih5aheDjxFT0REz0VGRgZ69uyJ27dvAwD27NmjMVGPni+O4ImI6Ln56aef8PXXXwMAzMzMsGTJEhgaGmo5qpcDEzwREZEM8RQ9ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDDHBExERyRATPBERkQwxwRMREckQEzwREZEMMcETERHJEBM8ERGRDP0/PgYuBILDikkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(<Figure size 432x288 with 1 Axes>,\n",
       " <matplotlib.axes._subplots.AxesSubplot at 0x7f1ce4cd93c8>)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.plot_critical_difference_diagram()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}