Ranking Table.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import pandas\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "adult_df = pandas.read_csv(\"./data/adult.csv\", index_col=0)\n",
    "compas_df = pandas.read_csv(\"./data/compas.csv\", index_col=0)\n",
    "bank_df = pandas.read_csv(\"./data/bank.csv\", index_col=0)\n",
    "german_df = pandas.read_csv(\"./data/german.csv\", index_col=0)\n",
    "\n",
    "\n",
    "for df in [adult_df, compas_df, bank_df, german_df]:\n",
    "    df[\"FPR Diff\"] = abs(df[\"False Positive Rate - Unprivileged\"] - df[\"False Positive Rate - Privileged\"])\n",
    "    df[\"FNR Diff\"] = abs(df[\"False Negative Rate - Unprivileged\"] - df[\"False Negative Rate - Privileged\"])\n",
    "    df[\"Accuracy Diff\"] = abs(df[\"Accuracy - Unprivileged\"] - df[\"Accuracy - Privileged\"])\n",
    "    df[\"FOR Diff\"] = abs(df[\"False Omission Rate - Unprivileged\"] - df[\"False Omission Rate - Privileged\"])\n",
    "    df[\"FDR Diff\"] = abs(df[\"False Discovery Rate - Unprivileged\"] - df[\"False Discovery Rate - Privileged\"])\n",
    "    df[\"Statistical Parity Diff\"] = abs(df[\"Unprivileged base Rate\"] - df[\"Privileged base Rate\"])\n",
    "    df[\"F1 Score Diff\"] = abs(df[\"F1 Score - Unprivileged\"] - df[\"F1 Score - Privileged\"])\n",
    "    for metric in df.columns:\n",
    "        if \"Num\" in metric or \"rivileged\" in metric:\n",
    "            df.drop(metric, axis=1, inplace=True)\n",
    "    df.set_index = df[df.columns[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1 Score</th>\n",
       "      <th>Theil Index</th>\n",
       "      <th>Consistency</th>\n",
       "      <th>FPR Diff</th>\n",
       "      <th>FNR Diff</th>\n",
       "      <th>Accuracy Diff</th>\n",
       "      <th>FOR Diff</th>\n",
       "      <th>FDR Diff</th>\n",
       "      <th>Statistical Parity Diff</th>\n",
       "      <th>F1 Score Diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CEOP</th>\n",
       "      <td>0.778617</td>\n",
       "      <td>0.558542</td>\n",
       "      <td>0.144825</td>\n",
       "      <td>0.916858</td>\n",
       "      <td>0.197183</td>\n",
       "      <td>0.346917</td>\n",
       "      <td>0.063383</td>\n",
       "      <td>0.073542</td>\n",
       "      <td>0.153208</td>\n",
       "      <td>0.235717</td>\n",
       "      <td>0.353617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR</th>\n",
       "      <td>0.790142</td>\n",
       "      <td>0.681533</td>\n",
       "      <td>0.130058</td>\n",
       "      <td>0.903408</td>\n",
       "      <td>0.063425</td>\n",
       "      <td>0.095142</td>\n",
       "      <td>0.091800</td>\n",
       "      <td>0.095150</td>\n",
       "      <td>0.085658</td>\n",
       "      <td>0.128625</td>\n",
       "      <td>0.100050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + CEOP</th>\n",
       "      <td>0.776742</td>\n",
       "      <td>0.578133</td>\n",
       "      <td>0.137025</td>\n",
       "      <td>0.923800</td>\n",
       "      <td>0.143667</td>\n",
       "      <td>0.328792</td>\n",
       "      <td>0.071875</td>\n",
       "      <td>0.079100</td>\n",
       "      <td>0.131833</td>\n",
       "      <td>0.196708</td>\n",
       "      <td>0.348875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR</th>\n",
       "      <td>0.785208</td>\n",
       "      <td>0.680767</td>\n",
       "      <td>0.131500</td>\n",
       "      <td>0.897950</td>\n",
       "      <td>0.056492</td>\n",
       "      <td>0.104133</td>\n",
       "      <td>0.103867</td>\n",
       "      <td>0.111025</td>\n",
       "      <td>0.115858</td>\n",
       "      <td>0.087208</td>\n",
       "      <td>0.119992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + CEOP</th>\n",
       "      <td>0.772817</td>\n",
       "      <td>0.575917</td>\n",
       "      <td>0.140692</td>\n",
       "      <td>0.916608</td>\n",
       "      <td>0.065100</td>\n",
       "      <td>0.174050</td>\n",
       "      <td>0.096567</td>\n",
       "      <td>0.096767</td>\n",
       "      <td>0.101833</td>\n",
       "      <td>0.117842</td>\n",
       "      <td>0.208825</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Accuracy  F1 Score  Theil Index  Consistency  FPR Diff  \\\n",
       "CEOP              0.778617  0.558542     0.144825     0.916858  0.197183   \n",
       "DIR               0.790142  0.681533     0.130058     0.903408  0.063425   \n",
       "DIR + CEOP        0.776742  0.578133     0.137025     0.923800  0.143667   \n",
       "DIR + EGR         0.785208  0.680767     0.131500     0.897950  0.056492   \n",
       "DIR + EGR + CEOP  0.772817  0.575917     0.140692     0.916608  0.065100   \n",
       "\n",
       "                  FNR Diff  Accuracy Diff  FOR Diff  FDR Diff  \\\n",
       "CEOP              0.346917       0.063383  0.073542  0.153208   \n",
       "DIR               0.095142       0.091800  0.095150  0.085658   \n",
       "DIR + CEOP        0.328792       0.071875  0.079100  0.131833   \n",
       "DIR + EGR         0.104133       0.103867  0.111025  0.115858   \n",
       "DIR + EGR + CEOP  0.174050       0.096567  0.096767  0.101833   \n",
       "\n",
       "                  Statistical Parity Diff  F1 Score Diff  \n",
       "CEOP                             0.235717       0.353617  \n",
       "DIR                              0.128625       0.100050  \n",
       "DIR + CEOP                       0.196708       0.348875  \n",
       "DIR + EGR                        0.087208       0.119992  \n",
       "DIR + EGR + CEOP                 0.117842       0.208825  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pandas.concat([adult_df, bank_df, german_df, compas_df])\n",
    "df = df.abs()\n",
    "df = df.groupby(df.index).mean()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 ['Logistic Regression']\n",
      "9 ['OP', 'DIR', 'EOP', 'CEOP', 'ROC', 'GFC', 'PR', 'EGR', 'GSR']\n",
      "26 ['OP + EOP', 'OP + CEOP', 'OP + ROC', 'OP + GFC', 'OP + PR', 'OP + EGR', 'OP + GSR', 'DIR + EOP', 'DIR + CEOP', 'DIR + ROC', 'DIR + GFC', 'DIR + PR', 'DIR + EGR', 'DIR + GSR', 'GFC + EOP', 'GFC + CEOP', 'GFC + ROC', 'PR + EOP', 'PR + CEOP', 'PR + ROC', 'EGR + EOP', 'EGR + CEOP', 'EGR + ROC', 'GSR + EOP', 'GSR + CEOP', 'GSR + ROC']\n",
      "24 ['OP + GFC + EOP', 'OP + GFC + CEOP', 'OP + GFC + ROC', 'OP + PR + EOP', 'OP + PR + CEOP', 'OP + PR + ROC', 'OP + EGR + EOP', 'OP + EGR + CEOP', 'OP + EGR + ROC', 'OP + GSR + EOP', 'OP + GSR + CEOP', 'OP + GSR + ROC', 'DIR + GFC + EOP', 'DIR + GFC + CEOP', 'DIR + GFC + ROC', 'DIR + PR + EOP', 'DIR + PR + CEOP', 'DIR + PR + ROC', 'DIR + EGR + EOP', 'DIR + EGR + CEOP', 'DIR + EGR + ROC', 'DIR + GSR + EOP', 'DIR + GSR + CEOP', 'DIR + GSR + ROC']\n"
     ]
    }
   ],
   "source": [
    "def num_interventions(name):\n",
    "    if \"Logistic Regression\" in name:\n",
    "        return 0\n",
    "    if name.count('+') == 0:\n",
    "        return 1\n",
    "    if name.count('+') == 1:\n",
    "        return 2\n",
    "    if name.count('+') == 2:\n",
    "        return 3\n",
    "    \n",
    "no_int = []\n",
    "one_int = []\n",
    "two_int = []\n",
    "three_int = []\n",
    "for row in list(adult_df.index):\n",
    "    if num_interventions(row) == 0:\n",
    "        no_int.append(row)\n",
    "    elif num_interventions(row) == 1:\n",
    "        one_int.append(row)\n",
    "    elif num_interventions(row) == 2:\n",
    "        two_int.append(row)\n",
    "    elif num_interventions(row) == 3:\n",
    "        three_int.append(row)\n",
    "        \n",
    "print(len(no_int), no_int)\n",
    "print(len(one_int), one_int)\n",
    "print(len(two_int), two_int)\n",
    "print(len(three_int), three_int)\n",
    "\n",
    "row_ordering = no_int + one_int + two_int + three_int\n",
    "df = df.loc[row_ordering]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save these results to file\n",
    "# this file contains mean metric score for all interventions\n",
    "df.to_csv(\"./results/R4_abs_mean_scores.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metric descending:  Accuracy\n",
      "metric descending:  F1 Score\n",
      "metric ascending:  Theil Index\n",
      "metric descending:  Consistency\n",
      "metric ascending:  FPR Diff\n",
      "metric ascending:  FNR Diff\n",
      "metric ascending:  Accuracy Diff\n",
      "metric ascending:  FOR Diff\n",
      "metric ascending:  FDR Diff\n",
      "metric ascending:  Statistical Parity Diff\n",
      "metric ascending:  F1 Score Diff\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rank</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1 Score</th>\n",
       "      <th>Theil Index</th>\n",
       "      <th>Consistency</th>\n",
       "      <th>FPR Diff</th>\n",
       "      <th>FNR Diff</th>\n",
       "      <th>Accuracy Diff</th>\n",
       "      <th>FOR Diff</th>\n",
       "      <th>FDR Diff</th>\n",
       "      <th>Statistical Parity Diff</th>\n",
       "      <th>F1 Score Diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>EGR + EOP</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>GSR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>DIR</td>\n",
       "      <td>DIR</td>\n",
       "      <td>DIR</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>DIR</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>GFC + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>GSR</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>GSR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>EGR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "      <td>OP</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>ROC</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>PR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>EOP</td>\n",
       "      <td>PR</td>\n",
       "      <td>GFC</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>EGR</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>OP</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>DIR + PR</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>GFC + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>GSR + EOP</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>EGR + ROC</td>\n",
       "      <td>DIR + PR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>GSR</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>PR + ROC</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>PR</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>Logistic Regression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "      <td>EGR + EOP</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>GSR + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>EGR + ROC</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>GSR + EOP</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>GFC</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>EGR</td>\n",
       "      <td>EGR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "      <td>GSR + EOP</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>DIR</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>EGR + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>EGR</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>DIR + PR</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "      <td>OP + EGR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>EOP</td>\n",
       "      <td>EOP</td>\n",
       "      <td>DIR + GSR + ROC</td>\n",
       "      <td>GFC</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>DIR + GFC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>PR + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>GSR</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>PR + ROC</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>GSR + EOP</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>DIR + GSR + ROC</td>\n",
       "      <td>DIR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>EGR + EOP</td>\n",
       "      <td>EOP</td>\n",
       "      <td>DIR + GSR + ROC</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "      <td>GSR + EOP</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>GFC</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>DIR + PR</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>DIR + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>GSR + EOP</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>ROC</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>21</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>DIR + GSR + ROC</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>EGR</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>GFC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>22</td>\n",
       "      <td>ROC</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>EGR + ROC</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "      <td>GSR</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>OP + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>23</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>EGR</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>EOP</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>24</td>\n",
       "      <td>PR + ROC</td>\n",
       "      <td>PR + ROC</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>25</td>\n",
       "      <td>EOP</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>EGR + ROC</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>26</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "      <td>EGR + EOP</td>\n",
       "      <td>OP</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "      <td>GSR</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>27</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "      <td>GFC</td>\n",
       "      <td>GFC</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "      <td>DIR + GSR + ROC</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "      <td>ROC</td>\n",
       "      <td>DIR + EGR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>28</td>\n",
       "      <td>OP</td>\n",
       "      <td>ROC</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>DIR</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>EGR + EOP</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "      <td>ROC</td>\n",
       "      <td>EGR + EOP</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>29</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>GSR + EOP</td>\n",
       "      <td>GSR + EOP</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>30</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>PR + ROC</td>\n",
       "      <td>PR + ROC</td>\n",
       "      <td>PR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>31</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>EOP</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>EGR</td>\n",
       "      <td>GSR</td>\n",
       "      <td>OP + GFC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>32</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>EGR + ROC</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "      <td>EGR + EOP</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>33</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>EGR + EOP</td>\n",
       "      <td>EGR + ROC</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>DIR + GSR + ROC</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "      <td>OP</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>EOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>34</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>DIR + PR</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>PR</td>\n",
       "      <td>PR</td>\n",
       "      <td>EGR + EOP</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>35</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>GFC</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>DIR + PR</td>\n",
       "      <td>PR + ROC</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>36</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "      <td>GSR</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>OP</td>\n",
       "      <td>DIR</td>\n",
       "      <td>ROC</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>37</td>\n",
       "      <td>GFC</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>PR + ROC</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>DIR</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>EGR + ROC</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>38</td>\n",
       "      <td>DIR + PR</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>ROC</td>\n",
       "      <td>PR</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>DIR</td>\n",
       "      <td>EGR</td>\n",
       "      <td>GSR</td>\n",
       "      <td>OP</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>OP + PR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>39</td>\n",
       "      <td>PR</td>\n",
       "      <td>EGR + ROC</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>EGR</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>ROC</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>GFC</td>\n",
       "      <td>DIR + GSR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>40</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "      <td>GSR</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>EGR</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>DIR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>41</td>\n",
       "      <td>DIR + GSR + ROC</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>ROC</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "      <td>PR + ROC</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "      <td>OP + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>42</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>EGR + ROC</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>43</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>DIR + PR</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>GSR</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>OP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>44</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>PR</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>PR + ROC</td>\n",
       "      <td>PR</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>45</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "      <td>DIR + GSR + ROC</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>OP + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>46</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>DIR + PR</td>\n",
       "      <td>DIR + GSR + ROC</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>EGR + ROC</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "      <td>DIR + PR</td>\n",
       "      <td>OP + GSR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>47</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>PR</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "      <td>DIR + GSR + ROC</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>48</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>GSR + EOP</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>OP</td>\n",
       "      <td>DIR + GSR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>49</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>EGR</td>\n",
       "      <td>OP</td>\n",
       "      <td>EOP</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>50</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>GFC</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>EGR + EOP</td>\n",
       "      <td>DIR + PR</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>GSR + EOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>DIR</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>51</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>52</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>PR</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>53</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "      <td>OP</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>54</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>55</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>56</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>EOP</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>57</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>58</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>PR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>59</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>60</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>CEOP</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Rank             Accuracy             F1 Score          Theil Index  \\\n",
       "0      1  Logistic Regression      DIR + EGR + ROC      DIR + EGR + ROC   \n",
       "1      2                  DIR                  DIR                  DIR   \n",
       "2      3                  GSR            DIR + GSR            DIR + EGR   \n",
       "3      4      DIR + EGR + ROC            DIR + EGR            DIR + GSR   \n",
       "4      5            DIR + EGR  Logistic Regression       DIR + PR + EOP   \n",
       "5      6            DIR + GSR            OP + CEOP            DIR + ROC   \n",
       "6      7                  EGR      OP + GSR + CEOP           DIR + CEOP   \n",
       "7      8      DIR + PR + CEOP                   OP  Logistic Regression   \n",
       "8      9                 CEOP             OP + GSR       DIR + PR + ROC   \n",
       "9     10            PR + CEOP                  GSR     DIR + GSR + CEOP   \n",
       "10    11           DIR + CEOP       DIR + PR + EOP     DIR + EGR + CEOP   \n",
       "11    12            EGR + ROC       OP + PR + CEOP            GSR + EOP   \n",
       "12    13           GSR + CEOP             PR + EOP             PR + EOP   \n",
       "13    14     DIR + GSR + CEOP                  EGR      DIR + PR + CEOP   \n",
       "14    15             PR + EOP       OP + EGR + ROC            DIR + EOP   \n",
       "15    16     DIR + EGR + CEOP             OP + EGR                 CEOP   \n",
       "16    17       DIR + PR + EOP            DIR + EOP                  GSR   \n",
       "17    18            EGR + EOP                  EOP      DIR + GSR + ROC   \n",
       "18    19           EGR + CEOP            GSR + EOP      OP + GSR + CEOP   \n",
       "19    20            GSR + EOP      OP + EGR + CEOP            OP + CEOP   \n",
       "20    21            DIR + EOP      DIR + GSR + ROC           GSR + CEOP   \n",
       "21    22                  ROC      DIR + EGR + EOP      DIR + EGR + EOP   \n",
       "22    23      DIR + GSR + EOP        OP + PR + ROC                  EGR   \n",
       "23    24             PR + ROC             PR + ROC             OP + GSR   \n",
       "24    25                  EOP             OP + ROC       OP + PR + CEOP   \n",
       "25    26      DIR + EGR + EOP            EGR + EOP                   OP   \n",
       "26    27            GSR + ROC       OP + EGR + EOP      DIR + GSR + EOP   \n",
       "27    28                   OP                  ROC            PR + CEOP   \n",
       "28    29            OP + CEOP      DIR + PR + CEOP           EGR + CEOP   \n",
       "29    30             OP + GSR            DIR + ROC       OP + EGR + ROC   \n",
       "30    31      OP + EGR + CEOP       DIR + PR + ROC                  EOP   \n",
       "31    32      OP + GSR + CEOP            GSR + ROC            EGR + ROC   \n",
       "32    33       OP + PR + CEOP            PR + CEOP            EGR + EOP   \n",
       "33    34             OP + EGR        OP + PR + EOP             OP + EGR   \n",
       "34    35       OP + EGR + ROC      DIR + GSR + EOP                  GFC   \n",
       "35    36            GFC + EOP           DIR + CEOP      OP + EGR + CEOP   \n",
       "36    37                  GFC     DIR + EGR + CEOP             PR + ROC   \n",
       "37    38             DIR + PR             OP + EOP                  ROC   \n",
       "38    39                   PR            EGR + ROC      OP + GFC + CEOP   \n",
       "39    40            GFC + ROC       OP + GSR + EOP       OP + GFC + ROC   \n",
       "40    41      DIR + GSR + ROC      OP + GFC + CEOP            GFC + EOP   \n",
       "41    42       OP + EGR + EOP       OP + GSR + ROC       OP + EGR + EOP   \n",
       "42    43             OP + GFC             DIR + PR            GSR + ROC   \n",
       "43    44             OP + EOP                   PR     DIR + GFC + CEOP   \n",
       "44    45       OP + GSR + EOP     DIR + GSR + CEOP      DIR + GFC + EOP   \n",
       "45    46       OP + GFC + ROC       OP + GFC + ROC             DIR + PR   \n",
       "46    47        OP + PR + EOP                 CEOP                   PR   \n",
       "47    48     DIR + GFC + CEOP           EGR + CEOP             OP + EOP   \n",
       "48    49           GFC + CEOP           GSR + CEOP        OP + PR + EOP   \n",
       "49    50      DIR + GFC + ROC                  GFC       OP + GSR + EOP   \n",
       "50    51      OP + GFC + CEOP              OP + PR        OP + PR + ROC   \n",
       "51    52        OP + PR + ROC       OP + GFC + EOP              OP + PR   \n",
       "52    53      DIR + GFC + EOP      DIR + GFC + EOP             OP + ROC   \n",
       "53    54            DIR + GFC            GFC + EOP             OP + GFC   \n",
       "54    55             OP + ROC             OP + GFC            DIR + GFC   \n",
       "55    56       OP + GFC + EOP            DIR + GFC           GFC + CEOP   \n",
       "56    57       OP + GSR + ROC            GFC + ROC       OP + GFC + EOP   \n",
       "57    58       DIR + PR + ROC     DIR + GFC + CEOP            GFC + ROC   \n",
       "58    59            DIR + ROC           GFC + CEOP       OP + GSR + ROC   \n",
       "59    60              OP + PR      DIR + GFC + ROC      DIR + GFC + ROC   \n",
       "\n",
       "            Consistency             FPR Diff             FNR Diff  \\\n",
       "0        OP + GFC + ROC            EGR + EOP       OP + GFC + EOP   \n",
       "1       OP + GFC + CEOP       OP + GFC + ROC            GFC + EOP   \n",
       "2              OP + GFC             PR + EOP       OP + EGR + EOP   \n",
       "3              OP + GSR       OP + EGR + EOP             OP + EOP   \n",
       "4                    OP       OP + GFC + EOP       OP + GSR + ROC   \n",
       "5        OP + EGR + ROC             OP + GFC       OP + GFC + ROC   \n",
       "6               OP + PR            DIR + GFC        OP + PR + EOP   \n",
       "7       OP + GSR + CEOP            GFC + EOP      DIR + GSR + EOP   \n",
       "8             OP + CEOP            DIR + EOP            GSR + EOP   \n",
       "9        OP + PR + CEOP      DIR + GSR + EOP             OP + GFC   \n",
       "10       OP + GSR + ROC      DIR + EGR + EOP            EGR + EOP   \n",
       "11             OP + ROC        OP + PR + EOP       OP + GSR + EOP   \n",
       "12      OP + EGR + CEOP            GSR + EOP      DIR + GFC + ROC   \n",
       "13             OP + EGR        OP + PR + ROC      DIR + EGR + EOP   \n",
       "14            DIR + GFC                  EOP                  EOP   \n",
       "15      DIR + GFC + ROC            GSR + ROC            DIR + GFC   \n",
       "16     DIR + GFC + CEOP             PR + ROC            DIR + EOP   \n",
       "17        OP + PR + ROC             OP + EOP       OP + EGR + ROC   \n",
       "18                  GFC             OP + ROC             DIR + PR   \n",
       "19           GFC + CEOP                  ROC       DIR + PR + ROC   \n",
       "20            GFC + ROC                  EGR             OP + ROC   \n",
       "21           DIR + CEOP            EGR + ROC            DIR + ROC   \n",
       "22     DIR + GSR + CEOP       OP + GSR + ROC             PR + EOP   \n",
       "23                 CEOP             OP + EGR        OP + PR + ROC   \n",
       "24     DIR + EGR + CEOP            GFC + ROC             OP + EGR   \n",
       "25           GSR + CEOP       OP + EGR + ROC       DIR + PR + EOP   \n",
       "26           EGR + CEOP                  GFC                  GFC   \n",
       "27                  DIR      OP + GFC + CEOP              OP + PR   \n",
       "28            DIR + GSR     DIR + GFC + CEOP            GFC + ROC   \n",
       "29      DIR + PR + CEOP       DIR + PR + EOP             OP + GSR   \n",
       "30            DIR + EGR      DIR + GFC + EOP      DIR + GFC + EOP   \n",
       "31      DIR + EGR + ROC       OP + GSR + EOP       OP + PR + CEOP   \n",
       "32            EGR + ROC            DIR + GSR      DIR + GSR + ROC   \n",
       "33             DIR + PR      DIR + GFC + ROC                   PR   \n",
       "34  Logistic Regression            DIR + EGR      OP + GSR + CEOP   \n",
       "35                  GSR             OP + GSR                   OP   \n",
       "36            PR + CEOP                  DIR  Logistic Regression   \n",
       "37                   PR      DIR + EGR + ROC                  DIR   \n",
       "38                  EGR     DIR + EGR + CEOP                  ROC   \n",
       "39       OP + EGR + EOP      OP + EGR + CEOP                  GSR   \n",
       "40                  ROC           EGR + CEOP             PR + ROC   \n",
       "41            DIR + ROC              OP + PR            DIR + EGR   \n",
       "42            GSR + ROC                  GSR            OP + CEOP   \n",
       "43             PR + ROC                   PR      DIR + EGR + ROC   \n",
       "44       DIR + PR + EOP      DIR + GSR + ROC            GSR + ROC   \n",
       "45      DIR + GSR + ROC       OP + PR + CEOP            EGR + ROC   \n",
       "46            GFC + EOP      OP + GSR + CEOP      OP + GFC + CEOP   \n",
       "47            GSR + EOP       DIR + PR + ROC      OP + EGR + CEOP   \n",
       "48      DIR + GSR + EOP            DIR + ROC                  EGR   \n",
       "49            EGR + EOP             DIR + PR            DIR + GSR   \n",
       "50             PR + EOP  Logistic Regression           GFC + CEOP   \n",
       "51       DIR + PR + ROC           GSR + CEOP     DIR + EGR + CEOP   \n",
       "52      DIR + EGR + EOP                   OP           GSR + CEOP   \n",
       "53            DIR + EOP            PR + CEOP     DIR + GFC + CEOP   \n",
       "54      DIR + GFC + EOP           GFC + CEOP           EGR + CEOP   \n",
       "55                  EOP     DIR + GSR + CEOP     DIR + GSR + CEOP   \n",
       "56       OP + GSR + EOP      DIR + PR + CEOP      DIR + PR + CEOP   \n",
       "57             OP + EOP            OP + CEOP            PR + CEOP   \n",
       "58        OP + PR + EOP           DIR + CEOP           DIR + CEOP   \n",
       "59       OP + GFC + EOP                 CEOP                 CEOP   \n",
       "\n",
       "          Accuracy Diff             FOR Diff             FDR Diff  \\\n",
       "0       DIR + GFC + ROC              OP + PR  Logistic Regression   \n",
       "1             DIR + GFC     DIR + GFC + CEOP                  DIR   \n",
       "2         OP + PR + ROC                 CEOP           GSR + CEOP   \n",
       "3              OP + ROC            PR + CEOP     DIR + EGR + CEOP   \n",
       "4                   ROC           DIR + CEOP           GFC + CEOP   \n",
       "5                   EOP                   PR                  GFC   \n",
       "6       DIR + GFC + EOP      DIR + PR + CEOP            DIR + GFC   \n",
       "7        OP + GSR + ROC       OP + GFC + ROC             DIR + PR   \n",
       "8            GFC + CEOP     DIR + GSR + CEOP            DIR + EGR   \n",
       "9              PR + ROC           GFC + CEOP                   PR   \n",
       "10            GFC + EOP             OP + GSR           EGR + CEOP   \n",
       "11                  GFC            GFC + ROC      DIR + EGR + ROC   \n",
       "12                 CEOP                  DIR           DIR + CEOP   \n",
       "13           GSR + CEOP             DIR + PR            PR + CEOP   \n",
       "14      DIR + GSR + ROC                  GFC            GFC + ROC   \n",
       "15            GSR + ROC  Logistic Regression            GFC + EOP   \n",
       "16            GSR + EOP     DIR + EGR + CEOP            GSR + ROC   \n",
       "17           DIR + CEOP             PR + EOP      DIR + PR + CEOP   \n",
       "18     DIR + GFC + CEOP      DIR + GFC + ROC            DIR + GSR   \n",
       "19     DIR + GSR + CEOP            DIR + GFC                 CEOP   \n",
       "20            PR + CEOP            DIR + ROC      DIR + GFC + ROC   \n",
       "21            DIR + EOP      OP + EGR + CEOP                  GSR   \n",
       "22             PR + EOP       DIR + PR + ROC                  EOP   \n",
       "23           EGR + CEOP             OP + GFC      DIR + GFC + EOP   \n",
       "24            GFC + ROC            DIR + EGR            EGR + ROC   \n",
       "25                  GSR      OP + GFC + CEOP     DIR + GSR + CEOP   \n",
       "26      DIR + EGR + EOP      DIR + GSR + ROC       DIR + PR + EOP   \n",
       "27            EGR + EOP           EGR + CEOP                  ROC   \n",
       "28      DIR + PR + CEOP      DIR + EGR + ROC            GSR + EOP   \n",
       "29      DIR + GSR + EOP            DIR + GSR             PR + ROC   \n",
       "30  Logistic Regression           GSR + CEOP                  EGR   \n",
       "31       OP + GFC + EOP       DIR + PR + EOP            EGR + EOP   \n",
       "32       DIR + PR + EOP                   OP            DIR + EOP   \n",
       "33                   PR            EGR + EOP             PR + EOP   \n",
       "34             DIR + PR             PR + ROC            OP + CEOP   \n",
       "35                  DIR                  ROC            DIR + ROC   \n",
       "36             OP + EOP            EGR + ROC       DIR + PR + ROC   \n",
       "37                  EGR                  GSR                   OP   \n",
       "38       DIR + PR + ROC       OP + GSR + EOP             OP + ROC   \n",
       "39            DIR + ROC                  EGR      OP + GSR + CEOP   \n",
       "40     DIR + EGR + CEOP      DIR + GSR + EOP      DIR + EGR + EOP   \n",
       "41            EGR + ROC        OP + PR + EOP             OP + GSR   \n",
       "42        OP + PR + EOP            DIR + EOP       OP + GFC + ROC   \n",
       "43            DIR + GSR       OP + PR + CEOP       OP + PR + CEOP   \n",
       "44            DIR + EGR       OP + GFC + EOP        OP + PR + ROC   \n",
       "45       OP + EGR + EOP      OP + GSR + CEOP      OP + EGR + CEOP   \n",
       "46      DIR + EGR + ROC      DIR + EGR + EOP      DIR + GSR + ROC   \n",
       "47       OP + GSR + EOP             OP + EOP             OP + GFC   \n",
       "48                   OP                  EOP     DIR + GFC + CEOP   \n",
       "49            OP + CEOP            GSR + EOP       OP + GSR + ROC   \n",
       "50             OP + EGR            GSR + ROC      OP + GFC + CEOP   \n",
       "51              OP + PR            GFC + EOP      DIR + GSR + EOP   \n",
       "52       OP + PR + CEOP             OP + ROC             OP + EGR   \n",
       "53       OP + EGR + ROC            OP + CEOP       OP + EGR + ROC   \n",
       "54             OP + GSR       OP + GSR + ROC              OP + PR   \n",
       "55      OP + GSR + CEOP        OP + PR + ROC             OP + EOP   \n",
       "56       OP + GFC + ROC       OP + EGR + EOP       OP + EGR + EOP   \n",
       "57      OP + EGR + CEOP      DIR + GFC + EOP        OP + PR + EOP   \n",
       "58             OP + GFC       OP + EGR + ROC       OP + GFC + EOP   \n",
       "59      OP + GFC + CEOP             OP + EGR       OP + GSR + EOP   \n",
       "\n",
       "   Statistical Parity Diff        F1 Score Diff  \n",
       "0           OP + GFC + EOP            GSR + ROC  \n",
       "1           OP + EGR + ROC            GFC + EOP  \n",
       "2                 OP + EGR                  GSR  \n",
       "3           OP + GSR + EOP            EGR + ROC  \n",
       "4           OP + GFC + ROC                   PR  \n",
       "5                GSR + ROC      DIR + GFC + EOP  \n",
       "6                DIR + GFC                  EOP  \n",
       "7                 OP + EOP            GFC + ROC  \n",
       "8                EGR + ROC             DIR + PR  \n",
       "9            OP + PR + EOP  Logistic Regression  \n",
       "10          OP + GSR + ROC            GSR + EOP  \n",
       "11                     EGR                  EGR  \n",
       "12                OP + GFC            EGR + EOP  \n",
       "13         DIR + EGR + EOP             OP + EGR  \n",
       "14          DIR + PR + ROC            DIR + GFC  \n",
       "15          OP + EGR + EOP             PR + EOP  \n",
       "16         DIR + GSR + ROC                  DIR  \n",
       "17           OP + PR + ROC      DIR + GSR + EOP  \n",
       "18         DIR + GSR + EOP            DIR + EOP  \n",
       "19               GFC + EOP       OP + EGR + ROC  \n",
       "20               DIR + EOP                  GFC  \n",
       "21         DIR + GFC + EOP             OP + EOP  \n",
       "22         DIR + GFC + ROC       OP + GFC + EOP  \n",
       "23               DIR + ROC      DIR + EGR + EOP  \n",
       "24          DIR + PR + EOP      DIR + GFC + ROC  \n",
       "25                OP + ROC                  ROC  \n",
       "26                     ROC            DIR + EGR  \n",
       "27               EGR + EOP       DIR + PR + EOP  \n",
       "28               GSR + EOP       OP + GSR + EOP  \n",
       "29                PR + ROC             PR + ROC  \n",
       "30                     GSR             OP + GFC  \n",
       "31         OP + GFC + CEOP        OP + PR + EOP  \n",
       "32                     EOP       OP + GSR + ROC  \n",
       "33         DIR + EGR + ROC      DIR + EGR + ROC  \n",
       "34                PR + EOP       OP + EGR + EOP  \n",
       "35        DIR + GFC + CEOP        OP + PR + ROC  \n",
       "36               GFC + ROC       OP + GFC + ROC  \n",
       "37                OP + GSR              OP + PR  \n",
       "38                     GFC      DIR + GSR + ROC  \n",
       "39               DIR + EGR            DIR + ROC  \n",
       "40         OP + EGR + CEOP             OP + ROC  \n",
       "41                 OP + PR       DIR + PR + ROC  \n",
       "42         OP + GSR + CEOP                   OP  \n",
       "43          OP + PR + CEOP       OP + PR + CEOP  \n",
       "44               DIR + GSR            OP + CEOP  \n",
       "45                DIR + PR             OP + GSR  \n",
       "46        DIR + EGR + CEOP      OP + GSR + CEOP  \n",
       "47                      OP            DIR + GSR  \n",
       "48              EGR + CEOP     DIR + EGR + CEOP  \n",
       "49                     DIR           GFC + CEOP  \n",
       "50              GFC + CEOP      OP + GFC + CEOP  \n",
       "51                      PR           EGR + CEOP  \n",
       "52               OP + CEOP     DIR + GFC + CEOP  \n",
       "53              GSR + CEOP      OP + EGR + CEOP  \n",
       "54         DIR + PR + CEOP           GSR + CEOP  \n",
       "55     Logistic Regression     DIR + GSR + CEOP  \n",
       "56        DIR + GSR + CEOP      DIR + PR + CEOP  \n",
       "57               PR + CEOP            PR + CEOP  \n",
       "58              DIR + CEOP           DIR + CEOP  \n",
       "59                    CEOP                 CEOP  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ranked = {}\n",
    "for metric in df.columns:\n",
    "    if \"Diff\" in metric or \"Theil Index\" in metric:\n",
    "        print(\"metric ascending: \", metric)\n",
    "        df_ranked[metric] = df[metric].sort_values(ascending=True).index\n",
    "    elif metric in [\"Accuracy\", \"F1 Score\", \"Consistency\"]:\n",
    "        print(\"metric descending: \", metric)\n",
    "        # accuracy, f1 score descending\n",
    "        df_ranked[metric] = df[metric].sort_values(ascending=False).index\n",
    "    else:\n",
    "        print(\"Unaccounted metric: \", metric)\n",
    "df_ranked = pandas.DataFrame(df_ranked)\n",
    "\n",
    "# add rank column\n",
    "df_ranked.insert(0, \"Rank\", range(1,61))\n",
    "df_ranked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ranked.to_csv(\"./results/R4_mean_ranked.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DIR + EGR + ROC        0.683392\n",
       "DIR                    0.681533\n",
       "DIR + GSR              0.681300\n",
       "DIR + EGR              0.680767\n",
       "Logistic Regression    0.679983\n",
       "OP + CEOP              0.669189\n",
       "OP + GSR + CEOP        0.667800\n",
       "OP                     0.667689\n",
       "OP + GSR               0.666300\n",
       "GSR                    0.663758\n",
       "Name: F1 Score, dtype: float64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"F1 Score\"].sort_values(ascending=False)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rank</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1 Score</th>\n",
       "      <th>Theil Index</th>\n",
       "      <th>Consistency</th>\n",
       "      <th>FPR Diff</th>\n",
       "      <th>FNR Diff</th>\n",
       "      <th>Accuracy Diff</th>\n",
       "      <th>FOR Diff</th>\n",
       "      <th>FDR Diff</th>\n",
       "      <th>Statistical Parity Diff</th>\n",
       "      <th>F1 Score Diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>EGR + EOP</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>GSR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>DIR</td>\n",
       "      <td>DIR</td>\n",
       "      <td>DIR</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>DIR</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>GFC + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>GSR</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>GSR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>DIR + EGR + ROC</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>EGR + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>DIR + PR + EOP</td>\n",
       "      <td>OP</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>ROC</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>PR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>DIR + GSR</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>EOP</td>\n",
       "      <td>PR</td>\n",
       "      <td>GFC</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>EGR</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>EOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>OP</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>DIR + PR</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>GFC + ROC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>GSR + EOP</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>DIR + EGR</td>\n",
       "      <td>EGR + ROC</td>\n",
       "      <td>DIR + PR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>GSR</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>PR + ROC</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>PR</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>Logistic Regression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>51</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>PR + EOP</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>GSR + ROC</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>52</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>DIR + EGR + CEOP</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>DIR + GSR + EOP</td>\n",
       "      <td>PR</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>53</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>DIR + EGR + EOP</td>\n",
       "      <td>OP</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>OP + PR + CEOP</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>54</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>GFC + EOP</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>DIR + EOP</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>55</td>\n",
       "      <td>OP + ROC</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>EGR + CEOP</td>\n",
       "      <td>OP + GSR</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>GSR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>56</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>DIR + GFC</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>EOP</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>OP + GSR + CEOP</td>\n",
       "      <td>OP + PR + ROC</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>57</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "      <td>OP + GFC + ROC</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>OP + EGR + EOP</td>\n",
       "      <td>DIR + GSR + CEOP</td>\n",
       "      <td>DIR + PR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>58</td>\n",
       "      <td>DIR + PR + ROC</td>\n",
       "      <td>DIR + GFC + CEOP</td>\n",
       "      <td>GFC + ROC</td>\n",
       "      <td>OP + EOP</td>\n",
       "      <td>OP + CEOP</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>OP + EGR + CEOP</td>\n",
       "      <td>DIR + GFC + EOP</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>PR + CEOP</td>\n",
       "      <td>PR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>59</td>\n",
       "      <td>DIR + ROC</td>\n",
       "      <td>GFC + CEOP</td>\n",
       "      <td>OP + GSR + ROC</td>\n",
       "      <td>OP + PR + EOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>OP + GFC</td>\n",
       "      <td>OP + EGR + ROC</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "      <td>DIR + CEOP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>60</td>\n",
       "      <td>OP + PR</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>DIR + GFC + ROC</td>\n",
       "      <td>OP + GFC + EOP</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>OP + GFC + CEOP</td>\n",
       "      <td>OP + EGR</td>\n",
       "      <td>OP + GSR + EOP</td>\n",
       "      <td>CEOP</td>\n",
       "      <td>CEOP</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Rank             Accuracy             F1 Score          Theil Index  \\\n",
       "0      1  Logistic Regression      DIR + EGR + ROC      DIR + EGR + ROC   \n",
       "1      2                  DIR                  DIR                  DIR   \n",
       "2      3                  GSR            DIR + GSR            DIR + EGR   \n",
       "3      4      DIR + EGR + ROC            DIR + EGR            DIR + GSR   \n",
       "4      5            DIR + EGR  Logistic Regression       DIR + PR + EOP   \n",
       "5      6            DIR + GSR            OP + CEOP            DIR + ROC   \n",
       "6      7                  EGR      OP + GSR + CEOP           DIR + CEOP   \n",
       "7      8      DIR + PR + CEOP                   OP  Logistic Regression   \n",
       "8      9                 CEOP             OP + GSR       DIR + PR + ROC   \n",
       "9     10            PR + CEOP                  GSR     DIR + GSR + CEOP   \n",
       "50    51      OP + GFC + CEOP              OP + PR        OP + PR + ROC   \n",
       "51    52        OP + PR + ROC       OP + GFC + EOP              OP + PR   \n",
       "52    53      DIR + GFC + EOP      DIR + GFC + EOP             OP + ROC   \n",
       "53    54            DIR + GFC            GFC + EOP             OP + GFC   \n",
       "54    55             OP + ROC             OP + GFC            DIR + GFC   \n",
       "55    56       OP + GFC + EOP            DIR + GFC           GFC + CEOP   \n",
       "56    57       OP + GSR + ROC            GFC + ROC       OP + GFC + EOP   \n",
       "57    58       DIR + PR + ROC     DIR + GFC + CEOP            GFC + ROC   \n",
       "58    59            DIR + ROC           GFC + CEOP       OP + GSR + ROC   \n",
       "59    60              OP + PR      DIR + GFC + ROC      DIR + GFC + ROC   \n",
       "\n",
       "        Consistency             FPR Diff          FNR Diff    Accuracy Diff  \\\n",
       "0    OP + GFC + ROC            EGR + EOP    OP + GFC + EOP  DIR + GFC + ROC   \n",
       "1   OP + GFC + CEOP       OP + GFC + ROC         GFC + EOP        DIR + GFC   \n",
       "2          OP + GFC             PR + EOP    OP + EGR + EOP    OP + PR + ROC   \n",
       "3          OP + GSR       OP + EGR + EOP          OP + EOP         OP + ROC   \n",
       "4                OP       OP + GFC + EOP    OP + GSR + ROC              ROC   \n",
       "5    OP + EGR + ROC             OP + GFC    OP + GFC + ROC              EOP   \n",
       "6           OP + PR            DIR + GFC     OP + PR + EOP  DIR + GFC + EOP   \n",
       "7   OP + GSR + CEOP            GFC + EOP   DIR + GSR + EOP   OP + GSR + ROC   \n",
       "8         OP + CEOP            DIR + EOP         GSR + EOP       GFC + CEOP   \n",
       "9    OP + PR + CEOP      DIR + GSR + EOP          OP + GFC         PR + ROC   \n",
       "50         PR + EOP  Logistic Regression        GFC + CEOP         OP + EGR   \n",
       "51   DIR + PR + ROC           GSR + CEOP  DIR + EGR + CEOP          OP + PR   \n",
       "52  DIR + EGR + EOP                   OP        GSR + CEOP   OP + PR + CEOP   \n",
       "53        DIR + EOP            PR + CEOP  DIR + GFC + CEOP   OP + EGR + ROC   \n",
       "54  DIR + GFC + EOP           GFC + CEOP        EGR + CEOP         OP + GSR   \n",
       "55              EOP     DIR + GSR + CEOP  DIR + GSR + CEOP  OP + GSR + CEOP   \n",
       "56   OP + GSR + EOP      DIR + PR + CEOP   DIR + PR + CEOP   OP + GFC + ROC   \n",
       "57         OP + EOP            OP + CEOP         PR + CEOP  OP + EGR + CEOP   \n",
       "58    OP + PR + EOP           DIR + CEOP        DIR + CEOP         OP + GFC   \n",
       "59   OP + GFC + EOP                 CEOP              CEOP  OP + GFC + CEOP   \n",
       "\n",
       "            FOR Diff             FDR Diff Statistical Parity Diff  \\\n",
       "0            OP + PR  Logistic Regression          OP + GFC + EOP   \n",
       "1   DIR + GFC + CEOP                  DIR          OP + EGR + ROC   \n",
       "2               CEOP           GSR + CEOP                OP + EGR   \n",
       "3          PR + CEOP     DIR + EGR + CEOP          OP + GSR + EOP   \n",
       "4         DIR + CEOP           GFC + CEOP          OP + GFC + ROC   \n",
       "5                 PR                  GFC               GSR + ROC   \n",
       "6    DIR + PR + CEOP            DIR + GFC               DIR + GFC   \n",
       "7     OP + GFC + ROC             DIR + PR                OP + EOP   \n",
       "8   DIR + GSR + CEOP            DIR + EGR               EGR + ROC   \n",
       "9         GFC + CEOP                   PR           OP + PR + EOP   \n",
       "50         GSR + ROC      OP + GFC + CEOP              GFC + CEOP   \n",
       "51         GFC + EOP      DIR + GSR + EOP                      PR   \n",
       "52          OP + ROC             OP + EGR               OP + CEOP   \n",
       "53         OP + CEOP       OP + EGR + ROC              GSR + CEOP   \n",
       "54    OP + GSR + ROC              OP + PR         DIR + PR + CEOP   \n",
       "55     OP + PR + ROC             OP + EOP     Logistic Regression   \n",
       "56    OP + EGR + EOP       OP + EGR + EOP        DIR + GSR + CEOP   \n",
       "57   DIR + GFC + EOP        OP + PR + EOP               PR + CEOP   \n",
       "58    OP + EGR + ROC       OP + GFC + EOP              DIR + CEOP   \n",
       "59          OP + EGR       OP + GSR + EOP                    CEOP   \n",
       "\n",
       "          F1 Score Diff  \n",
       "0             GSR + ROC  \n",
       "1             GFC + EOP  \n",
       "2                   GSR  \n",
       "3             EGR + ROC  \n",
       "4                    PR  \n",
       "5       DIR + GFC + EOP  \n",
       "6                   EOP  \n",
       "7             GFC + ROC  \n",
       "8              DIR + PR  \n",
       "9   Logistic Regression  \n",
       "50      OP + GFC + CEOP  \n",
       "51           EGR + CEOP  \n",
       "52     DIR + GFC + CEOP  \n",
       "53      OP + EGR + CEOP  \n",
       "54           GSR + CEOP  \n",
       "55     DIR + GSR + CEOP  \n",
       "56      DIR + PR + CEOP  \n",
       "57            PR + CEOP  \n",
       "58           DIR + CEOP  \n",
       "59                 CEOP  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# trimmed version of df_ranked for the paper - top 10 and bottom 10\n",
    "df_rank_trimmed = pandas.concat([df_ranked.head(10), df_ranked.tail(10)])\n",
    "df_rank_trimmed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "paper_tbl = df_rank_trimmed.copy()\n",
    "del paper_tbl[\"F1 Score Diff\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OP + GFC + ROC         5\n",
       "Logistic Regression    4\n",
       "DIR + EGR              4\n",
       "DIR                    4\n",
       "DIR + GFC              4\n",
       "                      ..\n",
       "5                      1\n",
       "DIR + PR + EOP         1\n",
       "ROC                    1\n",
       "6                      1\n",
       "PR + ROC               1\n",
       "Length: 65, dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pandas.value_counts(paper_tbl.loc[0:9].values.flatten())  #.value_counts().to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OP + GFC + EOP         5\n",
       "OP + PR                4\n",
       "DIR + GFC + EOP        4\n",
       "GFC + CEOP             3\n",
       "OP + ROC               3\n",
       "DIR + GSR + CEOP       3\n",
       "DIR + PR + CEOP        3\n",
       "OP + GSR + ROC         3\n",
       "OP + EGR + ROC         3\n",
       "PR + CEOP              3\n",
       "OP + GFC               3\n",
       "DIR + GFC              3\n",
       "OP + CEOP              3\n",
       "DIR + CEOP             3\n",
       "GSR + CEOP             3\n",
       "CEOP                   3\n",
       "GFC + EOP              2\n",
       "OP + GSR + EOP         2\n",
       "OP + PR + EOP          2\n",
       "OP + EGR               2\n",
       "OP + EOP               2\n",
       "DIR + PR + ROC         2\n",
       "GFC + ROC              2\n",
       "DIR + GFC + ROC        2\n",
       "DIR + GFC + CEOP       2\n",
       "OP + EGR + EOP         2\n",
       "OP + PR + ROC          2\n",
       "57                     1\n",
       "DIR + ROC              1\n",
       "OP + GFC + ROC         1\n",
       "59                     1\n",
       "58                     1\n",
       "60                     1\n",
       "OP + EGR + CEOP        1\n",
       "Logistic Regression    1\n",
       "52                     1\n",
       "OP + GSR + CEOP        1\n",
       "EOP                    1\n",
       "56                     1\n",
       "OP + GSR               1\n",
       "EGR + CEOP             1\n",
       "55                     1\n",
       "DIR + EOP              1\n",
       "54                     1\n",
       "OP + PR + CEOP         1\n",
       "OP                     1\n",
       "DIR + EGR + EOP        1\n",
       "53                     1\n",
       "PR                     1\n",
       "DIR + GSR + EOP        1\n",
       "DIR + EGR + CEOP       1\n",
       "OP + GFC + CEOP        1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pandas.value_counts(paper_tbl.loc[51:60].values.flatten())  #.value_counts().to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_rank_trimmed.to_csv(\"./results/R4_rank_trimmed.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>EXTRA</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Theil Index</th>\n",
       "      <th>Consistency</th>\n",
       "      <th>F1 Score</th>\n",
       "      <th>False Positive Rate Diff</th>\n",
       "      <th>False Negative Rate Diff</th>\n",
       "      <th>Accuracy Diff</th>\n",
       "      <th>False Omission Rate Diff</th>\n",
       "      <th>False Discovery Rate Diff</th>\n",
       "      <th>Statistical Parity Diff</th>\n",
       "      <th>F1 Score Diff</th>\n",
       "      <th>Rank Score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Adult</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>RW</th>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>14</td>\n",
       "      <td>36</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + ROC</th>\n",
       "      <td>31</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR</th>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>33</td>\n",
       "      <td>13</td>\n",
       "      <td>38</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>8</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>31</td>\n",
       "      <td>14</td>\n",
       "      <td>36</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>9</td>\n",
       "      <td>149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ROC</th>\n",
       "      <td>31</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>14</td>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + ROC</th>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>30</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR + ROC</th>\n",
       "      <td>35</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "      <td>30</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + ROC</th>\n",
       "      <td>8</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>19</td>\n",
       "      <td>28</td>\n",
       "      <td>15</td>\n",
       "      <td>41</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + ROC</th>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>19</td>\n",
       "      <td>6</td>\n",
       "      <td>29</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>27</td>\n",
       "      <td>177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + ROC</th>\n",
       "      <td>35</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "      <td>30</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>25</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EOP</th>\n",
       "      <td>23</td>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>33</td>\n",
       "      <td>21</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + EOP</th>\n",
       "      <td>20</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>38</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>26</td>\n",
       "      <td>35</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EOP</th>\n",
       "      <td>25</td>\n",
       "      <td>31</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>32</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EOP</th>\n",
       "      <td>27</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>26</td>\n",
       "      <td>31</td>\n",
       "      <td>17</td>\n",
       "      <td>24</td>\n",
       "      <td>191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + ROC</th>\n",
       "      <td>8</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>19</td>\n",
       "      <td>28</td>\n",
       "      <td>15</td>\n",
       "      <td>41</td>\n",
       "      <td>22</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR</th>\n",
       "      <td>10</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>26</td>\n",
       "      <td>39</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + EOP</th>\n",
       "      <td>18</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>27</td>\n",
       "      <td>36</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>11</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + ROC</th>\n",
       "      <td>13</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>28</td>\n",
       "      <td>30</td>\n",
       "      <td>9</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR + ROC</th>\n",
       "      <td>46</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>24</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>27</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR</th>\n",
       "      <td>10</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>26</td>\n",
       "      <td>39</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>21</td>\n",
       "      <td>213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + CEOP</th>\n",
       "      <td>4</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>34</td>\n",
       "      <td>40</td>\n",
       "      <td>18</td>\n",
       "      <td>8</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>46</td>\n",
       "      <td>219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR</th>\n",
       "      <td>16</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>22</td>\n",
       "      <td>31</td>\n",
       "      <td>22</td>\n",
       "      <td>44</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + ROC</th>\n",
       "      <td>46</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>24</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>27</td>\n",
       "      <td>12</td>\n",
       "      <td>17</td>\n",
       "      <td>31</td>\n",
       "      <td>223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + EOP</th>\n",
       "      <td>26</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "      <td>34</td>\n",
       "      <td>28</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR</th>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>36</td>\n",
       "      <td>25</td>\n",
       "      <td>41</td>\n",
       "      <td>21</td>\n",
       "      <td>6</td>\n",
       "      <td>17</td>\n",
       "      <td>23</td>\n",
       "      <td>235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CEOP</th>\n",
       "      <td>4</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>34</td>\n",
       "      <td>40</td>\n",
       "      <td>18</td>\n",
       "      <td>8</td>\n",
       "      <td>19</td>\n",
       "      <td>17</td>\n",
       "      <td>46</td>\n",
       "      <td>235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + ROC</th>\n",
       "      <td>48</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>33</td>\n",
       "      <td>29</td>\n",
       "      <td>16</td>\n",
       "      <td>11</td>\n",
       "      <td>22</td>\n",
       "      <td>24</td>\n",
       "      <td>17</td>\n",
       "      <td>33</td>\n",
       "      <td>238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR</th>\n",
       "      <td>16</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>22</td>\n",
       "      <td>31</td>\n",
       "      <td>22</td>\n",
       "      <td>44</td>\n",
       "      <td>26</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR + EOP</th>\n",
       "      <td>29</td>\n",
       "      <td>43</td>\n",
       "      <td>1</td>\n",
       "      <td>43</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>38</td>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR</th>\n",
       "      <td>19</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>30</td>\n",
       "      <td>33</td>\n",
       "      <td>12</td>\n",
       "      <td>46</td>\n",
       "      <td>39</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + CEOP</th>\n",
       "      <td>6</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>42</td>\n",
       "      <td>42</td>\n",
       "      <td>21</td>\n",
       "      <td>7</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>48</td>\n",
       "      <td>250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR + EOP</th>\n",
       "      <td>45</td>\n",
       "      <td>47</td>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>35</td>\n",
       "      <td>47</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + EOP</th>\n",
       "      <td>28</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>34</td>\n",
       "      <td>39</td>\n",
       "      <td>29</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + EOP</th>\n",
       "      <td>41</td>\n",
       "      <td>48</td>\n",
       "      <td>1</td>\n",
       "      <td>48</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>42</td>\n",
       "      <td>48</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + CEOP</th>\n",
       "      <td>7</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>21</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>15</td>\n",
       "      <td>40</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + EOP</th>\n",
       "      <td>30</td>\n",
       "      <td>44</td>\n",
       "      <td>1</td>\n",
       "      <td>44</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>28</td>\n",
       "      <td>37</td>\n",
       "      <td>33</td>\n",
       "      <td>17</td>\n",
       "      <td>16</td>\n",
       "      <td>271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR</th>\n",
       "      <td>37</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>45</td>\n",
       "      <td>3</td>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + CEOP</th>\n",
       "      <td>15</td>\n",
       "      <td>41</td>\n",
       "      <td>1</td>\n",
       "      <td>41</td>\n",
       "      <td>26</td>\n",
       "      <td>35</td>\n",
       "      <td>24</td>\n",
       "      <td>14</td>\n",
       "      <td>47</td>\n",
       "      <td>1</td>\n",
       "      <td>38</td>\n",
       "      <td>283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR + CEOP</th>\n",
       "      <td>21</td>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>40</td>\n",
       "      <td>38</td>\n",
       "      <td>30</td>\n",
       "      <td>10</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>41</td>\n",
       "      <td>288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR</th>\n",
       "      <td>37</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>45</td>\n",
       "      <td>3</td>\n",
       "      <td>42</td>\n",
       "      <td>17</td>\n",
       "      <td>34</td>\n",
       "      <td>289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR + CEOP</th>\n",
       "      <td>39</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>45</td>\n",
       "      <td>45</td>\n",
       "      <td>43</td>\n",
       "      <td>1</td>\n",
       "      <td>44</td>\n",
       "      <td>1</td>\n",
       "      <td>43</td>\n",
       "      <td>301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + CEOP</th>\n",
       "      <td>14</td>\n",
       "      <td>41</td>\n",
       "      <td>1</td>\n",
       "      <td>41</td>\n",
       "      <td>27</td>\n",
       "      <td>36</td>\n",
       "      <td>25</td>\n",
       "      <td>13</td>\n",
       "      <td>48</td>\n",
       "      <td>17</td>\n",
       "      <td>39</td>\n",
       "      <td>302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + CEOP</th>\n",
       "      <td>21</td>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>40</td>\n",
       "      <td>38</td>\n",
       "      <td>30</td>\n",
       "      <td>10</td>\n",
       "      <td>37</td>\n",
       "      <td>17</td>\n",
       "      <td>41</td>\n",
       "      <td>304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + EOP</th>\n",
       "      <td>44</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "      <td>46</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>33</td>\n",
       "      <td>43</td>\n",
       "      <td>32</td>\n",
       "      <td>17</td>\n",
       "      <td>20</td>\n",
       "      <td>306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR</th>\n",
       "      <td>42</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>47</td>\n",
       "      <td>47</td>\n",
       "      <td>48</td>\n",
       "      <td>5</td>\n",
       "      <td>41</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + CEOP</th>\n",
       "      <td>24</td>\n",
       "      <td>39</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>39</td>\n",
       "      <td>37</td>\n",
       "      <td>32</td>\n",
       "      <td>12</td>\n",
       "      <td>36</td>\n",
       "      <td>17</td>\n",
       "      <td>40</td>\n",
       "      <td>316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + CEOP</th>\n",
       "      <td>39</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>45</td>\n",
       "      <td>45</td>\n",
       "      <td>43</td>\n",
       "      <td>1</td>\n",
       "      <td>44</td>\n",
       "      <td>17</td>\n",
       "      <td>43</td>\n",
       "      <td>317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + CEOP</th>\n",
       "      <td>43</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>47</td>\n",
       "      <td>6</td>\n",
       "      <td>46</td>\n",
       "      <td>17</td>\n",
       "      <td>45</td>\n",
       "      <td>327</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Accuracy  Theil Index  Consistency  F1 Score  \\\n",
       "Adult                                                               \n",
       "RW                          1           17            1         2   \n",
       "RW + ROC                   31            2            1         5   \n",
       "DIR                         3           14            1         1   \n",
       "Logistic Regression         1           17            1         2   \n",
       "ROC                        31            2            1         5   \n",
       "DIR + ROC                  33            1            1         4   \n",
       "RW + PR + ROC              35            8            1         9   \n",
       "RW + EGR + ROC              8           22            1        15   \n",
       "DIR + PR + ROC             34            7            1         8   \n",
       "PR + ROC                   35            8            1         9   \n",
       "EOP                        23           30            1        29   \n",
       "RW + EGR + EOP             20           38            1        38   \n",
       "RW + EOP                   25           31            1        30   \n",
       "DIR + EOP                  27           20            1        14   \n",
       "EGR + ROC                   8           22            1        15   \n",
       "RW + PR                    10           24            1        17   \n",
       "EGR + EOP                  18           37            1        34   \n",
       "DIR + EGR + ROC            13           19            1         7   \n",
       "RW + GSR + ROC             46           10            1        31   \n",
       "PR                         10           24            1        17   \n",
       "RW + CEOP                   4           27            1        21   \n",
       "RW + EGR                   16           32            1        27   \n",
       "GSR + ROC                  46           10            1        31   \n",
       "DIR + EGR + EOP            26           36            1        37   \n",
       "DIR + PR                   12           27            1        26   \n",
       "CEOP                        4           27            1        21   \n",
       "DIR + GSR + ROC            48            4            1        33   \n",
       "EGR                        16           32            1        27   \n",
       "RW + PR + EOP              29           43            1        43   \n",
       "DIR + EGR                  19           26            1        23   \n",
       "DIR + CEOP                  6           21            1        11   \n",
       "RW + GSR + EOP             45           47            1        47   \n",
       "DIR + PR + EOP             28           45            1        45   \n",
       "DIR + GSR + EOP            41           48            1        48   \n",
       "DIR + EGR + CEOP            7           40            1        40   \n",
       "PR + EOP                   30           44            1        44   \n",
       "RW + GSR                   37           12            1        12   \n",
       "RW + EGR + CEOP            15           41            1        41   \n",
       "RW + PR + CEOP             21           34            1        35   \n",
       "GSR                        37           12            1        12   \n",
       "RW + GSR + CEOP            39           15            1        24   \n",
       "EGR + CEOP                 14           41            1        41   \n",
       "PR + CEOP                  21           34            1        35   \n",
       "GSR + EOP                  44           46            1        46   \n",
       "DIR + GSR                  42            5            1        19   \n",
       "DIR + PR + CEOP            24           39            1        39   \n",
       "GSR + CEOP                 39           15            1        24   \n",
       "DIR + GSR + CEOP           43            6            1        20   \n",
       "\n",
       "                     False Positive Rate Diff  False Negative Rate Diff  \\\n",
       "Adult                                                                     \n",
       "RW                                         31                        14   \n",
       "RW + ROC                                   14                        22   \n",
       "DIR                                        33                        13   \n",
       "Logistic Regression                        31                        14   \n",
       "ROC                                        14                        22   \n",
       "DIR + ROC                                  16                        24   \n",
       "RW + PR + ROC                              17                        20   \n",
       "RW + EGR + ROC                             19                        28   \n",
       "DIR + PR + ROC                             13                        19   \n",
       "PR + ROC                                   17                        20   \n",
       "EOP                                         1                         1   \n",
       "RW + EGR + EOP                              3                         4   \n",
       "RW + EOP                                    6                         7   \n",
       "DIR + EOP                                  11                        10   \n",
       "EGR + ROC                                  19                        28   \n",
       "RW + PR                                    37                        26   \n",
       "EGR + EOP                                   4                         8   \n",
       "DIR + EGR + ROC                            28                        30   \n",
       "RW + GSR + ROC                             24                        17   \n",
       "PR                                         37                        26   \n",
       "RW + CEOP                                  34                        40   \n",
       "RW + EGR                                   22                        31   \n",
       "GSR + ROC                                  24                        17   \n",
       "DIR + EGR + EOP                             6                         9   \n",
       "DIR + PR                                   36                        25   \n",
       "CEOP                                       34                        40   \n",
       "DIR + GSR + ROC                            29                        16   \n",
       "EGR                                        22                        31   \n",
       "RW + PR + EOP                               9                         2   \n",
       "DIR + EGR                                  30                        33   \n",
       "DIR + CEOP                                 42                        42   \n",
       "RW + GSR + EOP                              5                         5   \n",
       "DIR + PR + EOP                              8                         3   \n",
       "DIR + GSR + EOP                             1                         6   \n",
       "DIR + EGR + CEOP                           21                        34   \n",
       "PR + EOP                                   10                        11   \n",
       "RW + GSR                                   43                        43   \n",
       "RW + EGR + CEOP                            26                        35   \n",
       "RW + PR + CEOP                             40                        38   \n",
       "GSR                                        43                        43   \n",
       "RW + GSR + CEOP                            45                        45   \n",
       "EGR + CEOP                                 27                        36   \n",
       "PR + CEOP                                  40                        38   \n",
       "GSR + EOP                                  12                        12   \n",
       "DIR + GSR                                  47                        47   \n",
       "DIR + PR + CEOP                            39                        37   \n",
       "GSR + CEOP                                 45                        45   \n",
       "DIR + GSR + CEOP                           48                        48   \n",
       "\n",
       "                     Accuracy Diff  False Omission Rate Diff  \\\n",
       "Adult                                                          \n",
       "RW                              36                        19   \n",
       "RW + ROC                         1                        24   \n",
       "DIR                             38                        16   \n",
       "Logistic Regression             36                        19   \n",
       "ROC                              1                        24   \n",
       "DIR + ROC                        3                        23   \n",
       "RW + PR + ROC                    4                        30   \n",
       "RW + EGR + ROC                  15                        41   \n",
       "DIR + PR + ROC                   6                        29   \n",
       "PR + ROC                         4                        30   \n",
       "EOP                             14                        33   \n",
       "RW + EGR + EOP                  26                        35   \n",
       "RW + EOP                        13                        32   \n",
       "DIR + EOP                       10                        26   \n",
       "EGR + ROC                       15                        41   \n",
       "RW + PR                         39                        17   \n",
       "EGR + EOP                       27                        36   \n",
       "DIR + EGR + ROC                  9                        40   \n",
       "RW + GSR + ROC                   7                        27   \n",
       "PR                              39                        17   \n",
       "RW + CEOP                       18                         8   \n",
       "RW + EGR                        22                        44   \n",
       "GSR + ROC                        7                        27   \n",
       "DIR + EGR + EOP                 20                        34   \n",
       "DIR + PR                        41                        21   \n",
       "CEOP                            18                         8   \n",
       "DIR + GSR + ROC                 11                        22   \n",
       "EGR                             22                        44   \n",
       "RW + PR + EOP                   29                        38   \n",
       "DIR + EGR                       12                        46   \n",
       "DIR + CEOP                      21                         7   \n",
       "RW + GSR + EOP                  35                        47   \n",
       "DIR + PR + EOP                  34                        39   \n",
       "DIR + GSR + EOP                 42                        48   \n",
       "DIR + EGR + CEOP                17                        15   \n",
       "PR + EOP                        28                        37   \n",
       "RW + GSR                        45                         3   \n",
       "RW + EGR + CEOP                 24                        14   \n",
       "RW + PR + CEOP                  30                        10   \n",
       "GSR                             45                         3   \n",
       "RW + GSR + CEOP                 43                         1   \n",
       "EGR + CEOP                      25                        13   \n",
       "PR + CEOP                       30                        10   \n",
       "GSR + EOP                       33                        43   \n",
       "DIR + GSR                       48                         5   \n",
       "DIR + PR + CEOP                 32                        12   \n",
       "GSR + CEOP                      43                         1   \n",
       "DIR + GSR + CEOP                47                         6   \n",
       "\n",
       "                     False Discovery Rate Diff  Statistical Parity Diff  \\\n",
       "Adult                                                                     \n",
       "RW                                           2                        1   \n",
       "RW + ROC                                    14                        1   \n",
       "DIR                                          1                       17   \n",
       "Logistic Regression                          2                       17   \n",
       "ROC                                         14                       17   \n",
       "DIR + ROC                                   11                       17   \n",
       "RW + PR + ROC                               17                        1   \n",
       "RW + EGR + ROC                              22                        1   \n",
       "DIR + PR + ROC                              16                       17   \n",
       "PR + ROC                                    17                       17   \n",
       "EOP                                         21                       17   \n",
       "RW + EGR + EOP                              10                        1   \n",
       "RW + EOP                                    25                        1   \n",
       "DIR + EOP                                   31                       17   \n",
       "EGR + ROC                                   22                       17   \n",
       "RW + PR                                      4                        1   \n",
       "EGR + EOP                                    7                       17   \n",
       "DIR + EGR + ROC                             35                       17   \n",
       "RW + GSR + ROC                              12                        1   \n",
       "PR                                           4                       17   \n",
       "RW + CEOP                                   19                        1   \n",
       "RW + EGR                                    26                        1   \n",
       "GSR + ROC                                   12                       17   \n",
       "DIR + EGR + EOP                             28                       17   \n",
       "DIR + PR                                     6                       17   \n",
       "CEOP                                        19                       17   \n",
       "DIR + GSR + ROC                             24                       17   \n",
       "EGR                                         26                       17   \n",
       "RW + PR + EOP                               30                        1   \n",
       "DIR + EGR                                   39                       17   \n",
       "DIR + CEOP                                  34                       17   \n",
       "RW + GSR + EOP                               8                        1   \n",
       "DIR + PR + EOP                              29                       17   \n",
       "DIR + GSR + EOP                              9                       17   \n",
       "DIR + EGR + CEOP                            40                       17   \n",
       "PR + EOP                                    33                       17   \n",
       "RW + GSR                                    42                        1   \n",
       "RW + EGR + CEOP                             47                        1   \n",
       "RW + PR + CEOP                              37                        1   \n",
       "GSR                                         42                       17   \n",
       "RW + GSR + CEOP                             44                        1   \n",
       "EGR + CEOP                                  48                       17   \n",
       "PR + CEOP                                   37                       17   \n",
       "GSR + EOP                                   32                       17   \n",
       "DIR + GSR                                   41                       17   \n",
       "DIR + PR + CEOP                             36                       17   \n",
       "GSR + CEOP                                  44                       17   \n",
       "DIR + GSR + CEOP                            46                       17   \n",
       "\n",
       "                     F1 Score Diff  Rank Score  \n",
       "Adult                                           \n",
       "RW                               9         133  \n",
       "RW + ROC                        28         143  \n",
       "DIR                              8         145  \n",
       "Logistic Regression              9         149  \n",
       "ROC                             28         159  \n",
       "DIR + ROC                       30         163  \n",
       "RW + PR + ROC                   25         167  \n",
       "RW + EGR + ROC                   4         176  \n",
       "DIR + PR + ROC                  27         177  \n",
       "PR + ROC                        25         183  \n",
       "EOP                             17         187  \n",
       "RW + EGR + EOP                  14         190  \n",
       "RW + EOP                        19         190  \n",
       "DIR + EOP                       24         191  \n",
       "EGR + ROC                        4         192  \n",
       "RW + PR                         21         197  \n",
       "EGR + EOP                       11         200  \n",
       "DIR + EGR + ROC                  7         206  \n",
       "RW + GSR + ROC                  31         207  \n",
       "PR                              21         213  \n",
       "RW + CEOP                       46         219  \n",
       "RW + EGR                         1         223  \n",
       "GSR + ROC                       31         223  \n",
       "DIR + EGR + EOP                 18         232  \n",
       "DIR + PR                        23         235  \n",
       "CEOP                            46         235  \n",
       "DIR + GSR + ROC                 33         238  \n",
       "EGR                              1         239  \n",
       "RW + PR + EOP                   15         240  \n",
       "DIR + EGR                        3         249  \n",
       "DIR + CEOP                      48         250  \n",
       "RW + GSR + EOP                  12         253  \n",
       "DIR + PR + EOP                  13         262  \n",
       "DIR + GSR + EOP                  6         267  \n",
       "DIR + EGR + CEOP                36         268  \n",
       "PR + EOP                        16         271  \n",
       "RW + GSR                        34         273  \n",
       "RW + EGR + CEOP                 38         283  \n",
       "RW + PR + CEOP                  41         288  \n",
       "GSR                             34         289  \n",
       "RW + GSR + CEOP                 43         301  \n",
       "EGR + CEOP                      39         302  \n",
       "PR + CEOP                       41         304  \n",
       "GSR + EOP                       20         306  \n",
       "DIR + GSR                       37         309  \n",
       "DIR + PR + CEOP                 40         316  \n",
       "GSR + CEOP                      43         317  \n",
       "DIR + GSR + CEOP                45         327  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult_ranking = pandas.DataFrame(data=adult_df.index, columns=['Adult'])\n",
    "for metric in adult_df.columns:\n",
    "    if metric in [\"Accuracy\", \"F1 Score\"]:\n",
    "        adult_ranking[metric] = adult_df[metric].rank(ascending=False, method=\"min\").astype(int)\n",
    "    else:\n",
    "        adult_ranking[metric] = adult_df[metric].rank(ascending=True, method=\"min\").astype(int)\n",
    "adult_ranking = adult_ranking.set_index(\"Adult\")\n",
    "rank_scores = []\n",
    "for intervention in adult_df[\"Adult\"]:\n",
    "    rank_scores.append(sum(adult_ranking.loc[intervention]))\n",
    "adult_ranking[\"Rank Score\"] = rank_scores\n",
    "adult_ranking.sort_values(\"Rank Score\", inplace=True)\n",
    "adult_ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>False Positive Rate Diff</th>\n",
       "      <th>False Negative Rate Diff</th>\n",
       "      <th>Accuracy Diff</th>\n",
       "      <th>False Omission Rate Diff</th>\n",
       "      <th>False Discovery Rate Diff</th>\n",
       "      <th>Statistical Parity Diff</th>\n",
       "      <th>F1 Score Diff</th>\n",
       "      <th>Rank Score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Compas</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>RW + PR + ROC</th>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>20</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + EOP</th>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>33</td>\n",
       "      <td>28</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + ROC</th>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>6</td>\n",
       "      <td>28</td>\n",
       "      <td>23</td>\n",
       "      <td>17</td>\n",
       "      <td>8</td>\n",
       "      <td>113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + EOP</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>22</td>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR + EOP</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>37</td>\n",
       "      <td>34</td>\n",
       "      <td>39</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR + EOP</th>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>31</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EOP</th>\n",
       "      <td>12</td>\n",
       "      <td>14</td>\n",
       "      <td>9</td>\n",
       "      <td>39</td>\n",
       "      <td>26</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + ROC</th>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>20</td>\n",
       "      <td>37</td>\n",
       "      <td>17</td>\n",
       "      <td>10</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR</th>\n",
       "      <td>24</td>\n",
       "      <td>19</td>\n",
       "      <td>17</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + CEOP</th>\n",
       "      <td>29</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + EOP</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>29</td>\n",
       "      <td>17</td>\n",
       "      <td>12</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EOP</th>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>34</td>\n",
       "      <td>26</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR + CEOP</th>\n",
       "      <td>39</td>\n",
       "      <td>37</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>29</td>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + CEOP</th>\n",
       "      <td>40</td>\n",
       "      <td>38</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + EOP</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>37</td>\n",
       "      <td>34</td>\n",
       "      <td>39</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + ROC</th>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR</th>\n",
       "      <td>24</td>\n",
       "      <td>19</td>\n",
       "      <td>17</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + EOP</th>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>23</td>\n",
       "      <td>33</td>\n",
       "      <td>17</td>\n",
       "      <td>20</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + CEOP</th>\n",
       "      <td>30</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + ROC</th>\n",
       "      <td>21</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>18</td>\n",
       "      <td>41</td>\n",
       "      <td>17</td>\n",
       "      <td>15</td>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CEOP</th>\n",
       "      <td>42</td>\n",
       "      <td>39</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>32</td>\n",
       "      <td>143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>22</td>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>33</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + CEOP</th>\n",
       "      <td>28</td>\n",
       "      <td>27</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>17</td>\n",
       "      <td>27</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR</th>\n",
       "      <td>36</td>\n",
       "      <td>35</td>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + ROC</th>\n",
       "      <td>19</td>\n",
       "      <td>17</td>\n",
       "      <td>27</td>\n",
       "      <td>16</td>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + CEOP</th>\n",
       "      <td>43</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>31</td>\n",
       "      <td>147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EOP</th>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>35</td>\n",
       "      <td>27</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>22</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + ROC</th>\n",
       "      <td>27</td>\n",
       "      <td>11</td>\n",
       "      <td>21</td>\n",
       "      <td>19</td>\n",
       "      <td>45</td>\n",
       "      <td>17</td>\n",
       "      <td>9</td>\n",
       "      <td>149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + CEOP</th>\n",
       "      <td>41</td>\n",
       "      <td>40</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + ROC</th>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>35</td>\n",
       "      <td>17</td>\n",
       "      <td>5</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + EOP</th>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>42</td>\n",
       "      <td>38</td>\n",
       "      <td>18</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR</th>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + CEOP</th>\n",
       "      <td>37</td>\n",
       "      <td>42</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>40</td>\n",
       "      <td>157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR</th>\n",
       "      <td>26</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>14</td>\n",
       "      <td>27</td>\n",
       "      <td>17</td>\n",
       "      <td>26</td>\n",
       "      <td>158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR</th>\n",
       "      <td>35</td>\n",
       "      <td>36</td>\n",
       "      <td>13</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>35</td>\n",
       "      <td>158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>33</td>\n",
       "      <td>33</td>\n",
       "      <td>22</td>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "      <td>17</td>\n",
       "      <td>33</td>\n",
       "      <td>160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ROC</th>\n",
       "      <td>19</td>\n",
       "      <td>17</td>\n",
       "      <td>27</td>\n",
       "      <td>16</td>\n",
       "      <td>42</td>\n",
       "      <td>17</td>\n",
       "      <td>23</td>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + EOP</th>\n",
       "      <td>16</td>\n",
       "      <td>15</td>\n",
       "      <td>36</td>\n",
       "      <td>15</td>\n",
       "      <td>44</td>\n",
       "      <td>17</td>\n",
       "      <td>25</td>\n",
       "      <td>168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR</th>\n",
       "      <td>31</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR + ROC</th>\n",
       "      <td>9</td>\n",
       "      <td>29</td>\n",
       "      <td>44</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>41</td>\n",
       "      <td>172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + ROC</th>\n",
       "      <td>9</td>\n",
       "      <td>29</td>\n",
       "      <td>44</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>41</td>\n",
       "      <td>188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + ROC</th>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>43</td>\n",
       "      <td>40</td>\n",
       "      <td>14</td>\n",
       "      <td>17</td>\n",
       "      <td>39</td>\n",
       "      <td>198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR</th>\n",
       "      <td>38</td>\n",
       "      <td>43</td>\n",
       "      <td>48</td>\n",
       "      <td>43</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>46</td>\n",
       "      <td>236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR</th>\n",
       "      <td>44</td>\n",
       "      <td>44</td>\n",
       "      <td>46</td>\n",
       "      <td>44</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "      <td>237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR</th>\n",
       "      <td>44</td>\n",
       "      <td>44</td>\n",
       "      <td>46</td>\n",
       "      <td>44</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>47</td>\n",
       "      <td>253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR + CEOP</th>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>39</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "      <td>43</td>\n",
       "      <td>267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + CEOP</th>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>39</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>17</td>\n",
       "      <td>43</td>\n",
       "      <td>283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + CEOP</th>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>39</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>17</td>\n",
       "      <td>43</td>\n",
       "      <td>283</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     False Positive Rate Diff  False Negative Rate Diff  \\\n",
       "Compas                                                                    \n",
       "RW + PR + ROC                              13                        11   \n",
       "DIR + EGR + EOP                            11                         1   \n",
       "DIR + ROC                                  15                        16   \n",
       "RW + EGR + EOP                              1                        10   \n",
       "RW + GSR + EOP                              3                         2   \n",
       "RW + PR + EOP                               2                         8   \n",
       "DIR + EOP                                  12                        14   \n",
       "PR + ROC                                   13                        11   \n",
       "RW + EGR                                   24                        19   \n",
       "RW + EGR + CEOP                            29                        25   \n",
       "EGR + EOP                                   5                         4   \n",
       "RW + EOP                                    8                         5   \n",
       "RW + PR + CEOP                             39                        37   \n",
       "RW + CEOP                                  40                        38   \n",
       "DIR + GSR + EOP                             3                         2   \n",
       "RW + EGR + ROC                             22                        22   \n",
       "EGR                                        24                        19   \n",
       "PR + EOP                                    7                         9   \n",
       "EGR + CEOP                                 30                        26   \n",
       "DIR + EGR + ROC                            21                         6   \n",
       "CEOP                                       42                        39   \n",
       "RW                                         33                        33   \n",
       "DIR + EGR + CEOP                           28                        27   \n",
       "DIR                                        36                        35   \n",
       "RW + ROC                                   19                        17   \n",
       "DIR + CEOP                                 43                        41   \n",
       "EOP                                         6                         7   \n",
       "DIR + PR + ROC                             27                        11   \n",
       "PR + CEOP                                  41                        40   \n",
       "EGR + ROC                                  22                        22   \n",
       "GSR + EOP                                  18                        21   \n",
       "RW + PR                                    31                        31   \n",
       "DIR + PR + CEOP                            37                        42   \n",
       "DIR + EGR                                  26                        24   \n",
       "DIR + PR                                   35                        36   \n",
       "Logistic Regression                        33                        33   \n",
       "ROC                                        19                        17   \n",
       "DIR + PR + EOP                             16                        15   \n",
       "PR                                         31                        31   \n",
       "RW + GSR + ROC                              9                        29   \n",
       "GSR + ROC                                   9                        29   \n",
       "DIR + GSR + ROC                            17                        28   \n",
       "DIR + GSR                                  38                        43   \n",
       "RW + GSR                                   44                        44   \n",
       "GSR                                        44                        44   \n",
       "RW + GSR + CEOP                            46                        46   \n",
       "GSR + CEOP                                 46                        46   \n",
       "DIR + GSR + CEOP                           46                        46   \n",
       "\n",
       "                     Accuracy Diff  False Omission Rate Diff  \\\n",
       "Compas                                                         \n",
       "RW + PR + ROC                   14                        20   \n",
       "DIR + EGR + EOP                 11                        33   \n",
       "DIR + ROC                        6                        28   \n",
       "RW + EGR + EOP                  31                        22   \n",
       "RW + GSR + EOP                  37                        34   \n",
       "RW + PR + EOP                   29                        29   \n",
       "DIR + EOP                        9                        39   \n",
       "PR + ROC                        14                        20   \n",
       "RW + EGR                        17                        24   \n",
       "RW + EGR + CEOP                  1                        32   \n",
       "EGR + EOP                       26                        30   \n",
       "RW + EOP                        34                        26   \n",
       "RW + PR + CEOP                   3                        12   \n",
       "RW + CEOP                        4                        10   \n",
       "DIR + GSR + EOP                 37                        34   \n",
       "RW + EGR + ROC                  17                        36   \n",
       "EGR                             17                        24   \n",
       "PR + EOP                        30                        23   \n",
       "EGR + CEOP                       1                        31   \n",
       "DIR + EGR + ROC                 24                        18   \n",
       "CEOP                             4                         5   \n",
       "RW                              22                         7   \n",
       "DIR + EGR + CEOP                12                        13   \n",
       "DIR                              8                        11   \n",
       "RW + ROC                        27                        16   \n",
       "DIR + CEOP                       7                         6   \n",
       "EOP                             35                        27   \n",
       "DIR + PR + ROC                  21                        19   \n",
       "PR + CEOP                       10                         1   \n",
       "EGR + ROC                       17                        36   \n",
       "GSR + EOP                       42                        38   \n",
       "RW + PR                         32                         3   \n",
       "DIR + PR + CEOP                 16                         2   \n",
       "DIR + EGR                       24                        14   \n",
       "DIR + PR                        13                         9   \n",
       "Logistic Regression             22                         7   \n",
       "ROC                             27                        16   \n",
       "DIR + PR + EOP                  36                        15   \n",
       "PR                              32                         3   \n",
       "RW + GSR + ROC                  44                        41   \n",
       "GSR + ROC                       44                        41   \n",
       "DIR + GSR + ROC                 43                        40   \n",
       "DIR + GSR                       48                        43   \n",
       "RW + GSR                        46                        44   \n",
       "GSR                             46                        44   \n",
       "RW + GSR + CEOP                 39                        46   \n",
       "GSR + CEOP                      39                        46   \n",
       "DIR + GSR + CEOP                39                        46   \n",
       "\n",
       "                     False Discovery Rate Diff  Statistical Parity Diff  \\\n",
       "Compas                                                                    \n",
       "RW + PR + ROC                               37                        1   \n",
       "DIR + EGR + EOP                             28                       17   \n",
       "DIR + ROC                                   23                       17   \n",
       "RW + EGR + EOP                              30                        1   \n",
       "RW + GSR + EOP                              39                        1   \n",
       "RW + PR + EOP                               31                        1   \n",
       "DIR + EOP                                   26                       17   \n",
       "PR + ROC                                    37                       17   \n",
       "RW + EGR                                    24                        1   \n",
       "RW + EGR + CEOP                             19                        1   \n",
       "EGR + EOP                                   29                       17   \n",
       "RW + EOP                                    32                        1   \n",
       "RW + PR + CEOP                               6                        1   \n",
       "RW + CEOP                                    5                        1   \n",
       "DIR + GSR + EOP                             39                       17   \n",
       "RW + EGR + ROC                              35                        1   \n",
       "EGR                                         24                       17   \n",
       "PR + EOP                                    33                       17   \n",
       "EGR + CEOP                                  17                       17   \n",
       "DIR + EGR + ROC                             41                       17   \n",
       "CEOP                                         4                       17   \n",
       "RW                                          15                        1   \n",
       "DIR + EGR + CEOP                            20                       17   \n",
       "DIR                                         10                       17   \n",
       "RW + ROC                                    42                        1   \n",
       "DIR + CEOP                                   2                       17   \n",
       "EOP                                         34                       17   \n",
       "DIR + PR + ROC                              45                       17   \n",
       "PR + CEOP                                    9                       17   \n",
       "EGR + ROC                                   35                       17   \n",
       "GSR + EOP                                   18                       17   \n",
       "RW + PR                                     21                        1   \n",
       "DIR + PR + CEOP                              3                       17   \n",
       "DIR + EGR                                   27                       17   \n",
       "DIR + PR                                    13                       17   \n",
       "Logistic Regression                         15                       17   \n",
       "ROC                                         42                       17   \n",
       "DIR + PR + EOP                              44                       17   \n",
       "PR                                          21                       17   \n",
       "RW + GSR + ROC                               7                        1   \n",
       "GSR + ROC                                    7                       17   \n",
       "DIR + GSR + ROC                             14                       17   \n",
       "DIR + GSR                                    1                       17   \n",
       "RW + GSR                                    11                        1   \n",
       "GSR                                         11                       17   \n",
       "RW + GSR + CEOP                             46                        1   \n",
       "GSR + CEOP                                  46                       17   \n",
       "DIR + GSR + CEOP                            46                       17   \n",
       "\n",
       "                     F1 Score Diff  Rank Score  \n",
       "Compas                                          \n",
       "RW + PR + ROC                   10         106  \n",
       "DIR + EGR + EOP                  7         108  \n",
       "DIR + ROC                        8         113  \n",
       "RW + EGR + EOP                  19         114  \n",
       "RW + GSR + EOP                   2         118  \n",
       "RW + PR + EOP                   18         118  \n",
       "DIR + EOP                        4         121  \n",
       "PR + ROC                        10         122  \n",
       "RW + EGR                        13         122  \n",
       "RW + EGR + CEOP                 16         123  \n",
       "EGR + EOP                       12         123  \n",
       "RW + EOP                        21         127  \n",
       "RW + PR + CEOP                  29         127  \n",
       "RW + CEOP                       30         128  \n",
       "DIR + GSR + EOP                  2         134  \n",
       "RW + EGR + ROC                   5         138  \n",
       "EGR                             13         138  \n",
       "PR + EOP                        20         139  \n",
       "EGR + CEOP                      17         139  \n",
       "DIR + EGR + ROC                 15         142  \n",
       "CEOP                            32         143  \n",
       "RW                              33         144  \n",
       "DIR + EGR + CEOP                27         144  \n",
       "DIR                             28         145  \n",
       "RW + ROC                        23         145  \n",
       "DIR + CEOP                      31         147  \n",
       "EOP                             22         148  \n",
       "DIR + PR + ROC                   9         149  \n",
       "PR + CEOP                       36         154  \n",
       "EGR + ROC                        5         154  \n",
       "GSR + EOP                        1         155  \n",
       "RW + PR                         37         156  \n",
       "DIR + PR + CEOP                 40         157  \n",
       "DIR + EGR                       26         158  \n",
       "DIR + PR                        35         158  \n",
       "Logistic Regression             33         160  \n",
       "ROC                             23         161  \n",
       "DIR + PR + EOP                  25         168  \n",
       "PR                              37         172  \n",
       "RW + GSR + ROC                  41         172  \n",
       "GSR + ROC                       41         188  \n",
       "DIR + GSR + ROC                 39         198  \n",
       "DIR + GSR                       46         236  \n",
       "RW + GSR                        47         237  \n",
       "GSR                             47         253  \n",
       "RW + GSR + CEOP                 43         267  \n",
       "GSR + CEOP                      43         283  \n",
       "DIR + GSR + CEOP                43         283  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compas_ranking = pandas.DataFrame(data=compas_df['Compas'], columns=['Compas'])\n",
    "for metric in compas_df.columns:\n",
    "    if metric == \"Compas\":\n",
    "        continue\n",
    "    if metric not in {\"Accuracy\", \"F1 Score\", \"Consistency\", \"Theil Index\"}:\n",
    "        compas_ranking[metric] = compas_df[metric].rank(ascending=True, method=\"min\").astype(int)\n",
    "compas_ranking = compas_ranking.set_index(\"Compas\")\n",
    "rank_scores = []\n",
    "for intervention in compas_df[\"Compas\"]:\n",
    "    rank_scores.append(sum(compas_ranking.loc[intervention]))\n",
    "compas_ranking[\"Rank Score\"] = rank_scores\n",
    "compas_ranking.sort_values(\"Rank Score\", inplace=True)\n",
    "compas_ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>False Positive Rate Diff</th>\n",
       "      <th>False Negative Rate Diff</th>\n",
       "      <th>Accuracy Diff</th>\n",
       "      <th>False Omission Rate Diff</th>\n",
       "      <th>False Discovery Rate Diff</th>\n",
       "      <th>Statistical Parity Diff</th>\n",
       "      <th>F1 Score Diff</th>\n",
       "      <th>Rank Score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>German</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>RW + GSR + EOP</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EOP</th>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>24</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR + EOP</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>23</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + EOP</th>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EOP</th>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>24</td>\n",
       "      <td>14</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + EOP</th>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EOP</th>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>27</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + EOP</th>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>28</td>\n",
       "      <td>17</td>\n",
       "      <td>16</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + CEOP</th>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>10</td>\n",
       "      <td>37</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + EOP</th>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>32</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>9</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + EOP</th>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>17</td>\n",
       "      <td>22</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + CEOP</th>\n",
       "      <td>20</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>16</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + CEOP</th>\n",
       "      <td>22</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>39</td>\n",
       "      <td>12</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + EOP</th>\n",
       "      <td>21</td>\n",
       "      <td>20</td>\n",
       "      <td>18</td>\n",
       "      <td>31</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + ROC</th>\n",
       "      <td>18</td>\n",
       "      <td>27</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR</th>\n",
       "      <td>16</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>26</td>\n",
       "      <td>24</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + EOP</th>\n",
       "      <td>17</td>\n",
       "      <td>14</td>\n",
       "      <td>9</td>\n",
       "      <td>15</td>\n",
       "      <td>33</td>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + ROC</th>\n",
       "      <td>23</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "      <td>21</td>\n",
       "      <td>25</td>\n",
       "      <td>17</td>\n",
       "      <td>14</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + ROC</th>\n",
       "      <td>18</td>\n",
       "      <td>27</td>\n",
       "      <td>24</td>\n",
       "      <td>5</td>\n",
       "      <td>19</td>\n",
       "      <td>17</td>\n",
       "      <td>26</td>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR</th>\n",
       "      <td>27</td>\n",
       "      <td>24</td>\n",
       "      <td>13</td>\n",
       "      <td>35</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR + ROC</th>\n",
       "      <td>33</td>\n",
       "      <td>15</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + ROC</th>\n",
       "      <td>33</td>\n",
       "      <td>15</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + ROC</th>\n",
       "      <td>24</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "      <td>9</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>20</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + ROC</th>\n",
       "      <td>24</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "      <td>9</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>20</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR</th>\n",
       "      <td>31</td>\n",
       "      <td>35</td>\n",
       "      <td>24</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR</th>\n",
       "      <td>31</td>\n",
       "      <td>35</td>\n",
       "      <td>24</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR</th>\n",
       "      <td>27</td>\n",
       "      <td>24</td>\n",
       "      <td>13</td>\n",
       "      <td>35</td>\n",
       "      <td>26</td>\n",
       "      <td>17</td>\n",
       "      <td>10</td>\n",
       "      <td>152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW</th>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>36</td>\n",
       "      <td>11</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR</th>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>36</td>\n",
       "      <td>11</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + ROC</th>\n",
       "      <td>33</td>\n",
       "      <td>15</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>17</td>\n",
       "      <td>22</td>\n",
       "      <td>159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ROC</th>\n",
       "      <td>33</td>\n",
       "      <td>15</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>17</td>\n",
       "      <td>22</td>\n",
       "      <td>159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>36</td>\n",
       "      <td>11</td>\n",
       "      <td>29</td>\n",
       "      <td>17</td>\n",
       "      <td>32</td>\n",
       "      <td>168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR</th>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>36</td>\n",
       "      <td>11</td>\n",
       "      <td>29</td>\n",
       "      <td>17</td>\n",
       "      <td>32</td>\n",
       "      <td>168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + CEOP</th>\n",
       "      <td>43</td>\n",
       "      <td>40</td>\n",
       "      <td>28</td>\n",
       "      <td>42</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR + CEOP</th>\n",
       "      <td>43</td>\n",
       "      <td>40</td>\n",
       "      <td>28</td>\n",
       "      <td>42</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + CEOP</th>\n",
       "      <td>47</td>\n",
       "      <td>40</td>\n",
       "      <td>20</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>30</td>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + CEOP</th>\n",
       "      <td>47</td>\n",
       "      <td>40</td>\n",
       "      <td>20</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>30</td>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CEOP</th>\n",
       "      <td>43</td>\n",
       "      <td>40</td>\n",
       "      <td>28</td>\n",
       "      <td>42</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + CEOP</th>\n",
       "      <td>43</td>\n",
       "      <td>40</td>\n",
       "      <td>28</td>\n",
       "      <td>42</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR</th>\n",
       "      <td>29</td>\n",
       "      <td>37</td>\n",
       "      <td>44</td>\n",
       "      <td>33</td>\n",
       "      <td>43</td>\n",
       "      <td>1</td>\n",
       "      <td>41</td>\n",
       "      <td>228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + ROC</th>\n",
       "      <td>40</td>\n",
       "      <td>26</td>\n",
       "      <td>40</td>\n",
       "      <td>28</td>\n",
       "      <td>45</td>\n",
       "      <td>17</td>\n",
       "      <td>40</td>\n",
       "      <td>236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR + ROC</th>\n",
       "      <td>41</td>\n",
       "      <td>29</td>\n",
       "      <td>46</td>\n",
       "      <td>29</td>\n",
       "      <td>47</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR</th>\n",
       "      <td>29</td>\n",
       "      <td>37</td>\n",
       "      <td>44</td>\n",
       "      <td>33</td>\n",
       "      <td>43</td>\n",
       "      <td>17</td>\n",
       "      <td>41</td>\n",
       "      <td>244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR + CEOP</th>\n",
       "      <td>37</td>\n",
       "      <td>46</td>\n",
       "      <td>41</td>\n",
       "      <td>47</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>43</td>\n",
       "      <td>251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + ROC</th>\n",
       "      <td>41</td>\n",
       "      <td>29</td>\n",
       "      <td>46</td>\n",
       "      <td>29</td>\n",
       "      <td>47</td>\n",
       "      <td>17</td>\n",
       "      <td>45</td>\n",
       "      <td>254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR</th>\n",
       "      <td>26</td>\n",
       "      <td>39</td>\n",
       "      <td>48</td>\n",
       "      <td>38</td>\n",
       "      <td>46</td>\n",
       "      <td>17</td>\n",
       "      <td>48</td>\n",
       "      <td>262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + CEOP</th>\n",
       "      <td>37</td>\n",
       "      <td>46</td>\n",
       "      <td>41</td>\n",
       "      <td>47</td>\n",
       "      <td>36</td>\n",
       "      <td>17</td>\n",
       "      <td>43</td>\n",
       "      <td>267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + CEOP</th>\n",
       "      <td>39</td>\n",
       "      <td>48</td>\n",
       "      <td>43</td>\n",
       "      <td>46</td>\n",
       "      <td>38</td>\n",
       "      <td>17</td>\n",
       "      <td>47</td>\n",
       "      <td>278</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     False Positive Rate Diff  False Negative Rate Diff  \\\n",
       "German                                                                    \n",
       "RW + GSR + EOP                              2                         1   \n",
       "RW + EOP                                    9                         4   \n",
       "RW + PR + EOP                               8                         9   \n",
       "GSR + EOP                                   6                         3   \n",
       "EOP                                         9                         4   \n",
       "RW + EGR + EOP                              4                         7   \n",
       "DIR + EOP                                  11                         6   \n",
       "PR + EOP                                    5                        10   \n",
       "RW + EGR + CEOP                             1                        21   \n",
       "EGR + EOP                                   3                        12   \n",
       "DIR + EGR + EOP                             7                         2   \n",
       "DIR + EGR + CEOP                           20                        11   \n",
       "EGR + CEOP                                 22                         8   \n",
       "DIR + GSR + EOP                            21                        20   \n",
       "RW + EGR + ROC                             18                        27   \n",
       "DIR + EGR                                  16                        13   \n",
       "DIR + PR + EOP                             17                        14   \n",
       "DIR + EGR + ROC                            23                        19   \n",
       "EGR + ROC                                  18                        27   \n",
       "RW + EGR                                   27                        24   \n",
       "RW + GSR + ROC                             33                        15   \n",
       "RW + ROC                                   33                        15   \n",
       "DIR + ROC                                  24                        22   \n",
       "DIR + GSR + ROC                            24                        22   \n",
       "DIR + GSR                                  31                        35   \n",
       "DIR                                        31                        35   \n",
       "EGR                                        27                        24   \n",
       "RW                                         12                        31   \n",
       "RW + GSR                                   12                        31   \n",
       "GSR + ROC                                  33                        15   \n",
       "ROC                                        33                        15   \n",
       "Logistic Regression                        12                        31   \n",
       "GSR                                        12                        31   \n",
       "RW + CEOP                                  43                        40   \n",
       "RW + GSR + CEOP                            43                        40   \n",
       "DIR + CEOP                                 47                        40   \n",
       "DIR + GSR + CEOP                           47                        40   \n",
       "CEOP                                       43                        40   \n",
       "GSR + CEOP                                 43                        40   \n",
       "RW + PR                                    29                        37   \n",
       "DIR + PR + ROC                             40                        26   \n",
       "RW + PR + ROC                              41                        29   \n",
       "PR                                         29                        37   \n",
       "RW + PR + CEOP                             37                        46   \n",
       "PR + ROC                                   41                        29   \n",
       "DIR + PR                                   26                        39   \n",
       "PR + CEOP                                  37                        46   \n",
       "DIR + PR + CEOP                            39                        48   \n",
       "\n",
       "                     Accuracy Diff  False Omission Rate Diff  \\\n",
       "German                                                         \n",
       "RW + GSR + EOP                   2                        19   \n",
       "RW + EOP                         4                        24   \n",
       "RW + PR + EOP                   17                        23   \n",
       "GSR + EOP                        3                        22   \n",
       "EOP                              4                        24   \n",
       "RW + EGR + EOP                  13                        20   \n",
       "DIR + EOP                        7                        27   \n",
       "PR + EOP                         1                        18   \n",
       "RW + EGR + CEOP                 10                        37   \n",
       "EGR + EOP                        8                        32   \n",
       "DIR + EGR + EOP                 18                        17   \n",
       "DIR + EGR + CEOP                12                        16   \n",
       "EGR + CEOP                       6                        39   \n",
       "DIR + GSR + EOP                 18                        31   \n",
       "RW + EGR + ROC                  24                         5   \n",
       "DIR + EGR                       11                        26   \n",
       "DIR + PR + EOP                   9                        15   \n",
       "DIR + EGR + ROC                 13                        21   \n",
       "EGR + ROC                       24                         5   \n",
       "RW + EGR                        13                        35   \n",
       "RW + GSR + ROC                  32                         1   \n",
       "RW + ROC                        32                         1   \n",
       "DIR + ROC                       22                         9   \n",
       "DIR + GSR + ROC                 22                         9   \n",
       "DIR + GSR                       24                         7   \n",
       "DIR                             24                         7   \n",
       "EGR                             13                        35   \n",
       "RW                              36                        11   \n",
       "RW + GSR                        36                        11   \n",
       "GSR + ROC                       32                         1   \n",
       "ROC                             32                         1   \n",
       "Logistic Regression             36                        11   \n",
       "GSR                             36                        11   \n",
       "RW + CEOP                       28                        42   \n",
       "RW + GSR + CEOP                 28                        42   \n",
       "DIR + CEOP                      20                        40   \n",
       "DIR + GSR + CEOP                20                        40   \n",
       "CEOP                            28                        42   \n",
       "GSR + CEOP                      28                        42   \n",
       "RW + PR                         44                        33   \n",
       "DIR + PR + ROC                  40                        28   \n",
       "RW + PR + ROC                   46                        29   \n",
       "PR                              44                        33   \n",
       "RW + PR + CEOP                  41                        47   \n",
       "PR + ROC                        46                        29   \n",
       "DIR + PR                        48                        38   \n",
       "PR + CEOP                       41                        47   \n",
       "DIR + PR + CEOP                 43                        46   \n",
       "\n",
       "                     False Discovery Rate Diff  Statistical Parity Diff  \\\n",
       "German                                                                    \n",
       "RW + GSR + EOP                              23                        1   \n",
       "RW + EOP                                    14                        1   \n",
       "RW + PR + EOP                                8                        1   \n",
       "GSR + EOP                                   17                       17   \n",
       "EOP                                         14                       17   \n",
       "RW + EGR + EOP                              21                        1   \n",
       "DIR + EOP                                   11                       17   \n",
       "PR + EOP                                    28                       17   \n",
       "RW + EGR + CEOP                             18                        1   \n",
       "EGR + EOP                                   16                       17   \n",
       "DIR + EGR + EOP                             22                       17   \n",
       "DIR + EGR + CEOP                            13                       17   \n",
       "EGR + CEOP                                  12                       17   \n",
       "DIR + GSR + EOP                              3                       17   \n",
       "RW + EGR + ROC                              19                        1   \n",
       "DIR + EGR                                   24                       17   \n",
       "DIR + PR + EOP                              33                       17   \n",
       "DIR + EGR + ROC                             25                       17   \n",
       "EGR + ROC                                   19                       17   \n",
       "RW + EGR                                    26                        1   \n",
       "RW + GSR + ROC                              39                        1   \n",
       "RW + ROC                                    39                        1   \n",
       "DIR + ROC                                   34                       17   \n",
       "DIR + GSR + ROC                             34                       17   \n",
       "DIR + GSR                                    9                       17   \n",
       "DIR                                          9                       17   \n",
       "EGR                                         26                       17   \n",
       "RW                                          29                        1   \n",
       "RW + GSR                                    29                        1   \n",
       "GSR + ROC                                   39                       17   \n",
       "ROC                                         39                       17   \n",
       "Logistic Regression                         29                       17   \n",
       "GSR                                         29                       17   \n",
       "RW + CEOP                                    4                        1   \n",
       "RW + GSR + CEOP                              4                        1   \n",
       "DIR + CEOP                                   1                       17   \n",
       "DIR + GSR + CEOP                             1                       17   \n",
       "CEOP                                         4                       17   \n",
       "GSR + CEOP                                   4                       17   \n",
       "RW + PR                                     43                        1   \n",
       "DIR + PR + ROC                              45                       17   \n",
       "RW + PR + ROC                               47                        1   \n",
       "PR                                          43                       17   \n",
       "RW + PR + CEOP                              36                        1   \n",
       "PR + ROC                                    47                       17   \n",
       "DIR + PR                                    46                       17   \n",
       "PR + CEOP                                   36                       17   \n",
       "DIR + PR + CEOP                             38                       17   \n",
       "\n",
       "                     F1 Score Diff  Rank Score  \n",
       "German                                          \n",
       "RW + GSR + EOP                  12          60  \n",
       "RW + EOP                         4          60  \n",
       "RW + PR + EOP                    1          67  \n",
       "GSR + EOP                        6          74  \n",
       "EOP                              4          76  \n",
       "RW + EGR + EOP                  15          81  \n",
       "DIR + EOP                        2          81  \n",
       "PR + EOP                        16          95  \n",
       "RW + EGR + CEOP                  8          96  \n",
       "EGR + EOP                        9          97  \n",
       "DIR + EGR + EOP                 18         101  \n",
       "DIR + EGR + CEOP                17         106  \n",
       "EGR + CEOP                       7         111  \n",
       "DIR + GSR + EOP                  3         113  \n",
       "RW + EGR + ROC                  26         120  \n",
       "DIR + EGR                       13         120  \n",
       "DIR + PR + EOP                  19         124  \n",
       "DIR + EGR + ROC                 14         132  \n",
       "EGR + ROC                       26         136  \n",
       "RW + EGR                        10         136  \n",
       "RW + GSR + ROC                  22         143  \n",
       "RW + ROC                        22         143  \n",
       "DIR + ROC                       20         148  \n",
       "DIR + GSR + ROC                 20         148  \n",
       "DIR + GSR                       28         151  \n",
       "DIR                             28         151  \n",
       "EGR                             10         152  \n",
       "RW                              32         152  \n",
       "RW + GSR                        32         152  \n",
       "GSR + ROC                       22         159  \n",
       "ROC                             22         159  \n",
       "Logistic Regression             32         168  \n",
       "GSR                             32         168  \n",
       "RW + CEOP                       36         194  \n",
       "RW + GSR + CEOP                 36         194  \n",
       "DIR + CEOP                      30         195  \n",
       "DIR + GSR + CEOP                30         195  \n",
       "CEOP                            36         210  \n",
       "GSR + CEOP                      36         210  \n",
       "RW + PR                         41         228  \n",
       "DIR + PR + ROC                  40         236  \n",
       "RW + PR + ROC                   45         238  \n",
       "PR                              41         244  \n",
       "RW + PR + CEOP                  43         251  \n",
       "PR + ROC                        45         254  \n",
       "DIR + PR                        48         262  \n",
       "PR + CEOP                       43         267  \n",
       "DIR + PR + CEOP                 47         278  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "german_ranking = pandas.DataFrame(data=german_df['German'], columns=['German'])\n",
    "for metric in german_df.columns:\n",
    "    if metric == \"German\":\n",
    "        continue\n",
    "    if metric not in {\"Accuracy\", \"F1 Score\", \"Consistency\", \"Theil Index\"}:\n",
    "        german_ranking[metric] = german_df[metric].rank(ascending=True, method=\"min\").astype(int)\n",
    "german_ranking = german_ranking.set_index(\"German\")\n",
    "rank_scores = []\n",
    "for intervention in german_df[\"German\"]:\n",
    "    rank_scores.append(sum(german_ranking.loc[intervention]))\n",
    "german_ranking[\"Rank Score\"] = rank_scores\n",
    "german_ranking.sort_values(\"Rank Score\", inplace=True)\n",
    "german_ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>False Positive Rate Diff</th>\n",
       "      <th>False Negative Rate Diff</th>\n",
       "      <th>Accuracy Diff</th>\n",
       "      <th>False Omission Rate Diff</th>\n",
       "      <th>False Discovery Rate Diff</th>\n",
       "      <th>Statistical Parity Diff</th>\n",
       "      <th>F1 Score Diff</th>\n",
       "      <th>Rank Score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bank</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>RW + PR + EOP</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR + EOP</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>25</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + EOP</th>\n",
       "      <td>10</td>\n",
       "      <td>13</td>\n",
       "      <td>30</td>\n",
       "      <td>28</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + EOP</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>20</td>\n",
       "      <td>32</td>\n",
       "      <td>17</td>\n",
       "      <td>25</td>\n",
       "      <td>113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EOP</th>\n",
       "      <td>24</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>29</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR + ROC</th>\n",
       "      <td>21</td>\n",
       "      <td>6</td>\n",
       "      <td>44</td>\n",
       "      <td>44</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR</th>\n",
       "      <td>43</td>\n",
       "      <td>10</td>\n",
       "      <td>36</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + EOP</th>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>26</td>\n",
       "      <td>24</td>\n",
       "      <td>27</td>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR + CEOP</th>\n",
       "      <td>29</td>\n",
       "      <td>43</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>43</td>\n",
       "      <td>134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + EOP</th>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>14</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EOP</th>\n",
       "      <td>23</td>\n",
       "      <td>14</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>36</td>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR + ROC</th>\n",
       "      <td>35</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW</th>\n",
       "      <td>41</td>\n",
       "      <td>32</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + ROC</th>\n",
       "      <td>21</td>\n",
       "      <td>6</td>\n",
       "      <td>44</td>\n",
       "      <td>44</td>\n",
       "      <td>8</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR</th>\n",
       "      <td>43</td>\n",
       "      <td>10</td>\n",
       "      <td>36</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>11</td>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + CEOP</th>\n",
       "      <td>1</td>\n",
       "      <td>38</td>\n",
       "      <td>29</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>38</td>\n",
       "      <td>143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + CEOP</th>\n",
       "      <td>19</td>\n",
       "      <td>39</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>28</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + ROC</th>\n",
       "      <td>5</td>\n",
       "      <td>28</td>\n",
       "      <td>34</td>\n",
       "      <td>29</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>33</td>\n",
       "      <td>149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + CEOP</th>\n",
       "      <td>29</td>\n",
       "      <td>43</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>43</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + ROC</th>\n",
       "      <td>8</td>\n",
       "      <td>23</td>\n",
       "      <td>32</td>\n",
       "      <td>27</td>\n",
       "      <td>21</td>\n",
       "      <td>17</td>\n",
       "      <td>24</td>\n",
       "      <td>152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + ROC</th>\n",
       "      <td>12</td>\n",
       "      <td>20</td>\n",
       "      <td>33</td>\n",
       "      <td>31</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>23</td>\n",
       "      <td>152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR + ROC</th>\n",
       "      <td>35</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>29</td>\n",
       "      <td>17</td>\n",
       "      <td>20</td>\n",
       "      <td>153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + CEOP</th>\n",
       "      <td>29</td>\n",
       "      <td>42</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>42</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>41</td>\n",
       "      <td>32</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>30</td>\n",
       "      <td>156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + GSR</th>\n",
       "      <td>33</td>\n",
       "      <td>21</td>\n",
       "      <td>23</td>\n",
       "      <td>32</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + EOP</th>\n",
       "      <td>28</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>18</td>\n",
       "      <td>40</td>\n",
       "      <td>17</td>\n",
       "      <td>27</td>\n",
       "      <td>160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR + ROC</th>\n",
       "      <td>37</td>\n",
       "      <td>24</td>\n",
       "      <td>17</td>\n",
       "      <td>34</td>\n",
       "      <td>44</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + EGR</th>\n",
       "      <td>37</td>\n",
       "      <td>24</td>\n",
       "      <td>17</td>\n",
       "      <td>34</td>\n",
       "      <td>44</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR</th>\n",
       "      <td>16</td>\n",
       "      <td>35</td>\n",
       "      <td>25</td>\n",
       "      <td>17</td>\n",
       "      <td>15</td>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + ROC</th>\n",
       "      <td>7</td>\n",
       "      <td>30</td>\n",
       "      <td>31</td>\n",
       "      <td>26</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>35</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR</th>\n",
       "      <td>14</td>\n",
       "      <td>19</td>\n",
       "      <td>40</td>\n",
       "      <td>38</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>22</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ROC</th>\n",
       "      <td>5</td>\n",
       "      <td>28</td>\n",
       "      <td>34</td>\n",
       "      <td>29</td>\n",
       "      <td>19</td>\n",
       "      <td>17</td>\n",
       "      <td>33</td>\n",
       "      <td>165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EOP</th>\n",
       "      <td>20</td>\n",
       "      <td>12</td>\n",
       "      <td>41</td>\n",
       "      <td>41</td>\n",
       "      <td>23</td>\n",
       "      <td>17</td>\n",
       "      <td>15</td>\n",
       "      <td>169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + CEOP</th>\n",
       "      <td>45</td>\n",
       "      <td>47</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "      <td>169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + ROC</th>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>39</td>\n",
       "      <td>39</td>\n",
       "      <td>24</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSR</th>\n",
       "      <td>33</td>\n",
       "      <td>21</td>\n",
       "      <td>23</td>\n",
       "      <td>32</td>\n",
       "      <td>38</td>\n",
       "      <td>17</td>\n",
       "      <td>8</td>\n",
       "      <td>172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR</th>\n",
       "      <td>37</td>\n",
       "      <td>24</td>\n",
       "      <td>17</td>\n",
       "      <td>34</td>\n",
       "      <td>44</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + CEOP</th>\n",
       "      <td>15</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>48</td>\n",
       "      <td>17</td>\n",
       "      <td>40</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EGR + ROC</th>\n",
       "      <td>37</td>\n",
       "      <td>24</td>\n",
       "      <td>17</td>\n",
       "      <td>34</td>\n",
       "      <td>44</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + PR + CEOP</th>\n",
       "      <td>29</td>\n",
       "      <td>40</td>\n",
       "      <td>28</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>17</td>\n",
       "      <td>41</td>\n",
       "      <td>179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + CEOP</th>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>46</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>37</td>\n",
       "      <td>182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR</th>\n",
       "      <td>26</td>\n",
       "      <td>8</td>\n",
       "      <td>42</td>\n",
       "      <td>42</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>14</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CEOP</th>\n",
       "      <td>45</td>\n",
       "      <td>47</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>17</td>\n",
       "      <td>47</td>\n",
       "      <td>185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + GSR + EOP</th>\n",
       "      <td>26</td>\n",
       "      <td>16</td>\n",
       "      <td>38</td>\n",
       "      <td>40</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR + EOP</th>\n",
       "      <td>18</td>\n",
       "      <td>9</td>\n",
       "      <td>47</td>\n",
       "      <td>47</td>\n",
       "      <td>41</td>\n",
       "      <td>17</td>\n",
       "      <td>13</td>\n",
       "      <td>192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RW + PR + CEOP</th>\n",
       "      <td>47</td>\n",
       "      <td>45</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PR + CEOP</th>\n",
       "      <td>47</td>\n",
       "      <td>45</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>42</td>\n",
       "      <td>17</td>\n",
       "      <td>45</td>\n",
       "      <td>220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIR + EGR</th>\n",
       "      <td>25</td>\n",
       "      <td>31</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>31</td>\n",
       "      <td>17</td>\n",
       "      <td>32</td>\n",
       "      <td>232</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     False Positive Rate Diff  False Negative Rate Diff  \\\n",
       "Bank                                                                      \n",
       "RW + PR + EOP                               3                         3   \n",
       "RW + GSR + EOP                              9                         1   \n",
       "RW + EGR + EOP                             10                        13   \n",
       "PR + EOP                                    3                         3   \n",
       "RW + EOP                                   24                        15   \n",
       "RW + PR + ROC                              21                         6   \n",
       "RW + PR                                    43                        10   \n",
       "GSR + EOP                                  13                         5   \n",
       "RW + GSR + CEOP                            29                        43   \n",
       "DIR + PR + EOP                             11                         2   \n",
       "EOP                                        23                        14   \n",
       "RW + GSR + ROC                             35                        36   \n",
       "RW                                         41                        32   \n",
       "PR + ROC                                   21                         6   \n",
       "PR                                         43                        10   \n",
       "DIR + GSR + CEOP                            1                        38   \n",
       "RW + EGR + CEOP                            19                        39   \n",
       "RW + ROC                                    5                        28   \n",
       "GSR + CEOP                                 29                        43   \n",
       "DIR + GSR + ROC                             8                        23   \n",
       "DIR + PR + ROC                             12                        20   \n",
       "GSR + ROC                                  35                        36   \n",
       "DIR + CEOP                                 29                        42   \n",
       "Logistic Regression                        41                        32   \n",
       "RW + GSR                                   33                        21   \n",
       "EGR + EOP                                  28                        18   \n",
       "RW + EGR + ROC                             37                        24   \n",
       "RW + EGR                                   37                        24   \n",
       "DIR                                        16                        35   \n",
       "DIR + ROC                                   7                        30   \n",
       "DIR + PR                                   14                        19   \n",
       "ROC                                         5                        28   \n",
       "DIR + EOP                                  20                        12   \n",
       "RW + CEOP                                  45                        47   \n",
       "DIR + EGR + ROC                            17                        17   \n",
       "GSR                                        33                        21   \n",
       "EGR                                        37                        24   \n",
       "EGR + CEOP                                 15                        41   \n",
       "EGR + ROC                                  37                        24   \n",
       "DIR + PR + CEOP                            29                        40   \n",
       "DIR + EGR + CEOP                            1                        34   \n",
       "DIR + GSR                                  26                         8   \n",
       "CEOP                                       45                        47   \n",
       "DIR + GSR + EOP                            26                        16   \n",
       "DIR + EGR + EOP                            18                         9   \n",
       "RW + PR + CEOP                             47                        45   \n",
       "PR + CEOP                                  47                        45   \n",
       "DIR + EGR                                  25                        31   \n",
       "\n",
       "                     Accuracy Diff  False Omission Rate Diff  \\\n",
       "Bank                                                           \n",
       "RW + PR + EOP                   13                        20   \n",
       "RW + GSR + EOP                  27                        25   \n",
       "RW + EGR + EOP                  30                        28   \n",
       "PR + EOP                        13                        20   \n",
       "RW + EOP                         5                        12   \n",
       "RW + PR + ROC                   44                        44   \n",
       "RW + PR                         36                        22   \n",
       "GSR + EOP                       26                        24   \n",
       "RW + GSR + CEOP                  8                         5   \n",
       "DIR + PR + EOP                  43                        43   \n",
       "EOP                              6                        13   \n",
       "RW + GSR + ROC                   1                        15   \n",
       "RW                              15                        10   \n",
       "PR + ROC                        44                        44   \n",
       "PR                              36                        22   \n",
       "DIR + GSR + CEOP                29                        19   \n",
       "RW + EGR + CEOP                 10                         9   \n",
       "RW + ROC                        34                        29   \n",
       "GSR + CEOP                       8                         5   \n",
       "DIR + GSR + ROC                 32                        27   \n",
       "DIR + PR + ROC                  33                        31   \n",
       "GSR + ROC                        1                        15   \n",
       "DIR + CEOP                      11                         7   \n",
       "Logistic Regression             15                        10   \n",
       "RW + GSR                        23                        32   \n",
       "EGR + EOP                       12                        18   \n",
       "RW + EGR + ROC                  17                        34   \n",
       "RW + EGR                        17                        34   \n",
       "DIR                             25                        17   \n",
       "DIR + ROC                       31                        26   \n",
       "DIR + PR                        40                        38   \n",
       "ROC                             34                        29   \n",
       "DIR + EOP                       41                        41   \n",
       "RW + CEOP                        3                         1   \n",
       "DIR + EGR + ROC                 39                        39   \n",
       "GSR                             23                        32   \n",
       "EGR                             17                        34   \n",
       "EGR + CEOP                       7                         8   \n",
       "EGR + ROC                       17                        34   \n",
       "DIR + PR + CEOP                 28                        14   \n",
       "DIR + EGR + CEOP                46                        46   \n",
       "DIR + GSR                       42                        42   \n",
       "CEOP                             3                         1   \n",
       "DIR + GSR + EOP                 38                        40   \n",
       "DIR + EGR + EOP                 47                        47   \n",
       "RW + PR + CEOP                  21                         3   \n",
       "PR + CEOP                       21                         3   \n",
       "DIR + EGR                       48                        48   \n",
       "\n",
       "                     False Discovery Rate Diff  Statistical Parity Diff  \\\n",
       "Bank                                                                      \n",
       "RW + PR + EOP                               32                        1   \n",
       "RW + GSR + EOP                              22                        1   \n",
       "RW + EGR + EOP                              18                        1   \n",
       "PR + EOP                                    32                       17   \n",
       "RW + EOP                                    37                        1   \n",
       "RW + PR + ROC                                8                        1   \n",
       "RW + PR                                      3                        1   \n",
       "GSR + EOP                                   27                       17   \n",
       "RW + GSR + CEOP                              5                        1   \n",
       "DIR + PR + EOP                              14                       17   \n",
       "EOP                                         36                       17   \n",
       "RW + GSR + ROC                              29                        1   \n",
       "RW                                          11                        1   \n",
       "PR + ROC                                     8                       17   \n",
       "PR                                           3                       17   \n",
       "DIR + GSR + CEOP                             1                       17   \n",
       "RW + EGR + CEOP                             28                        1   \n",
       "RW + ROC                                    19                        1   \n",
       "GSR + CEOP                                   5                       17   \n",
       "DIR + GSR + ROC                             21                       17   \n",
       "DIR + PR + ROC                              16                       17   \n",
       "GSR + ROC                                   29                       17   \n",
       "DIR + CEOP                                   7                       17   \n",
       "Logistic Regression                         11                       17   \n",
       "RW + GSR                                    38                        1   \n",
       "EGR + EOP                                   40                       17   \n",
       "RW + EGR + ROC                              44                        1   \n",
       "RW + EGR                                    44                        1   \n",
       "DIR                                         15                       17   \n",
       "DIR + ROC                                   17                       17   \n",
       "DIR + PR                                    13                       17   \n",
       "ROC                                         19                       17   \n",
       "DIR + EOP                                   23                       17   \n",
       "RW + CEOP                                   25                        1   \n",
       "DIR + EGR + ROC                             24                       17   \n",
       "GSR                                         38                       17   \n",
       "EGR                                         44                       17   \n",
       "EGR + CEOP                                  48                       17   \n",
       "EGR + ROC                                   44                       17   \n",
       "DIR + PR + CEOP                             10                       17   \n",
       "DIR + EGR + CEOP                             1                       17   \n",
       "DIR + GSR                                   34                       17   \n",
       "CEOP                                        25                       17   \n",
       "DIR + GSR + EOP                             34                       17   \n",
       "DIR + EGR + EOP                             41                       17   \n",
       "RW + PR + CEOP                              42                        1   \n",
       "PR + CEOP                                   42                       17   \n",
       "DIR + EGR                                   31                       17   \n",
       "\n",
       "                     F1 Score Diff  Rank Score  \n",
       "Bank                                            \n",
       "RW + PR + EOP                   25          97  \n",
       "RW + GSR + EOP                  16         101  \n",
       "RW + EGR + EOP                  10         110  \n",
       "PR + EOP                        25         113  \n",
       "RW + EOP                        29         123  \n",
       "RW + PR + ROC                    1         125  \n",
       "RW + PR                         11         126  \n",
       "GSR + EOP                       19         131  \n",
       "RW + GSR + CEOP                 43         134  \n",
       "DIR + PR + EOP                   7         137  \n",
       "EOP                             28         137  \n",
       "RW + GSR + ROC                  20         137  \n",
       "RW                              30         140  \n",
       "PR + ROC                         1         141  \n",
       "PR                              11         142  \n",
       "DIR + GSR + CEOP                38         143  \n",
       "RW + EGR + CEOP                 39         145  \n",
       "RW + ROC                        33         149  \n",
       "GSR + CEOP                      43         150  \n",
       "DIR + GSR + ROC                 24         152  \n",
       "DIR + PR + ROC                  23         152  \n",
       "GSR + ROC                       20         153  \n",
       "DIR + CEOP                      42         155  \n",
       "Logistic Regression             30         156  \n",
       "RW + GSR                         8         156  \n",
       "EGR + EOP                       27         160  \n",
       "RW + EGR + ROC                   3         160  \n",
       "RW + EGR                         3         160  \n",
       "DIR                             36         161  \n",
       "DIR + ROC                       35         163  \n",
       "DIR + PR                        22         163  \n",
       "ROC                             33         165  \n",
       "DIR + EOP                       15         169  \n",
       "RW + CEOP                       47         169  \n",
       "DIR + EGR + ROC                 18         171  \n",
       "GSR                              8         172  \n",
       "EGR                              3         176  \n",
       "EGR + CEOP                      40         176  \n",
       "EGR + ROC                        3         176  \n",
       "DIR + PR + CEOP                 41         179  \n",
       "DIR + EGR + CEOP                37         182  \n",
       "DIR + GSR                       14         183  \n",
       "CEOP                            47         185  \n",
       "DIR + GSR + EOP                 17         188  \n",
       "DIR + EGR + EOP                 13         192  \n",
       "RW + PR + CEOP                  45         204  \n",
       "PR + CEOP                       45         220  \n",
       "DIR + EGR                       32         232  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bank_ranking = pandas.DataFrame(data=bank_df['Bank'], columns=['Bank'])\n",
    "for metric in bank_df.columns:\n",
    "    if metric == \"Bank\":\n",
    "        continue\n",
    "    if metric not in {\"Accuracy\", \"F1 Score\", \"Consistency\", \"Theil Index\"}:\n",
    "        bank_ranking[metric] = bank_df[metric].rank(ascending=True, method=\"min\").astype(int)\n",
    "bank_ranking = bank_ranking.set_index(\"Bank\")\n",
    "rank_scores = []\n",
    "for intervention in bank_df[\"Bank\"]:\n",
    "    rank_scores.append(sum(bank_ranking.loc[intervention]))\n",
    "bank_ranking[\"Rank Score\"] = rank_scores\n",
    "bank_ranking.sort_values(\"Rank Score\", inplace=True)\n",
    "bank_ranking"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}