01_general_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "271b4fe3",
   "metadata": {},
   "source": [
    "# Evaluation\n",
    "In this notebook the negation prediction methods are compared to each other. This notebook contains the following:\n",
    "- Entities which have predictions from all 3 methods are joined in a single dataframe.\n",
    "- The merged dataframe is saved in this repo.\n",
    "- A voting ensemble method is applied.\n",
    "- Precision, recall and F1-scores are calculated per category and all combined.\n",
    "- Average methods for score calculations are compared.\n",
    "- RobBERT methods are compared."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "93c28e36",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from evaluation_utils import print_statistics, get_document_text\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "import pandas as pd\n",
    "import json\n",
    "import numpy as np\n",
    "\n",
    "# Categories in DCC dataset\n",
    "DOCUMENT_CATEGORIES = ['DL', 'GP', 'RD', 'SP']\n",
    "\n",
    "# Predictions methods to compare\n",
    "PREDICTION_METHODS = ['rule_based', 'bilstm_cv', 'robbert_v2_dutch_base_3_32_512', 'ensemble']\n",
    "ROBBERT_PREDICTION_METHODS = ['robbert_v2_dutch_base_3_32_512','robbert_v2_dutch_base_DAPT_3_32_512','MedRoBERTa_3_32_512']\n",
    "\n",
    "# Pretty names:\n",
    "PRETTY_NAMES = {'rule_based': 'Rule-based', \n",
    "                'bilstm_cv': 'BiLSTM', \n",
    "                'robbert_v2_dutch_base_3_32_512': 'RobBERT',\n",
    "                'ensemble': 'Voting ensemble',\n",
    "                'robbert_v2_dutch_base_DAPT_3_32_512': 'RobBERT DAPT',\n",
    "                'MedRoBERTa_3_32_512': 'MedRoBERTa.nl',\n",
    "                'average_method': 'Weighting average', \n",
    "                'category': 'Letter category',\n",
    "                'prediction_method': 'Prediction method',\n",
    "                'precision': 'Precision', \n",
    "                'recall': 'Recall',\n",
    "                'f1': 'F1',\n",
    "                'GP': 'General Practitioner entries',\n",
    "                'SP': 'Specialist letters',\n",
    "                'RD': 'Radiology reports',\n",
    "                'DL': 'Discharge letters'}\n",
    "\n",
    "# See https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html\n",
    "AVERAGE_METHODS = ['binary', 'micro', 'macro', 'weighted']\n",
    "\n",
    "# Input and output\n",
    "data_dir = Path('data')\n",
    "annotation_file = data_dir / 'emc-dcc_ann.json'\n",
    "dcc_dir = data_dir / 'EMCDutchClinicalCorpus'\n",
    "result_dir = Path('results')\n",
    "bilstm_predictions_cv_file = result_dir / 'bilstm_predictions_cv.csv.gz'\n",
    "rule_based_predictions_file = result_dir / 'rule-based_predictions.csv.gz'\n",
    "robbert_predictions_file = result_dir / 'robbert_v2_dutch_base_3_32_512_predictions.csv.gz'\n",
    "robbertdapt_predictions_file = result_dir / 'robbert_v2_dutch_base_DAPT_3_32_512_predictions.csv.gz'\n",
    "medrobertanl_predictions_file = result_dir / 'MedRoBERTa_3_32_512_predictions.csv.gz'\n",
    "merged_predictions_file = result_dir / 'merged_predictions.csv.gz'\n",
    "\n",
    "# Load annotated data\n",
    "with open(annotation_file) as f:\n",
    "    annotations = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7a4da107",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12551\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity_id</th>\n",
       "      <th>bilstm_cv</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DL1111_32_46</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DL1111_272_280</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DL1111_363_377</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DL1116_32_41</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DL1116_137_148</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        entity_id    bilstm_cv\n",
       "0    DL1111_32_46  not negated\n",
       "1  DL1111_272_280  not negated\n",
       "2  DL1111_363_377  not negated\n",
       "3    DL1116_32_41  not negated\n",
       "4  DL1116_137_148  not negated"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load biLSTM (cross validation) predictions\n",
    "bilstm_predictions_cv = pd.read_csv(bilstm_predictions_cv_file)\n",
    "print(len(bilstm_predictions_cv))\n",
    "bilstm_predictions_cv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a9f3ece5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12551\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity_id</th>\n",
       "      <th>rule_based</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DL1111_32_46</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DL1111_272_280</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DL1111_363_377</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DL1112_22_28</td>\n",
       "      <td>negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DL1113_59_67</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        entity_id   rule_based\n",
       "0    DL1111_32_46  not negated\n",
       "1  DL1111_272_280  not negated\n",
       "2  DL1111_363_377  not negated\n",
       "3    DL1112_22_28      negated\n",
       "4    DL1113_59_67  not negated"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load rule based predictions\n",
    "ruled_based_predictions = pd.read_csv(rule_based_predictions_file, usecols = ['entity_id','rule_based'])\n",
    "print(len(ruled_based_predictions))\n",
    "ruled_based_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "20498b6d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12419\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity_id</th>\n",
       "      <th>robbert_v2_dutch_base_3_32_512</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DL1111_272_280</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DL1111_32_46</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DL1111_363_377</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DL1112_22_28</td>\n",
       "      <td>negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DL1113_141_155</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        entity_id robbert_v2_dutch_base_3_32_512\n",
       "0  DL1111_272_280                    not negated\n",
       "1    DL1111_32_46                    not negated\n",
       "2  DL1111_363_377                    not negated\n",
       "3    DL1112_22_28                        negated\n",
       "4  DL1113_141_155                    not negated"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load robbert predictions\n",
    "robbert_predictions = pd.read_csv(robbert_predictions_file, usecols = ['entity_id', 'robbert_v2_dutch_base_3_32_512'])\n",
    "print(len(robbert_predictions))\n",
    "robbert_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4d62d405",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12551, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity_id</th>\n",
       "      <th>category</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DL1111_32_46</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DL1111_272_280</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DL1111_363_377</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DL1112_22_28</td>\n",
       "      <td>DL</td>\n",
       "      <td>negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DL1113_59_67</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        entity_id category        label\n",
       "0    DL1111_32_46       DL  not negated\n",
       "1  DL1111_272_280       DL  not negated\n",
       "2  DL1111_363_377       DL  not negated\n",
       "3    DL1112_22_28       DL      negated\n",
       "4    DL1113_59_67       DL  not negated"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load annotated Dutch Clinical Corpus data from MedCAT format\n",
    "annotated_records = []\n",
    "for document in annotations['projects'][0]['documents']:\n",
    "    document_name = document['name']\n",
    "    text = document['text']\n",
    "\n",
    "    for annotation in document['annotations']:\n",
    "\n",
    "        # Extract data\n",
    "        start_char = annotation['start']\n",
    "        end_char = annotation['end']\n",
    "        negation_value = annotation['meta_anns']['Negation']['value']\n",
    "\n",
    "        # Create custom ID\n",
    "        entity_id = f'{document_name}_{start_char}_{end_char}'\n",
    "        \n",
    "        # Extract category\n",
    "        if 'DL' in document_name:\n",
    "            category = 'DL'\n",
    "        elif 'GP' in document_name:\n",
    "            category = 'GP'\n",
    "        elif 'RD' in document_name:\n",
    "            category = 'RD'\n",
    "        else:\n",
    "            category = 'SP'\n",
    "        \n",
    "        # Create row\n",
    "        annotated_records.append([entity_id, category, negation_value])\n",
    "\n",
    "annotated_data = pd.DataFrame(annotated_records, columns=['entity_id', 'category', 'label'])\n",
    "print(annotated_data.shape)\n",
    "annotated_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2fb1ee5",
   "metadata": {},
   "source": [
    "## Merge annotations from different methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ca76bfca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total entities: 12551\n",
      "Total entities with prediction in all methods: 12419\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity_id</th>\n",
       "      <th>category</th>\n",
       "      <th>label</th>\n",
       "      <th>bilstm_cv</th>\n",
       "      <th>rule_based</th>\n",
       "      <th>robbert_v2_dutch_base_3_32_512</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DL1111_32_46</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DL1111_272_280</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DL1111_363_377</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DL1112_22_28</td>\n",
       "      <td>DL</td>\n",
       "      <td>negated</td>\n",
       "      <td>negated</td>\n",
       "      <td>negated</td>\n",
       "      <td>negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DL1113_59_67</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        entity_id category        label    bilstm_cv   rule_based  \\\n",
       "0    DL1111_32_46       DL  not negated  not negated  not negated   \n",
       "1  DL1111_272_280       DL  not negated  not negated  not negated   \n",
       "2  DL1111_363_377       DL  not negated  not negated  not negated   \n",
       "3    DL1112_22_28       DL      negated      negated      negated   \n",
       "4    DL1113_59_67       DL  not negated  not negated  not negated   \n",
       "\n",
       "  robbert_v2_dutch_base_3_32_512  \n",
       "0                    not negated  \n",
       "1                    not negated  \n",
       "2                    not negated  \n",
       "3                        negated  \n",
       "4                    not negated  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = pd.merge(left=annotated_data, right = bilstm_predictions_cv, left_on='entity_id', right_on='entity_id', how='left')\n",
    "predictions = pd.merge(left=predictions, right = ruled_based_predictions, left_on='entity_id', right_on='entity_id', how='left')\n",
    "predictions = pd.merge(left=predictions, right = robbert_predictions, left_on='entity_id', right_on='entity_id', how='left')\n",
    "print(f'Total entities: {len(predictions)}')\n",
    "predictions.dropna(inplace=True)\n",
    "print(f'Total entities with prediction in all methods: {len(predictions)}')\n",
    "predictions.to_csv(merged_predictions_file, index=False, compression='gzip', line_terminator='\\n')\n",
    "predictions.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "26832a49",
   "metadata": {},
   "outputs": [],
   "source": [
    "def majority_vote(row):\n",
    "    predicted_negated = 0\n",
    "    for method in ['rule_based', 'bilstm_cv', 'robbert_v2_dutch_base_3_32_512']:\n",
    "        if row[method] == 'negated':\n",
    "            predicted_negated += 1\n",
    "    if predicted_negated > 1:\n",
    "        return 'negated'\n",
    "    else:\n",
    "        return 'not negated'\n",
    "\n",
    "predictions['ensemble'] = predictions.apply(majority_vote, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a1ed72e",
   "metadata": {},
   "source": [
    "## Function to calculate scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "556f3ffb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_score_record(prediction_df, prediction_method, average_method, category='all'):\n",
    "    \"\"\"\n",
    "    Wrapper for precision_recall_fscore_support().\n",
    "\n",
    "    Returns list containing method parameters and scores that can be used as row in DataFrame.\n",
    "    See https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html\n",
    "    \"\"\"\n",
    "    if average_method == 'binary':\n",
    "        pos_label = 'negated'\n",
    "    else:\n",
    "        pos_label=1\n",
    "    \n",
    "    if category == 'all':\n",
    "        # Calculate score for all categories. \n",
    "        subset = prediction_df[['label', prediction_method]]\n",
    "    else:\n",
    "        # Calculate scores for specific category\n",
    "        subset = prediction_df.loc[prediction_df.category == category, ['label', prediction_method]]\n",
    "    return [average_method, category, prediction_method] + (list(precision_recall_fscore_support(subset.label, subset[prediction_method], labels=['negated', 'not negated'], average = average_method, pos_label=pos_label)))\n",
    "\n",
    "def create_score_dataframe(score_list):\n",
    "    scores = pd.DataFrame(score_list, columns=['average_method', 'category', 'prediction_method', 'precision', 'recall', 'f1', 'support'])\n",
    "    scores.drop(['support'], axis=1, inplace=True)\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3aae6d6b",
   "metadata": {},
   "source": [
    "## Compare different ways of calculating scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5b5c9b3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>average_method</th>\n",
       "      <th>category</th>\n",
       "      <th>prediction_method</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>binary</td>\n",
       "      <td>all</td>\n",
       "      <td>rule_based</td>\n",
       "      <td>0.824868</td>\n",
       "      <td>0.891876</td>\n",
       "      <td>0.857064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>micro</td>\n",
       "      <td>all</td>\n",
       "      <td>rule_based</td>\n",
       "      <td>0.958129</td>\n",
       "      <td>0.958129</td>\n",
       "      <td>0.958129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>macro</td>\n",
       "      <td>all</td>\n",
       "      <td>rule_based</td>\n",
       "      <td>0.903459</td>\n",
       "      <td>0.930429</td>\n",
       "      <td>0.916268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>weighted</td>\n",
       "      <td>all</td>\n",
       "      <td>rule_based</td>\n",
       "      <td>0.959926</td>\n",
       "      <td>0.958129</td>\n",
       "      <td>0.958806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>binary</td>\n",
       "      <td>all</td>\n",
       "      <td>bilstm_cv</td>\n",
       "      <td>0.925926</td>\n",
       "      <td>0.901030</td>\n",
       "      <td>0.913308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>micro</td>\n",
       "      <td>all</td>\n",
       "      <td>bilstm_cv</td>\n",
       "      <td>0.975924</td>\n",
       "      <td>0.975924</td>\n",
       "      <td>0.975924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>macro</td>\n",
       "      <td>all</td>\n",
       "      <td>bilstm_cv</td>\n",
       "      <td>0.954892</td>\n",
       "      <td>0.944611</td>\n",
       "      <td>0.949665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>weighted</td>\n",
       "      <td>all</td>\n",
       "      <td>bilstm_cv</td>\n",
       "      <td>0.975705</td>\n",
       "      <td>0.975924</td>\n",
       "      <td>0.975786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>binary</td>\n",
       "      <td>all</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.950696</td>\n",
       "      <td>0.937643</td>\n",
       "      <td>0.944124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>micro</td>\n",
       "      <td>all</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.984379</td>\n",
       "      <td>0.984379</td>\n",
       "      <td>0.984379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>macro</td>\n",
       "      <td>all</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.970252</td>\n",
       "      <td>0.964839</td>\n",
       "      <td>0.967522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>weighted</td>\n",
       "      <td>all</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.984303</td>\n",
       "      <td>0.984379</td>\n",
       "      <td>0.984334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>binary</td>\n",
       "      <td>all</td>\n",
       "      <td>ensemble</td>\n",
       "      <td>0.947801</td>\n",
       "      <td>0.924485</td>\n",
       "      <td>0.935998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>micro</td>\n",
       "      <td>all</td>\n",
       "      <td>ensemble</td>\n",
       "      <td>0.982205</td>\n",
       "      <td>0.982205</td>\n",
       "      <td>0.982205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>macro</td>\n",
       "      <td>all</td>\n",
       "      <td>ensemble</td>\n",
       "      <td>0.967740</td>\n",
       "      <td>0.958072</td>\n",
       "      <td>0.962832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>weighted</td>\n",
       "      <td>all</td>\n",
       "      <td>ensemble</td>\n",
       "      <td>0.982067</td>\n",
       "      <td>0.982205</td>\n",
       "      <td>0.982112</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   average_method category               prediction_method  precision  \\\n",
       "0          binary      all                      rule_based   0.824868   \n",
       "1           micro      all                      rule_based   0.958129   \n",
       "2           macro      all                      rule_based   0.903459   \n",
       "3        weighted      all                      rule_based   0.959926   \n",
       "4          binary      all                       bilstm_cv   0.925926   \n",
       "5           micro      all                       bilstm_cv   0.975924   \n",
       "6           macro      all                       bilstm_cv   0.954892   \n",
       "7        weighted      all                       bilstm_cv   0.975705   \n",
       "8          binary      all  robbert_v2_dutch_base_3_32_512   0.950696   \n",
       "9           micro      all  robbert_v2_dutch_base_3_32_512   0.984379   \n",
       "10          macro      all  robbert_v2_dutch_base_3_32_512   0.970252   \n",
       "11       weighted      all  robbert_v2_dutch_base_3_32_512   0.984303   \n",
       "12         binary      all                        ensemble   0.947801   \n",
       "13          micro      all                        ensemble   0.982205   \n",
       "14          macro      all                        ensemble   0.967740   \n",
       "15       weighted      all                        ensemble   0.982067   \n",
       "\n",
       "      recall        f1  \n",
       "0   0.891876  0.857064  \n",
       "1   0.958129  0.958129  \n",
       "2   0.930429  0.916268  \n",
       "3   0.958129  0.958806  \n",
       "4   0.901030  0.913308  \n",
       "5   0.975924  0.975924  \n",
       "6   0.944611  0.949665  \n",
       "7   0.975924  0.975786  \n",
       "8   0.937643  0.944124  \n",
       "9   0.984379  0.984379  \n",
       "10  0.964839  0.967522  \n",
       "11  0.984379  0.984334  \n",
       "12  0.924485  0.935998  \n",
       "13  0.982205  0.982205  \n",
       "14  0.958072  0.962832  \n",
       "15  0.982205  0.982112  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_list = []\n",
    "for prediction_method in PREDICTION_METHODS:\n",
    "    for average_method in AVERAGE_METHODS:\n",
    "        score_list.append(create_score_record(predictions, prediction_method, average_method, category = 'all'))\n",
    "create_score_dataframe(score_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1ba66d6",
   "metadata": {},
   "source": [
    "Based on these results we dediced to use the `binary` avering methods, because it shows that largest differences between methods."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f288d15",
   "metadata": {},
   "source": [
    "## Scores per category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ef8de822",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>average_method</th>\n",
       "      <th>category</th>\n",
       "      <th>prediction_method</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>binary</td>\n",
       "      <td>DL</td>\n",
       "      <td>rule_based</td>\n",
       "      <td>0.892583</td>\n",
       "      <td>0.920844</td>\n",
       "      <td>0.906494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>binary</td>\n",
       "      <td>DL</td>\n",
       "      <td>bilstm_cv</td>\n",
       "      <td>0.956640</td>\n",
       "      <td>0.931398</td>\n",
       "      <td>0.943850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>binary</td>\n",
       "      <td>DL</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.953488</td>\n",
       "      <td>0.973615</td>\n",
       "      <td>0.963446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>binary</td>\n",
       "      <td>DL</td>\n",
       "      <td>ensemble</td>\n",
       "      <td>0.963158</td>\n",
       "      <td>0.965699</td>\n",
       "      <td>0.964427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>binary</td>\n",
       "      <td>GP</td>\n",
       "      <td>rule_based</td>\n",
       "      <td>0.674107</td>\n",
       "      <td>0.801061</td>\n",
       "      <td>0.732121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>binary</td>\n",
       "      <td>GP</td>\n",
       "      <td>bilstm_cv</td>\n",
       "      <td>0.888594</td>\n",
       "      <td>0.888594</td>\n",
       "      <td>0.888594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>binary</td>\n",
       "      <td>GP</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.950276</td>\n",
       "      <td>0.912467</td>\n",
       "      <td>0.930988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>binary</td>\n",
       "      <td>GP</td>\n",
       "      <td>ensemble</td>\n",
       "      <td>0.930362</td>\n",
       "      <td>0.885942</td>\n",
       "      <td>0.907609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>binary</td>\n",
       "      <td>RD</td>\n",
       "      <td>rule_based</td>\n",
       "      <td>0.900943</td>\n",
       "      <td>0.966273</td>\n",
       "      <td>0.932465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>binary</td>\n",
       "      <td>RD</td>\n",
       "      <td>bilstm_cv</td>\n",
       "      <td>0.932660</td>\n",
       "      <td>0.934233</td>\n",
       "      <td>0.933446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>binary</td>\n",
       "      <td>RD</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.959664</td>\n",
       "      <td>0.962901</td>\n",
       "      <td>0.961279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>binary</td>\n",
       "      <td>RD</td>\n",
       "      <td>ensemble</td>\n",
       "      <td>0.954925</td>\n",
       "      <td>0.964587</td>\n",
       "      <td>0.959732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>binary</td>\n",
       "      <td>SP</td>\n",
       "      <td>rule_based</td>\n",
       "      <td>0.807229</td>\n",
       "      <td>0.839599</td>\n",
       "      <td>0.823096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>binary</td>\n",
       "      <td>SP</td>\n",
       "      <td>bilstm_cv</td>\n",
       "      <td>0.922438</td>\n",
       "      <td>0.834586</td>\n",
       "      <td>0.876316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>binary</td>\n",
       "      <td>SP</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.934211</td>\n",
       "      <td>0.889724</td>\n",
       "      <td>0.911425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>binary</td>\n",
       "      <td>SP</td>\n",
       "      <td>ensemble</td>\n",
       "      <td>0.937330</td>\n",
       "      <td>0.862155</td>\n",
       "      <td>0.898172</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   average_method category               prediction_method  precision  \\\n",
       "0          binary       DL                      rule_based   0.892583   \n",
       "1          binary       DL                       bilstm_cv   0.956640   \n",
       "2          binary       DL  robbert_v2_dutch_base_3_32_512   0.953488   \n",
       "3          binary       DL                        ensemble   0.963158   \n",
       "4          binary       GP                      rule_based   0.674107   \n",
       "5          binary       GP                       bilstm_cv   0.888594   \n",
       "6          binary       GP  robbert_v2_dutch_base_3_32_512   0.950276   \n",
       "7          binary       GP                        ensemble   0.930362   \n",
       "8          binary       RD                      rule_based   0.900943   \n",
       "9          binary       RD                       bilstm_cv   0.932660   \n",
       "10         binary       RD  robbert_v2_dutch_base_3_32_512   0.959664   \n",
       "11         binary       RD                        ensemble   0.954925   \n",
       "12         binary       SP                      rule_based   0.807229   \n",
       "13         binary       SP                       bilstm_cv   0.922438   \n",
       "14         binary       SP  robbert_v2_dutch_base_3_32_512   0.934211   \n",
       "15         binary       SP                        ensemble   0.937330   \n",
       "\n",
       "      recall        f1  \n",
       "0   0.920844  0.906494  \n",
       "1   0.931398  0.943850  \n",
       "2   0.973615  0.963446  \n",
       "3   0.965699  0.964427  \n",
       "4   0.801061  0.732121  \n",
       "5   0.888594  0.888594  \n",
       "6   0.912467  0.930988  \n",
       "7   0.885942  0.907609  \n",
       "8   0.966273  0.932465  \n",
       "9   0.934233  0.933446  \n",
       "10  0.962901  0.961279  \n",
       "11  0.964587  0.959732  \n",
       "12  0.839599  0.823096  \n",
       "13  0.834586  0.876316  \n",
       "14  0.889724  0.911425  \n",
       "15  0.862155  0.898172  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_method = 'binary'\n",
    "score_list = []\n",
    "for category in DOCUMENT_CATEGORIES:\n",
    "    for prediction_method in PREDICTION_METHODS:\n",
    "        score_list.append(create_score_record(predictions, prediction_method, average_method, category = category))\n",
    "score_df = create_score_dataframe(score_list)\n",
    "score_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bbbf0c67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llrrr}\n",
      "\\toprule\n",
      "             Letter category & Prediction method &  Precision &  Recall &    F1 \\\\\n",
      "\\midrule\n",
      "           Discharge letters &        Rule-based &      0.893 &   0.921 & 0.906 \\\\\n",
      "           Discharge letters &            BiLSTM &      0.957 &   0.931 & 0.944 \\\\\n",
      "           Discharge letters &           RobBERT &      0.953 &   0.974 & 0.963 \\\\\n",
      "           Discharge letters &   Voting ensemble &      0.963 &   0.966 & 0.964 \\\\\n",
      "General Practitioner entries &        Rule-based &      0.674 &   0.801 & 0.732 \\\\\n",
      "General Practitioner entries &            BiLSTM &      0.889 &   0.889 & 0.889 \\\\\n",
      "General Practitioner entries &           RobBERT &      0.950 &   0.912 & 0.931 \\\\\n",
      "General Practitioner entries &   Voting ensemble &      0.930 &   0.886 & 0.908 \\\\\n",
      "           Radiology reports &        Rule-based &      0.901 &   0.966 & 0.932 \\\\\n",
      "           Radiology reports &            BiLSTM &      0.933 &   0.934 & 0.933 \\\\\n",
      "           Radiology reports &           RobBERT &      0.960 &   0.963 & 0.961 \\\\\n",
      "           Radiology reports &   Voting ensemble &      0.955 &   0.965 & 0.960 \\\\\n",
      "          Specialist letters &        Rule-based &      0.807 &   0.840 & 0.823 \\\\\n",
      "          Specialist letters &            BiLSTM &      0.922 &   0.835 & 0.876 \\\\\n",
      "          Specialist letters &           RobBERT &      0.934 &   0.890 & 0.911 \\\\\n",
      "          Specialist letters &   Voting ensemble &      0.937 &   0.862 & 0.898 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create pretty table\n",
    "pretty_df = score_df.round(3)\n",
    "pretty_df = pretty_df[['category', 'prediction_method', 'precision', 'recall', 'f1']]\n",
    "pretty_df.rename(columns=PRETTY_NAMES, inplace=True)\n",
    "pretty_df.replace(PRETTY_NAMES, inplace=True)\n",
    "print(pretty_df.to_latex(index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43dbcab6",
   "metadata": {},
   "source": [
    "## Scores for all categories combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0ced52f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12419\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>average_method</th>\n",
       "      <th>category</th>\n",
       "      <th>prediction_method</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>binary</td>\n",
       "      <td>all</td>\n",
       "      <td>rule_based</td>\n",
       "      <td>0.824868</td>\n",
       "      <td>0.891876</td>\n",
       "      <td>0.857064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>binary</td>\n",
       "      <td>all</td>\n",
       "      <td>bilstm_cv</td>\n",
       "      <td>0.925926</td>\n",
       "      <td>0.901030</td>\n",
       "      <td>0.913308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>binary</td>\n",
       "      <td>all</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.950696</td>\n",
       "      <td>0.937643</td>\n",
       "      <td>0.944124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>binary</td>\n",
       "      <td>all</td>\n",
       "      <td>ensemble</td>\n",
       "      <td>0.947801</td>\n",
       "      <td>0.924485</td>\n",
       "      <td>0.935998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  average_method category               prediction_method  precision  \\\n",
       "0         binary      all                      rule_based   0.824868   \n",
       "1         binary      all                       bilstm_cv   0.925926   \n",
       "2         binary      all  robbert_v2_dutch_base_3_32_512   0.950696   \n",
       "3         binary      all                        ensemble   0.947801   \n",
       "\n",
       "     recall        f1  \n",
       "0  0.891876  0.857064  \n",
       "1  0.901030  0.913308  \n",
       "2  0.937643  0.944124  \n",
       "3  0.924485  0.935998  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate F1 Scores\n",
    "score_list = []\n",
    "for prediction_method in PREDICTION_METHODS:\n",
    "    score_list.append(create_score_record(predictions, prediction_method, average_method='binary', category='all'))\n",
    "print(len(predictions))\n",
    "score_df = create_score_dataframe(score_list)\n",
    "score_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4f729474",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrr}\n",
      "\\toprule\n",
      "Prediction method &  Precision &  Recall &    F1 \\\\\n",
      "\\midrule\n",
      "       Rule-based &      0.825 &   0.892 & 0.857 \\\\\n",
      "           BiLSTM &      0.926 &   0.901 & 0.913 \\\\\n",
      "          RobBERT &      0.951 &   0.938 & 0.944 \\\\\n",
      "  Voting ensemble &      0.948 &   0.924 & 0.936 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create pretty table\n",
    "pretty_df = score_df.round(3)\n",
    "pretty_df = pretty_df[['prediction_method', 'precision', 'recall', 'f1']]\n",
    "pretty_df.rename(columns=PRETTY_NAMES, inplace=True)\n",
    "pretty_df.replace(PRETTY_NAMES, inplace=True)\n",
    "print(pretty_df.to_latex(index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8ced9a6",
   "metadata": {},
   "source": [
    "## Add other RobBERT models\n",
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9a756412",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12419\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity_id</th>\n",
       "      <th>robbert_v2_dutch_base_DAPT_3_32_512</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DL1111_272_280</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DL1111_32_46</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DL1111_363_377</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DL1112_22_28</td>\n",
       "      <td>negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DL1113_141_155</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        entity_id robbert_v2_dutch_base_DAPT_3_32_512\n",
       "0  DL1111_272_280                         not negated\n",
       "1    DL1111_32_46                         not negated\n",
       "2  DL1111_363_377                         not negated\n",
       "3    DL1112_22_28                             negated\n",
       "4  DL1113_141_155                         not negated"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load robbert dapt predictions\n",
    "robbertdapt_predictions = pd.read_csv(robbertdapt_predictions_file, usecols = ['entity_id', 'robbert_v2_dutch_base_DAPT_3_32_512'])\n",
    "print(len(robbertdapt_predictions))\n",
    "robbertdapt_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f40ca616",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12431\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity_id</th>\n",
       "      <th>MedRoBERTa_3_32_512</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DL1111_272_280</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DL1111_32_46</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DL1111_363_377</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DL1112_22_28</td>\n",
       "      <td>negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DL1113_141_155</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        entity_id MedRoBERTa_3_32_512\n",
       "0  DL1111_272_280         not negated\n",
       "1    DL1111_32_46         not negated\n",
       "2  DL1111_363_377         not negated\n",
       "3    DL1112_22_28             negated\n",
       "4  DL1113_141_155         not negated"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load MedRoberta.nl predictions\n",
    "medrobertanl_predictions = pd.read_csv(medrobertanl_predictions_file, usecols = ['entity_id', 'MedRoBERTa_3_32_512'])\n",
    "print(len(medrobertanl_predictions))\n",
    "medrobertanl_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7092f65a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity_id</th>\n",
       "      <th>category</th>\n",
       "      <th>label</th>\n",
       "      <th>robbert_v2_dutch_base_3_32_512</th>\n",
       "      <th>robbert_v2_dutch_base_DAPT_3_32_512</th>\n",
       "      <th>MedRoBERTa_3_32_512</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DL1111_32_46</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DL1111_272_280</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DL1111_363_377</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DL1112_22_28</td>\n",
       "      <td>DL</td>\n",
       "      <td>negated</td>\n",
       "      <td>negated</td>\n",
       "      <td>negated</td>\n",
       "      <td>negated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DL1113_59_67</td>\n",
       "      <td>DL</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "      <td>not negated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        entity_id category        label robbert_v2_dutch_base_3_32_512  \\\n",
       "0    DL1111_32_46       DL  not negated                    not negated   \n",
       "1  DL1111_272_280       DL  not negated                    not negated   \n",
       "2  DL1111_363_377       DL  not negated                    not negated   \n",
       "3    DL1112_22_28       DL      negated                        negated   \n",
       "4    DL1113_59_67       DL  not negated                    not negated   \n",
       "\n",
       "  robbert_v2_dutch_base_DAPT_3_32_512 MedRoBERTa_3_32_512  \n",
       "0                         not negated         not negated  \n",
       "1                         not negated         not negated  \n",
       "2                         not negated         not negated  \n",
       "3                             negated             negated  \n",
       "4                         not negated         not negated  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multiple_robbert_preditions = annotated_data.copy()\n",
    "multiple_robbert_preditions = pd.merge(left=multiple_robbert_preditions, right = robbert_predictions, left_on='entity_id', right_on='entity_id', how='left')\n",
    "multiple_robbert_preditions = pd.merge(left=multiple_robbert_preditions, right = robbertdapt_predictions, left_on='entity_id', right_on='entity_id', how='left')\n",
    "multiple_robbert_preditions = pd.merge(left=multiple_robbert_preditions, right = medrobertanl_predictions, left_on='entity_id', right_on='entity_id', how='left')\n",
    "multiple_robbert_preditions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "58ccc736",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total entities: 12419\n",
      "Entities for which there is a prediction for all methods: 12419\n"
     ]
    }
   ],
   "source": [
    "# Check whether all entities have a prediction\n",
    "print(f'Total entities: {len(robbert_predictions)}')\n",
    "multiple_robbert_preditions.dropna(inplace=True)\n",
    "multiple_robbert_preditions.drop_duplicates(inplace=True)\n",
    "print(f'Entities for which there is a prediction for all methods: {len(robbert_predictions)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd82e737",
   "metadata": {},
   "source": [
    "### Compare RobBERT methods using all categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "56f1b372",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12419\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>average_method</th>\n",
       "      <th>category</th>\n",
       "      <th>prediction_method</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>binary</td>\n",
       "      <td>all</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.950696</td>\n",
       "      <td>0.937643</td>\n",
       "      <td>0.944124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>binary</td>\n",
       "      <td>all</td>\n",
       "      <td>robbert_v2_dutch_base_DAPT_3_32_512</td>\n",
       "      <td>0.952134</td>\n",
       "      <td>0.944508</td>\n",
       "      <td>0.948306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>binary</td>\n",
       "      <td>all</td>\n",
       "      <td>MedRoBERTa_3_32_512</td>\n",
       "      <td>0.954678</td>\n",
       "      <td>0.939931</td>\n",
       "      <td>0.947247</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  average_method category                    prediction_method  precision  \\\n",
       "0         binary      all       robbert_v2_dutch_base_3_32_512   0.950696   \n",
       "1         binary      all  robbert_v2_dutch_base_DAPT_3_32_512   0.952134   \n",
       "2         binary      all                  MedRoBERTa_3_32_512   0.954678   \n",
       "\n",
       "     recall        f1  \n",
       "0  0.937643  0.944124  \n",
       "1  0.944508  0.948306  \n",
       "2  0.939931  0.947247  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_list = []\n",
    "for prediction_method in ROBBERT_PREDICTION_METHODS:\n",
    "    score_list.append(create_score_record(multiple_robbert_preditions, prediction_method, average_method='binary', category = 'all'))\n",
    "print(len(robbert_predictions))\n",
    "score_df = create_score_dataframe(score_list)\n",
    "score_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7eb2b96c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrr}\n",
      "\\toprule\n",
      "Prediction method &  Precision &  Recall &    F1 \\\\\n",
      "\\midrule\n",
      "          RobBERT &      0.951 &   0.938 & 0.944 \\\\\n",
      "     RobBERT DAPT &      0.952 &   0.945 & 0.948 \\\\\n",
      "    MedRoBERTa.nl &      0.955 &   0.940 & 0.947 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pretty_df = score_df.round(3)\n",
    "pretty_df = pretty_df[['prediction_method', 'precision', 'recall', 'f1']]\n",
    "pretty_df.rename(columns=PRETTY_NAMES, inplace=True)\n",
    "pretty_df.replace(PRETTY_NAMES, inplace=True)\n",
    "print(pretty_df.to_latex(index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2aab01c",
   "metadata": {},
   "source": [
    "### Compare RobBERT methods per category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9449c121",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>average_method</th>\n",
       "      <th>category</th>\n",
       "      <th>prediction_method</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>binary</td>\n",
       "      <td>DL</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.953488</td>\n",
       "      <td>0.973615</td>\n",
       "      <td>0.963446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>binary</td>\n",
       "      <td>DL</td>\n",
       "      <td>robbert_v2_dutch_base_DAPT_3_32_512</td>\n",
       "      <td>0.963824</td>\n",
       "      <td>0.984169</td>\n",
       "      <td>0.973890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>binary</td>\n",
       "      <td>DL</td>\n",
       "      <td>MedRoBERTa_3_32_512</td>\n",
       "      <td>0.973404</td>\n",
       "      <td>0.965699</td>\n",
       "      <td>0.969536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>binary</td>\n",
       "      <td>GP</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.950276</td>\n",
       "      <td>0.912467</td>\n",
       "      <td>0.930988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>binary</td>\n",
       "      <td>GP</td>\n",
       "      <td>robbert_v2_dutch_base_DAPT_3_32_512</td>\n",
       "      <td>0.940541</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>0.931727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>binary</td>\n",
       "      <td>GP</td>\n",
       "      <td>MedRoBERTa_3_32_512</td>\n",
       "      <td>0.943089</td>\n",
       "      <td>0.923077</td>\n",
       "      <td>0.932976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>binary</td>\n",
       "      <td>RD</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.959664</td>\n",
       "      <td>0.962901</td>\n",
       "      <td>0.961279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>binary</td>\n",
       "      <td>RD</td>\n",
       "      <td>robbert_v2_dutch_base_DAPT_3_32_512</td>\n",
       "      <td>0.956303</td>\n",
       "      <td>0.959528</td>\n",
       "      <td>0.957912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>binary</td>\n",
       "      <td>RD</td>\n",
       "      <td>MedRoBERTa_3_32_512</td>\n",
       "      <td>0.962457</td>\n",
       "      <td>0.951096</td>\n",
       "      <td>0.956743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>binary</td>\n",
       "      <td>SP</td>\n",
       "      <td>robbert_v2_dutch_base_3_32_512</td>\n",
       "      <td>0.934211</td>\n",
       "      <td>0.889724</td>\n",
       "      <td>0.911425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>binary</td>\n",
       "      <td>SP</td>\n",
       "      <td>robbert_v2_dutch_base_DAPT_3_32_512</td>\n",
       "      <td>0.945026</td>\n",
       "      <td>0.904762</td>\n",
       "      <td>0.924456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>binary</td>\n",
       "      <td>SP</td>\n",
       "      <td>MedRoBERTa_3_32_512</td>\n",
       "      <td>0.935897</td>\n",
       "      <td>0.914787</td>\n",
       "      <td>0.925222</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   average_method category                    prediction_method  precision  \\\n",
       "0          binary       DL       robbert_v2_dutch_base_3_32_512   0.953488   \n",
       "1          binary       DL  robbert_v2_dutch_base_DAPT_3_32_512   0.963824   \n",
       "2          binary       DL                  MedRoBERTa_3_32_512   0.973404   \n",
       "3          binary       GP       robbert_v2_dutch_base_3_32_512   0.950276   \n",
       "4          binary       GP  robbert_v2_dutch_base_DAPT_3_32_512   0.940541   \n",
       "5          binary       GP                  MedRoBERTa_3_32_512   0.943089   \n",
       "6          binary       RD       robbert_v2_dutch_base_3_32_512   0.959664   \n",
       "7          binary       RD  robbert_v2_dutch_base_DAPT_3_32_512   0.956303   \n",
       "8          binary       RD                  MedRoBERTa_3_32_512   0.962457   \n",
       "9          binary       SP       robbert_v2_dutch_base_3_32_512   0.934211   \n",
       "10         binary       SP  robbert_v2_dutch_base_DAPT_3_32_512   0.945026   \n",
       "11         binary       SP                  MedRoBERTa_3_32_512   0.935897   \n",
       "\n",
       "      recall        f1  \n",
       "0   0.973615  0.963446  \n",
       "1   0.984169  0.973890  \n",
       "2   0.965699  0.969536  \n",
       "3   0.912467  0.930988  \n",
       "4   0.923077  0.931727  \n",
       "5   0.923077  0.932976  \n",
       "6   0.962901  0.961279  \n",
       "7   0.959528  0.957912  \n",
       "8   0.951096  0.956743  \n",
       "9   0.889724  0.911425  \n",
       "10  0.904762  0.924456  \n",
       "11  0.914787  0.925222  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_method = 'binary'\n",
    "score_list = []\n",
    "for category in DOCUMENT_CATEGORIES:\n",
    "    for prediction_method in ROBBERT_PREDICTION_METHODS:\n",
    "        score_list.append(create_score_record(multiple_robbert_preditions, prediction_method, average_method, category = category))\n",
    "score_df = create_score_dataframe(score_list)\n",
    "score_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "610aac59",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llrrr}\n",
      "\\toprule\n",
      "             Letter category & Prediction method &  Precision &  Recall &    F1 \\\\\n",
      "\\midrule\n",
      "           Discharge letters &           RobBERT &      0.953 &   0.974 & 0.963 \\\\\n",
      "           Discharge letters &      RobBERT DAPT &      0.964 &   0.984 & 0.974 \\\\\n",
      "           Discharge letters &     MedRoBERTa.nl &      0.973 &   0.966 & 0.970 \\\\\n",
      "General Practitioner entries &           RobBERT &      0.950 &   0.912 & 0.931 \\\\\n",
      "General Practitioner entries &      RobBERT DAPT &      0.941 &   0.923 & 0.932 \\\\\n",
      "General Practitioner entries &     MedRoBERTa.nl &      0.943 &   0.923 & 0.933 \\\\\n",
      "           Radiology reports &           RobBERT &      0.960 &   0.963 & 0.961 \\\\\n",
      "           Radiology reports &      RobBERT DAPT &      0.956 &   0.960 & 0.958 \\\\\n",
      "           Radiology reports &     MedRoBERTa.nl &      0.962 &   0.951 & 0.957 \\\\\n",
      "          Specialist letters &           RobBERT &      0.934 &   0.890 & 0.911 \\\\\n",
      "          Specialist letters &      RobBERT DAPT &      0.945 &   0.905 & 0.924 \\\\\n",
      "          Specialist letters &     MedRoBERTa.nl &      0.936 &   0.915 & 0.925 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pretty_df = score_df.round(3)\n",
    "pretty_df = pretty_df[['category', 'prediction_method', 'precision', 'recall', 'f1']]\n",
    "pretty_df.rename(columns=PRETTY_NAMES, inplace=True)\n",
    "pretty_df.replace(PRETTY_NAMES, inplace=True)\n",
    "print(pretty_df.to_latex(index=False))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}