quality_f1scores.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42df5d32-21dd-434e-a858-1a7755e0c09f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import precision_recall_fscore_support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38e444c0-cf38-4930-9b2f-e86a60f671ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_decisions_df = pd.read_csv(\"results/all_decisions_df.csv\")\n",
    "all_decisions_df.fillna({\"model\": \"\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4007e8ad-55bc-4c56-88a6-141bd0887776",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_df = all_decisions_df.groupby(\n",
    "    [\"model\", \"dataset\", \"task_scope\", \"experiment_run\"]\n",
    ").apply(\n",
    "    lambda group: pd.Series(\n",
    "        precision_recall_fscore_support(\n",
    "            group[\"benchmark\"],\n",
    "            group[\"decision\"] == \"yes\",\n",
    "            average=\"binary\",\n",
    "            pos_label=True,\n",
    "            zero_division=0.0\n",
    "        ),\n",
    "        index=[\"precision\", \"recall\", \"f1-score\", \"support\"]\n",
    "    ),\n",
    "    include_groups=False,\n",
    ").reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd58aea5-0288-4d0b-a221-5d30b9278b70",
   "metadata": {},
   "outputs": [],
   "source": [
    "median_f1_scores = scores_df.groupby([\"model\", \"task_scope\", \"dataset\"])[\"f1-score\"].median()\n",
    "\n",
    "median_experiments = []\n",
    "for (model, task_scope, dataset), f1_score in median_f1_scores.items():\n",
    "    _df = scores_df.query((\n",
    "        f\"(model == @model) and (task_scope == @task_scope) and\"\n",
    "        f\" (dataset == @dataset) and (`f1-score` == @f1_score)\"\n",
    "    ))\n",
    "    if _df.empty:\n",
    "        raise ValueError(f\"{model}, {task_scope}, {dataset}, {f1_score}\")\n",
    "    median_experiments.append(\n",
    "        _df.iloc[0].to_dict()\n",
    "    )\n",
    "median_experiments = pd.DataFrame(median_experiments)\n",
    "\n",
    "median_scores = pd.pivot(\n",
    "    median_experiments[[\"model\", \"task_scope\", \"dataset\", \"f1-score\", \"precision\", \"recall\"]].groupby([\"model\", \"task_scope\", \"dataset\"]).median().reset_index(),\n",
    "    index=\"dataset\",\n",
    "    columns=[\"model\", \"task_scope\"],\n",
    "    values=[\"f1-score\", \"precision\", \"recall\"],\n",
    ")\n",
    "\n",
    "median_scores = pd.concat((\n",
    "    median_scores,\n",
    "    median_scores.mean().to_frame(name=\"mean\").T,\n",
    "), axis=\"index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0ae43d3-1d89-4ab8-a58a-aaf53d5276b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.graph_objects as go\n",
    "\n",
    "median_f1_scores = median_scores.loc[:, (\"f1-score\", slice(None), slice(None))]\n",
    "values = [row - row[0] for row in median_f1_scores.values]\n",
    "texts = [\n",
    "    [\n",
    "        f\"{f1_score:.3f} ({median_scores.loc[dataset, ('precision', model, task_scope)]:.2f}, {median_scores.loc[dataset, ('recall', model, task_scope)]:.2f})\"\n",
    "        for (_, model, task_scope), f1_score in row.items()\n",
    "    ]\n",
    "    for dataset, row in median_f1_scores.iterrows()\n",
    "]\n",
    "fig = go.Figure(\n",
    "    data=go.Heatmap(\n",
    "        x=[median_f1_scores.columns.get_level_values(1), median_f1_scores.columns.get_level_values(2)],\n",
    "        y=median_scores.index,\n",
    "        z=values,\n",
    "        text=texts,\n",
    "        texttemplate=\"%{text}\",\n",
    "        textfont={\"size\": 12},\n",
    "        colorscale=\"PRGn\",\n",
    "        zmin=-1.0,\n",
    "        zmax=1.0,\n",
    "        showscale=False,\n",
    "    ),\n",
    "    layout=dict(\n",
    "        title=\"Median F1-scores compared to baseline (green: better, purple: worse)\",\n",
    "        height=600,\n",
    "        width=1000,\n",
    "        yaxis={\"autorange\": \"reversed\"}\n",
    "    ),\n",
    ")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be8a189b-e49b-45db-957d-7ef186a03006",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}