benchmark_overview.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b1b4de71-6c0a-48cb-aa0f-07eb63cb681b",
   "metadata": {},
   "source": [
    "# Benchmark Overview\n",
    "\n",
    "This notebook generates an overview of the benchmark: the table combinations (aka the datsets), the number of attributes and attribute pairs and the number of true semantic matches found in the ground truth."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0da2f43-2add-4cb1-9ffe-c3e34531705e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import functools\n",
    "import os\n",
    "from typing import Dict, List, Optional\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from fm_matcher.utils.models import Attribute, Parameters, Relation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3f30268-a597-40b1-b527-c6df523fd4a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "benchmark = pd.read_csv(\"benchmark/ground_truth.csv\")\n",
    "for side in (\"source\", \"target\"):\n",
    "    benchmark[[f\"{side}_schema\", f\"{side}_relation\", f\"{side}_attribute\"]] = benchmark[side].str.split(\".\", expand=True)\n",
    "    benchmark[side] = benchmark[side].str.lower()\n",
    "benchmark[\"benchmark\"] = True\n",
    "\n",
    "relation_combinations = benchmark[[\"source_relation\", \"target_relation\"]].drop_duplicates().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a30d32f-ba90-4f44-a7a8-9138047280f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "@functools.cache\n",
    "def get_description(schema: str, table: str, attribute: Optional[str] = None) -> str:\n",
    "    if attribute:\n",
    "        filename = f\"{schema}_{table}_{attribute}.txt\"\n",
    "    else:\n",
    "        filename = f\"{schema}_table_{table}.txt\"\n",
    "    filename_filter = lambda f: f.lower() == filename.lower()\n",
    "    filename = next(filter(filename_filter, os.listdir(\"schema_documentations\")))\n",
    "    with open(os.path.join(\"schema_documentations\", filename), \"r\") as desc_file:\n",
    "        description = desc_file.read()\n",
    "    return description\n",
    "\n",
    "\n",
    "@functools.cache\n",
    "def get_attributes(schema: str, table: str) -> List[str]:\n",
    "    table_filter = lambda f: f.lower().startswith(f\"{schema}_{table}_\") and f.endswith(\".txt\")\n",
    "    extract_attr_name = lambda f: f[len(f\"{schema}_{table}_\"):-len(\".txt\")]\n",
    "    return [\n",
    "        extract_attr_name(attr_file)\n",
    "        for attr_file in filter(table_filter, os.listdir(\"schema_documentations/\"))\n",
    "    ]\n",
    "\n",
    "\n",
    "@functools.cache\n",
    "def get_relation(schema: str, table: str) -> Relation:\n",
    "    schema, table = schema.lower(), table.lower()\n",
    "    description = get_description(schema, table)\n",
    "    attributes = [\n",
    "        Attribute(\n",
    "            name=attr_name.capitalize(),\n",
    "            description=get_description(schema, table, attr_name),\n",
    "        ) for attr_name in get_attributes(schema, table)\n",
    "    ]\n",
    "    return Relation(\n",
    "        name=table.capitalize(),\n",
    "        description=description,\n",
    "        attributes=attributes,\n",
    "    )\n",
    "\n",
    "\n",
    "parameters = [\n",
    "    Parameters(\n",
    "        source_relation=get_relation(\"mimic\", source),\n",
    "        target_relation=get_relation(\"omop\", target),\n",
    "    ) for source, target in relation_combinations\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "635d458e-b041-42cf-ab0f-8fb688bf9ada",
   "metadata": {},
   "outputs": [],
   "source": [
    "def abbreviate(source: str, target: str) -> str:\n",
    "    return {\n",
    "        (\"patients\", \"person\"): \"PaPe\",\n",
    "        (\"admissions\", \"visit_occurrence\"): \"AdVO\",\n",
    "        (\"prescriptions\", \"drug_exposure\"): \"PrDE\",\n",
    "        (\"admissions\", \"condition_occurrence\"): \"AdCO\",\n",
    "        (\"diagnoses_icd\", \"condition_occurrence\"): \"DiCO\",\n",
    "        (\"labevents\", \"measurement\"): \"LaMe\",\n",
    "        (\"admissions\", \"visit_detail\"): \"AdVD\",\n",
    "        (\"services\", \"visit_detail\"): \"SeVD\",\n",
    "        (\"transfers\", \"visit_detail\"): \"TrVD\",\n",
    "    }[(source.lower(), target.lower())]\n",
    "\n",
    "\n",
    "dataset_table = []\n",
    "for param in parameters:\n",
    "    dataset_table.append({\n",
    "        \"dataset\": abbreviate(param.source_relation.name, param.target_relation.name),\n",
    "        \"source\": param.source_relation.name,\n",
    "        \"source_attributes\": len(param.source_relation.attributes),\n",
    "        \"target\": param.target_relation.name,\n",
    "        \"target_attributes\": len(param.target_relation.attributes),\n",
    "        \"attribute_pairs\": len(param.source_relation.attributes) * len(param.target_relation.attributes),\n",
    "        \"matches\": benchmark.query((\n",
    "            \"(source_relation.str.lower() == @param.source_relation.name.lower()) and \"\n",
    "            \"(target_relation.str.lower() == @param.target_relation.name.lower()) and \"\n",
    "            \"benchmark\"\n",
    "        )).shape[0],\n",
    "    })\n",
    "dataset_df = pd.DataFrame(dataset_table).set_index(\"dataset\")\n",
    "dataset_df.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bdfd4c8-0f40-4bc3-a0bc-6a4b23865a1e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}