EvaluationCode/evaluations_rerank-covid.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a40dc0a1-265e-46fa-8f92-3ab7d8ddde97",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from tqdm.std import tqdm\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26769814-03ea-498f-9889-5473b0c2e638",
   "metadata": {},
   "source": [
    "## load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bfbc110-8355-423e-b4d4-b46e421ee6dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = 'covid-19/emb_rank/' ## change it to your own path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b151b2c-a165-49fe-bf91-0066929490d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename_list = os.listdir(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c62a422-a99e-4b2c-a6c5-5dbc016d36c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08c5b76b-ddd8-4fbb-b1c4-6643fa343a9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = []\n",
    "for filename in filename_list:\n",
    "    df = pd.read_csv(os.path.join(file_path,filename))\n",
    "    df_list.append(df)\n",
    "\n",
    "covid_df = pd.concat(df_list).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52594392-abaf-4423-a882-679c1d582780",
   "metadata": {},
   "outputs": [],
   "source": [
    "covid_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30a0eb72-75d0-426f-842b-c9e1f2942219",
   "metadata": {},
   "outputs": [],
   "source": [
    "## extract valid colums\n",
    "df = covid_df[[\"Query\",\"GoldName\",\"GoldID\",\"standardConcept\",\"conceptId\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2abaa52e-af54-4a1e-8c3d-40dc9914bb7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.fillna(\"null\",inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aef5d9f7-a40a-4ffe-ac9d-abaad0cc22db",
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11ba815f-14f5-4126-9b61-1243a9bbde1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "## extracting goldId and predId\n",
    "def extract_GoldIdAndPredId(df):\n",
    "    goldIds = []\n",
    "    predIds = []\n",
    "    for row in df.itertuples():\n",
    "        goldId = row.GoldID.split(\"||\")\n",
    "        goldIds.append(goldId)\n",
    "        \n",
    "        predId = row.conceptId\n",
    "        predIds.append([predId])\n",
    "    return goldIds, predIds\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63ecb3d7-472f-4570-a6a0-9ac3f41e5a88",
   "metadata": {},
   "outputs": [],
   "source": [
    "goldIds, predIds = extract_GoldIdAndPredId(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d8bab67-8140-4bcc-9899-01149a3a84cc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99578593-5d90-486b-8803-d29b56eb7fd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_acc_at_k(goldId, predId, k):\n",
    "    correct = 0\n",
    "    for gold, pred in zip(goldId, predId):\n",
    "        if any(p in gold for p in pred[:k]):\n",
    "            correct += 1\n",
    "    return correct / len(goldId)\n",
    "\n",
    "\n",
    "def calculate_precision_recall_f1(goldId, predId):\n",
    "    true_positive = 0\n",
    "    total_predicted = 0\n",
    "    total_gold = 0\n",
    "    \n",
    "    for gold, pred in zip(goldId, predId):\n",
    "        true_positive += len(set(gold) & set(pred))\n",
    "        total_predicted += len(pred)\n",
    "        total_gold += len(gold)\n",
    "    \n",
    "    precision = true_positive / total_predicted if total_predicted > 0 else 0\n",
    "    recall = true_positive / total_gold if total_gold > 0 else 0\n",
    "    f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n",
    "    \n",
    "    return precision, recall, f1\n",
    "\n",
    "\n",
    "def calculate_precision_recall_at_k(goldId, predId, k):\n",
    "    true_positive = 0\n",
    "    total_predicted = 0\n",
    "    total_gold = 0\n",
    "    \n",
    "    for gold, pred in zip(goldId, predId):\n",
    "        true_positive += len(set(gold) & set(pred[:k]))\n",
    "        total_predicted += min(len(pred[:k]), k)\n",
    "        total_gold += len(gold)\n",
    "    \n",
    "    precision = true_positive / total_predicted if total_predicted > 0 else 0\n",
    "    recall = true_positive / total_gold if total_gold > 0 else 0\n",
    "    \n",
    "    return precision, recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bec9de54-e439-4585-9005-b4462c40badf",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_acc = {}\n",
    "\n",
    "p, r, f1 = calculate_precision_recall_f1(goldIds,predIds)\n",
    "total_acc['precision'] = p\n",
    "total_acc['recall'] = r\n",
    "total_acc['f1'] = f1\n",
    "\n",
    "total_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a7126d2-da59-43ce-90f0-5645d934bc11",
   "metadata": {},
   "source": [
    "## remove Mapping not found"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "549cadaf-f930-4402-8b7f-871bb26c3aa8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df[df.GoldID != \"Mapping_Not_Found\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d8466f3-373c-46bf-ae8c-28f7df277aca",
   "metadata": {},
   "outputs": [],
   "source": [
    "goldIds, predIds = extract_GoldIdAndPredId(df1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73721688-1fa8-4123-9b89-1438c9e511fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_total_acc = {}\n",
    "\n",
    "p, r, f1 = calculate_precision_recall_f1(goldIds,predIds)\n",
    "new_total_acc['precision'] = p\n",
    "new_total_acc['recall'] = r\n",
    "new_total_acc['f1'] = f1\n",
    "\n",
    "new_total_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9029a4f5-55f4-4866-ac00-b534b177d967",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf-cuda",
   "language": "python",
   "name": "tf-cuda"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}