scripts/preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f90c34bf-ee43-4d01-901c-b1b7f3dc711a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7f3d80da-04bd-4ef0-9b4e-3ad569bea89b",
   "metadata": {},
   "outputs": [],
   "source": [
    "excel_dict = pd.read_excel(r\"..\\data files\\Pubmed5k.xlsx\", sheet_name=None)\n",
    "# excel_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0b667ba4-03f2-4dae-a1bc-f4e25c119f6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ArticleID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>34153941</td>\n",
       "      <td>Stable Coordination Variability in Overground ...</td>\n",
       "      <td>Coordination variability (CV) is commonly anal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34153942</td>\n",
       "      <td>Weak Hip Strength Increases Dynamic Knee Valgu...</td>\n",
       "      <td>Clinical Scenario: Dynamic knee valgus (DKV) i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>34153964</td>\n",
       "      <td>Current and Future Projections of Amyotrophic ...</td>\n",
       "      <td>Various methodologies have been reported to as...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34153968</td>\n",
       "      <td>Disparities between Asian and Non-Asian Thromb...</td>\n",
       "      <td>As outcomes for acute ischemic stroke (AIS) va...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>34153978</td>\n",
       "      <td>Maternal Factors Predicting Loss to Follow-Up ...</td>\n",
       "      <td>Because hearing loss in children can result in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4994</th>\n",
       "      <td>34444567</td>\n",
       "      <td>Mind the Differences: How Diagnoses and Hospit...</td>\n",
       "      <td>Integrated care pathway (ICP) is a prevailing ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4995</th>\n",
       "      <td>34444568</td>\n",
       "      <td>The Ethics of Dying: Deciphering Pandemic-Resu...</td>\n",
       "      <td>The objective of medicine is to provide humans...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4996</th>\n",
       "      <td>34444569</td>\n",
       "      <td>Research on Adolescents Regarding the Indirect...</td>\n",
       "      <td>This research involved the participation of 30...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4997</th>\n",
       "      <td>34444571</td>\n",
       "      <td>Pre-Intervention Effects of a Community-Based ...</td>\n",
       "      <td>This study explores the impact of the 'pre-int...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4998</th>\n",
       "      <td>34444572</td>\n",
       "      <td>Pharmacists' Role in Older Adults' Medication ...</td>\n",
       "      <td>Medication regimen complexity (MRC) may influe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4999 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ArticleID                                              Title  \\\n",
       "0      34153941  Stable Coordination Variability in Overground ...   \n",
       "1      34153942  Weak Hip Strength Increases Dynamic Knee Valgu...   \n",
       "2      34153964  Current and Future Projections of Amyotrophic ...   \n",
       "3      34153968  Disparities between Asian and Non-Asian Thromb...   \n",
       "4      34153978  Maternal Factors Predicting Loss to Follow-Up ...   \n",
       "...         ...                                                ...   \n",
       "4994   34444567  Mind the Differences: How Diagnoses and Hospit...   \n",
       "4995   34444568  The Ethics of Dying: Deciphering Pandemic-Resu...   \n",
       "4996   34444569  Research on Adolescents Regarding the Indirect...   \n",
       "4997   34444571  Pre-Intervention Effects of a Community-Based ...   \n",
       "4998   34444572  Pharmacists' Role in Older Adults' Medication ...   \n",
       "\n",
       "                                               Abstract  \n",
       "0     Coordination variability (CV) is commonly anal...  \n",
       "1     Clinical Scenario: Dynamic knee valgus (DKV) i...  \n",
       "2     Various methodologies have been reported to as...  \n",
       "3     As outcomes for acute ischemic stroke (AIS) va...  \n",
       "4     Because hearing loss in children can result in...  \n",
       "...                                                 ...  \n",
       "4994  Integrated care pathway (ICP) is a prevailing ...  \n",
       "4995  The objective of medicine is to provide humans...  \n",
       "4996  This research involved the participation of 30...  \n",
       "4997  This study explores the impact of the 'pre-int...  \n",
       "4998  Medication regimen complexity (MRC) may influe...  \n",
       "\n",
       "[4999 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "excel_dict[\"random 5k\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "070d0b3f-a698-4417-a437-1ab166216c32",
   "metadata": {},
   "outputs": [],
   "source": [
    "excel_dict[\"random 5k\"].to_csv(r\"..\\data files\\Pubmed5k.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f5ad554d-4efd-434d-9352-b8be16c5a254",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Abstract</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArticleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>34153941</th>\n",
       "      <td>Stable Coordination Variability in Overground ...</td>\n",
       "      <td>Coordination variability (CV) is commonly anal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153942</th>\n",
       "      <td>Weak Hip Strength Increases Dynamic Knee Valgu...</td>\n",
       "      <td>Clinical Scenario: Dynamic knee valgus (DKV) i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153964</th>\n",
       "      <td>Current and Future Projections of Amyotrophic ...</td>\n",
       "      <td>Various methodologies have been reported to as...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153968</th>\n",
       "      <td>Disparities between Asian and Non-Asian Thromb...</td>\n",
       "      <td>As outcomes for acute ischemic stroke (AIS) va...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153978</th>\n",
       "      <td>Maternal Factors Predicting Loss to Follow-Up ...</td>\n",
       "      <td>Because hearing loss in children can result in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444567</th>\n",
       "      <td>Mind the Differences: How Diagnoses and Hospit...</td>\n",
       "      <td>Integrated care pathway (ICP) is a prevailing ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444568</th>\n",
       "      <td>The Ethics of Dying: Deciphering Pandemic-Resu...</td>\n",
       "      <td>The objective of medicine is to provide humans...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444569</th>\n",
       "      <td>Research on Adolescents Regarding the Indirect...</td>\n",
       "      <td>This research involved the participation of 30...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444571</th>\n",
       "      <td>Pre-Intervention Effects of a Community-Based ...</td>\n",
       "      <td>This study explores the impact of the 'pre-int...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444572</th>\n",
       "      <td>Pharmacists' Role in Older Adults' Medication ...</td>\n",
       "      <td>Medication regimen complexity (MRC) may influe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4999 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       Title  \\\n",
       "ArticleID                                                      \n",
       "34153941   Stable Coordination Variability in Overground ...   \n",
       "34153942   Weak Hip Strength Increases Dynamic Knee Valgu...   \n",
       "34153964   Current and Future Projections of Amyotrophic ...   \n",
       "34153968   Disparities between Asian and Non-Asian Thromb...   \n",
       "34153978   Maternal Factors Predicting Loss to Follow-Up ...   \n",
       "...                                                      ...   \n",
       "34444567   Mind the Differences: How Diagnoses and Hospit...   \n",
       "34444568   The Ethics of Dying: Deciphering Pandemic-Resu...   \n",
       "34444569   Research on Adolescents Regarding the Indirect...   \n",
       "34444571   Pre-Intervention Effects of a Community-Based ...   \n",
       "34444572   Pharmacists' Role in Older Adults' Medication ...   \n",
       "\n",
       "                                                    Abstract  \n",
       "ArticleID                                                     \n",
       "34153941   Coordination variability (CV) is commonly anal...  \n",
       "34153942   Clinical Scenario: Dynamic knee valgus (DKV) i...  \n",
       "34153964   Various methodologies have been reported to as...  \n",
       "34153968   As outcomes for acute ischemic stroke (AIS) va...  \n",
       "34153978   Because hearing loss in children can result in...  \n",
       "...                                                      ...  \n",
       "34444567   Integrated care pathway (ICP) is a prevailing ...  \n",
       "34444568   The objective of medicine is to provide humans...  \n",
       "34444569   This research involved the participation of 30...  \n",
       "34444571   This study explores the impact of the 'pre-int...  \n",
       "34444572   Medication regimen complexity (MRC) may influe...  \n",
       "\n",
       "[4999 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(r\"..\\data files\\Pubmed5k.csv\", index_col=\"ArticleID\")\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85ed18aa-c961-4c95-b229-93f891b8fdde",
   "metadata": {},
   "source": [
    "# EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c6f215da-e78a-4f70-a6a3-f6f915076710",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4999, 2)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ef0bd3f6-2c32-473d-b6f8-6813260286a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4999"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ensure that the column \"ArticleID\" is really valid to be an ID and not a fake name\n",
    "df.index.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4995d159-da28-493e-8aa1-067e84a08de1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 4999 entries, 34153941 to 34444572\n",
      "Data columns (total 2 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   Title     4999 non-null   object\n",
      " 1   Abstract  4999 non-null   object\n",
      "dtypes: object(2)\n",
      "memory usage: 246.2+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ef89b88c-6f1c-451c-8d07-8117a6b77bfc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Title       0.0\n",
       "Abstract    0.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9df16176-2480-4cf2-a2fa-40b530cb3366",
   "metadata": {},
   "source": [
    "#### there is no null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c811dc53-dd5a-4e84-8743-5ccf59c2fa75",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Title       4999\n",
       "Abstract    4989\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f960c875-eaec-4b12-b1df-161ecc988e36",
   "metadata": {},
   "source": [
    "#### there are 10 values repeated in the column of \"Abstract\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6380eb34-32da-4b8f-8a5c-f824eb47b268",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4999</td>\n",
       "      <td>4999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>4999</td>\n",
       "      <td>4989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>Stable Coordination Variability in Overground ...</td>\n",
       "      <td>[Figure: see text].</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    Title             Abstract\n",
       "count                                                4999                 4999\n",
       "unique                                               4999                 4989\n",
       "top     Stable Coordination Variability in Overground ...  [Figure: see text].\n",
       "freq                                                    1                    6"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "68d1357d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Abstract</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArticleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Title, Abstract]\n",
       "Index: []"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.duplicated()]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7bd5040-f910-4d4f-82db-f5b4136a1401",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7cb552f-7549-46c6-8613-d5b649f7c6b9",
   "metadata": {},
   "source": [
    "Text Analysis is a major application field for machine learning algorithms. However the raw data, a sequence of symbols cannot be fed directly to the algorithms themselves as most of them expect numerical feature vectors with a fixed size rather than the raw text documents with variable length."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e4c1bd0-423d-4e7e-af2a-ba49fd69b16f",
   "metadata": {},
   "source": [
    "We will perform the following text preprocessing steps:-\n",
    "- Convert the text into lowercase\n",
    "- Split text into words (Tokenize)\n",
    "- Remove the stop words\n",
    "- Remove the Punctuation, any symbols, and special characters\n",
    "- Normalize the word (I’ll be using Lemmatization for normalization)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a710609c",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'scispacy'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_5296/4240021308.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# to replace abbreviation with it's original text\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mspacy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mscispacy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mscispacy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mabbreviation\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mAbbreviationDetector\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'scispacy'"
     ]
    }
   ],
   "source": [
    "# to replace abbreviation with it's original text\n",
    "import spacy\n",
    "import scispacy\n",
    "from scispacy.abbreviation import AbbreviationDetector\n",
    "\n",
    "nlp = spacy.load('en_core_sci_lg')\n",
    "nlp.add_pipe(\"abbreviation_detector\")\n",
    "\n",
    "def replace_acronyms(text):\n",
    "    doc = nlp(text)\n",
    "    altered_tok = [tok.text for tok in doc]\n",
    "    for abrv in doc._.abbreviations:\n",
    "        altered_tok[abrv.start] = str(abrv._.long_form)\n",
    "\n",
    "    return(\" \".join(altered_tok))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f19d8530-2a14-4d13-937d-234c72bd4de6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string\n",
    "import nltk\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "\n",
    "# nltk lemmatization method\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "# nltk.download('wordnet')\n",
    "# nltk.download('omw-1.4')\n",
    "\n",
    "# nltk.download('words')\n",
    "# eng_words = set(nltk.corpus.words.words())\n",
    "eng_words = list(nlp.vocab.strings)\n",
    "\n",
    "# # nltk method for tokenization\n",
    "# from nltk.tokenize import word_tokenize\n",
    "# nltk.download('punkt')\n",
    "\n",
    "\n",
    "# stop words\n",
    "# i make stop words is the set of the stop words from the 4 libraries to increase number of stop words, i.e accept all the definitions\n",
    "# of the stop words from all libraries to remove higher numbers of stop words\n",
    "# nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords \n",
    "from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS\n",
    "from gensim.parsing.preprocessing import STOPWORDS \n",
    "stop_words = list(set(list(nlp.Defaults.stop_words)+         #spacy\n",
    "                      list(ENGLISH_STOP_WORDS)+              #scikit-learn\n",
    "                      list(STOPWORDS)+                       #gensim\n",
    "                      list(stopwords.words('english'))))     #nltk\n",
    "\n",
    "\n",
    "wnl = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c19390f3-aee4-48d7-ad05-6ad3d8945bf1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# this step may take alot of time, you can reduce the mount of time by \n",
    "# using \"\"\"nlp = spacy.load('en_core_sci_sm')\"\"\" instead of \"\"\"nlp = spacy.load('en_core_sci_lg')\"\"\" that i used\n",
    "\n",
    "def clean_txt(txt):\n",
    "    \"\"\"this function is to clean the text\"\"\"\n",
    "    \n",
    "    # replace the abbreviation wit it's origin\n",
    "    txt = replace_acronyms(txt)\n",
    "    \n",
    "    # remove punctuation\n",
    "    txt = re.sub('\\W+',' ', txt) \n",
    "    # txt = (txt.translate(str.maketrans('', '', string.punctuation))) # remove the punctuation, it's the fastest method \n",
    "    \n",
    "    # lower all characters\n",
    "    txt = txt.lower()\n",
    "    \n",
    "    # splite the words after every space(tokenize)\n",
    "    txt = txt.split()    \n",
    "    # txt = word_tokenize(txt) # nltk tokenization alternative\n",
    "\n",
    "    # remove stop words and do lemmatization\n",
    "    txt = [wnl.lemmatize(word) for word in txt if ((word.isalpha()) and (word not in stop_words))]    \n",
    "    \n",
    "    # remove non-english words\n",
    "    # this step may take alot of time, you can reduce the mount of time by \n",
    "    # using \"\"\"nlp = spacy.load('en_core_sci_sm')\"\"\" instead of \"\"\"nlp = spacy.load('en_core_sci_lg')\"\"\" that i used\n",
    "    # txt = [w for w in txt if w in eng_words]\n",
    "    \n",
    "    # convert list of tokens to text\n",
    "    # txt = \" \".join(txt)\n",
    "    \n",
    "    # returning the result\n",
    "    return txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "86979dd4-a770-4efc-8e18-e9fdc7aeb65e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|▏                                                                               | 12/4999 [00:03<09:38,  8.62it/s]C:\\Users\\huzyfa\\anaconda3\\envs\\mp\\lib\\site-packages\\scispacy\\abbreviation.py:230: UserWarning: [W036] The component 'matcher' does not have any patterns defined.\n",
      "  global_matches = self.global_matcher(doc)\n",
      "100%|██████████████████████████████████████████████████████████████████████████████| 4999/4999 [04:33<00:00, 18.26it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ArticleID\n",
       "34153941    [coordination, variability, coordination, vari...\n",
       "34153942    [clinical, scenario, dynamic, knee, valgus, dy...\n",
       "34153964    [methodology, reported, ass, real, world, epid...\n",
       "34153968    [outcome, acute, ischemic, stroke, acute, isch...\n",
       "34153978    [hearing, loss, child, result, developmental, ...\n",
       "Name: Abstract_processed, dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# this step may take alot of time, you can reduce the mount of time by \n",
    "# using \"\"\"nlp = spacy.load('en_core_sci_sm')\"\"\" instead of \"\"\"nlp = spacy.load('en_core_sci_lg')\"\"\" that i used,\n",
    "\n",
    "# if you find that this take along time, i did this step and save the results as a csv file, so you can use it directly in the next cell\n",
    "df[\"Abstract_processed\"] = [clean_txt(txt) for txt in tqdm(df.Abstract)]\n",
    "df[\"Abstract_processed\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "8f1f8599-7247-4c9e-95ed-11d301cd7bf3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Abstract_processed</th>\n",
       "      <th>words_count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArticleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>34153941</th>\n",
       "      <td>Stable Coordination Variability in Overground ...</td>\n",
       "      <td>Coordination variability (CV) is commonly anal...</td>\n",
       "      <td>['coordination', 'variability', 'coordination'...</td>\n",
       "      <td>1421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153942</th>\n",
       "      <td>Weak Hip Strength Increases Dynamic Knee Valgu...</td>\n",
       "      <td>Clinical Scenario: Dynamic knee valgus (DKV) i...</td>\n",
       "      <td>['clinical', 'scenario', 'dynamic', 'knee', 'v...</td>\n",
       "      <td>1941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153964</th>\n",
       "      <td>Current and Future Projections of Amyotrophic ...</td>\n",
       "      <td>Various methodologies have been reported to as...</td>\n",
       "      <td>['methodology', 'reported', 'ass', 'real', 'wo...</td>\n",
       "      <td>1821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153968</th>\n",
       "      <td>Disparities between Asian and Non-Asian Thromb...</td>\n",
       "      <td>As outcomes for acute ischemic stroke (AIS) va...</td>\n",
       "      <td>['outcome', 'acute', 'ischemic', 'stroke', 'ac...</td>\n",
       "      <td>2358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153978</th>\n",
       "      <td>Maternal Factors Predicting Loss to Follow-Up ...</td>\n",
       "      <td>Because hearing loss in children can result in...</td>\n",
       "      <td>['hearing', 'loss', 'child', 'result', 'develo...</td>\n",
       "      <td>1339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444567</th>\n",
       "      <td>Mind the Differences: How Diagnoses and Hospit...</td>\n",
       "      <td>Integrated care pathway (ICP) is a prevailing ...</td>\n",
       "      <td>['integrated', 'care', 'pathway', 'integrated'...</td>\n",
       "      <td>1456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444568</th>\n",
       "      <td>The Ethics of Dying: Deciphering Pandemic-Resu...</td>\n",
       "      <td>The objective of medicine is to provide humans...</td>\n",
       "      <td>['objective', 'medicine', 'provide', 'human', ...</td>\n",
       "      <td>1286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444569</th>\n",
       "      <td>Research on Adolescents Regarding the Indirect...</td>\n",
       "      <td>This research involved the participation of 30...</td>\n",
       "      <td>['research', 'involved', 'participation', 'chi...</td>\n",
       "      <td>850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444571</th>\n",
       "      <td>Pre-Intervention Effects of a Community-Based ...</td>\n",
       "      <td>This study explores the impact of the 'pre-int...</td>\n",
       "      <td>['study', 'explores', 'impact', 'pre', 'interv...</td>\n",
       "      <td>1653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444572</th>\n",
       "      <td>Pharmacists' Role in Older Adults' Medication ...</td>\n",
       "      <td>Medication regimen complexity (MRC) may influe...</td>\n",
       "      <td>['medication', 'regimen', 'complexity', 'medic...</td>\n",
       "      <td>1064</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4989 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       Title  \\\n",
       "ArticleID                                                      \n",
       "34153941   Stable Coordination Variability in Overground ...   \n",
       "34153942   Weak Hip Strength Increases Dynamic Knee Valgu...   \n",
       "34153964   Current and Future Projections of Amyotrophic ...   \n",
       "34153968   Disparities between Asian and Non-Asian Thromb...   \n",
       "34153978   Maternal Factors Predicting Loss to Follow-Up ...   \n",
       "...                                                      ...   \n",
       "34444567   Mind the Differences: How Diagnoses and Hospit...   \n",
       "34444568   The Ethics of Dying: Deciphering Pandemic-Resu...   \n",
       "34444569   Research on Adolescents Regarding the Indirect...   \n",
       "34444571   Pre-Intervention Effects of a Community-Based ...   \n",
       "34444572   Pharmacists' Role in Older Adults' Medication ...   \n",
       "\n",
       "                                                    Abstract  \\\n",
       "ArticleID                                                      \n",
       "34153941   Coordination variability (CV) is commonly anal...   \n",
       "34153942   Clinical Scenario: Dynamic knee valgus (DKV) i...   \n",
       "34153964   Various methodologies have been reported to as...   \n",
       "34153968   As outcomes for acute ischemic stroke (AIS) va...   \n",
       "34153978   Because hearing loss in children can result in...   \n",
       "...                                                      ...   \n",
       "34444567   Integrated care pathway (ICP) is a prevailing ...   \n",
       "34444568   The objective of medicine is to provide humans...   \n",
       "34444569   This research involved the participation of 30...   \n",
       "34444571   This study explores the impact of the 'pre-int...   \n",
       "34444572   Medication regimen complexity (MRC) may influe...   \n",
       "\n",
       "                                          Abstract_processed  words_count  \n",
       "ArticleID                                                                  \n",
       "34153941   ['coordination', 'variability', 'coordination'...         1421  \n",
       "34153942   ['clinical', 'scenario', 'dynamic', 'knee', 'v...         1941  \n",
       "34153964   ['methodology', 'reported', 'ass', 'real', 'wo...         1821  \n",
       "34153968   ['outcome', 'acute', 'ischemic', 'stroke', 'ac...         2358  \n",
       "34153978   ['hearing', 'loss', 'child', 'result', 'develo...         1339  \n",
       "...                                                      ...          ...  \n",
       "34444567   ['integrated', 'care', 'pathway', 'integrated'...         1456  \n",
       "34444568   ['objective', 'medicine', 'provide', 'human', ...         1286  \n",
       "34444569   ['research', 'involved', 'participation', 'chi...          850  \n",
       "34444571   ['study', 'explores', 'impact', 'pre', 'interv...         1653  \n",
       "34444572   ['medication', 'regimen', 'complexity', 'medic...         1064  \n",
       "\n",
       "[4989 rows x 4 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df = pd.read_csv(r\"..\\data files\\Pubmed5k_processed.csv\", index_col=\"ArticleID\")\n",
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "79500d59-a370-43ce-8f3b-d595a24b9ca5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Abstract_processed</th>\n",
       "      <th>words_count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArticleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Title, Abstract, Abstract_processed, words_count]\n",
       "Index: []"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['Abstract_processed'].duplicated(keep=False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "38249597-8284-464e-84b0-b990b6078b9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(df[df['Abstract_processed'].duplicated()].index, axis=0, inplace=True)\n",
    "# df.drop_duplicates(subset=\"Abstract_processed\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "91f3d92b-e349-4d03-887c-a51ee7f168f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1421, 1941, 1821, ..., 2037,  963, 1824], dtype=int64)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"words_count\"] = [len(i) for i in df['Abstract_processed']]\n",
    "df[\"words_count\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "3e37f950-3c6f-4538-9f90-dd0cd315c804",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4989, 4)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "aa556e57-127e-494d-a66f-440abcd30c06",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(df[df.words_count<20].index, axis=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "22af9171-1ccc-475e-9961-5001cf6c3e60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4985, 4)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e52939d1-fbbf-4554-ac8c-6b2a96678d86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1421, 1941, 1821, ..., 2037,  963, 1824], dtype=int64)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"words_count\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "56f357bd-767a-44f7-b7dd-7723c6550e35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Abstract_processed</th>\n",
       "      <th>words_count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ArticleID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>34153941</th>\n",
       "      <td>Stable Coordination Variability in Overground ...</td>\n",
       "      <td>Coordination variability (CV) is commonly anal...</td>\n",
       "      <td>['coordination', 'variability', 'coordination'...</td>\n",
       "      <td>1421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153942</th>\n",
       "      <td>Weak Hip Strength Increases Dynamic Knee Valgu...</td>\n",
       "      <td>Clinical Scenario: Dynamic knee valgus (DKV) i...</td>\n",
       "      <td>['clinical', 'scenario', 'dynamic', 'knee', 'v...</td>\n",
       "      <td>1941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153964</th>\n",
       "      <td>Current and Future Projections of Amyotrophic ...</td>\n",
       "      <td>Various methodologies have been reported to as...</td>\n",
       "      <td>['methodology', 'reported', 'ass', 'real', 'wo...</td>\n",
       "      <td>1821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153968</th>\n",
       "      <td>Disparities between Asian and Non-Asian Thromb...</td>\n",
       "      <td>As outcomes for acute ischemic stroke (AIS) va...</td>\n",
       "      <td>['outcome', 'acute', 'ischemic', 'stroke', 'ac...</td>\n",
       "      <td>2358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34153978</th>\n",
       "      <td>Maternal Factors Predicting Loss to Follow-Up ...</td>\n",
       "      <td>Because hearing loss in children can result in...</td>\n",
       "      <td>['hearing', 'loss', 'child', 'result', 'develo...</td>\n",
       "      <td>1339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444567</th>\n",
       "      <td>Mind the Differences: How Diagnoses and Hospit...</td>\n",
       "      <td>Integrated care pathway (ICP) is a prevailing ...</td>\n",
       "      <td>['integrated', 'care', 'pathway', 'integrated'...</td>\n",
       "      <td>1456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444568</th>\n",
       "      <td>The Ethics of Dying: Deciphering Pandemic-Resu...</td>\n",
       "      <td>The objective of medicine is to provide humans...</td>\n",
       "      <td>['objective', 'medicine', 'provide', 'human', ...</td>\n",
       "      <td>1286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444569</th>\n",
       "      <td>Research on Adolescents Regarding the Indirect...</td>\n",
       "      <td>This research involved the participation of 30...</td>\n",
       "      <td>['research', 'involved', 'participation', 'chi...</td>\n",
       "      <td>850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444571</th>\n",
       "      <td>Pre-Intervention Effects of a Community-Based ...</td>\n",
       "      <td>This study explores the impact of the 'pre-int...</td>\n",
       "      <td>['study', 'explores', 'impact', 'pre', 'interv...</td>\n",
       "      <td>1653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34444572</th>\n",
       "      <td>Pharmacists' Role in Older Adults' Medication ...</td>\n",
       "      <td>Medication regimen complexity (MRC) may influe...</td>\n",
       "      <td>['medication', 'regimen', 'complexity', 'medic...</td>\n",
       "      <td>1064</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4985 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       Title  \\\n",
       "ArticleID                                                      \n",
       "34153941   Stable Coordination Variability in Overground ...   \n",
       "34153942   Weak Hip Strength Increases Dynamic Knee Valgu...   \n",
       "34153964   Current and Future Projections of Amyotrophic ...   \n",
       "34153968   Disparities between Asian and Non-Asian Thromb...   \n",
       "34153978   Maternal Factors Predicting Loss to Follow-Up ...   \n",
       "...                                                      ...   \n",
       "34444567   Mind the Differences: How Diagnoses and Hospit...   \n",
       "34444568   The Ethics of Dying: Deciphering Pandemic-Resu...   \n",
       "34444569   Research on Adolescents Regarding the Indirect...   \n",
       "34444571   Pre-Intervention Effects of a Community-Based ...   \n",
       "34444572   Pharmacists' Role in Older Adults' Medication ...   \n",
       "\n",
       "                                                    Abstract  \\\n",
       "ArticleID                                                      \n",
       "34153941   Coordination variability (CV) is commonly anal...   \n",
       "34153942   Clinical Scenario: Dynamic knee valgus (DKV) i...   \n",
       "34153964   Various methodologies have been reported to as...   \n",
       "34153968   As outcomes for acute ischemic stroke (AIS) va...   \n",
       "34153978   Because hearing loss in children can result in...   \n",
       "...                                                      ...   \n",
       "34444567   Integrated care pathway (ICP) is a prevailing ...   \n",
       "34444568   The objective of medicine is to provide humans...   \n",
       "34444569   This research involved the participation of 30...   \n",
       "34444571   This study explores the impact of the 'pre-int...   \n",
       "34444572   Medication regimen complexity (MRC) may influe...   \n",
       "\n",
       "                                          Abstract_processed  words_count  \n",
       "ArticleID                                                                  \n",
       "34153941   ['coordination', 'variability', 'coordination'...         1421  \n",
       "34153942   ['clinical', 'scenario', 'dynamic', 'knee', 'v...         1941  \n",
       "34153964   ['methodology', 'reported', 'ass', 'real', 'wo...         1821  \n",
       "34153968   ['outcome', 'acute', 'ischemic', 'stroke', 'ac...         2358  \n",
       "34153978   ['hearing', 'loss', 'child', 'result', 'develo...         1339  \n",
       "...                                                      ...          ...  \n",
       "34444567   ['integrated', 'care', 'pathway', 'integrated'...         1456  \n",
       "34444568   ['objective', 'medicine', 'provide', 'human', ...         1286  \n",
       "34444569   ['research', 'involved', 'participation', 'chi...          850  \n",
       "34444571   ['study', 'explores', 'impact', 'pre', 'interv...         1653  \n",
       "34444572   ['medication', 'regimen', 'complexity', 'medic...         1064  \n",
       "\n",
       "[4985 rows x 4 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.to_csv(r\"..\\data files\\Pubmed5k_processed.csv\", index=True, header=True)\n",
    "df = pd.read_csv(r\"..\\data files\\Pubmed5k_processed.csv\", index_col=\"ArticleID\")\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94d3bc7c-78f8-46c9-b240-4dd376a78a6d",
   "metadata": {},
   "source": [
    "### This script written at April-2022 by Ahmad salama\n",
    "- salama4ai@gmail.com\n",
    "- www.linkedin.com/in/salama4ai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d125262-2f39-4a86-9b1f-a386c25e4bba",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}