notebooks/01-download-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "02192f47-5024-473f-9b3e-e2b78c94bdef",
   "metadata": {},
   "source": [
    "## Periodontal Diease Model ##\n",
    "### Download the model weights and the test data ###\n",
    "Running this notebook requires AWS keys which can be obtained from the authors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0d961bc8-1fdb-4943-9685-c91195ac8a1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Project version:  0.0.post1.dev19+gf9f2c51\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tarfile\n",
    "import glob\n",
    "from pathlib import Path\n",
    "\n",
    "# Imports from this project\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import periomodel\n",
    "from periomodel.fileutils import FileOP\n",
    "from periomodel.imageproc import is_image\n",
    "print(f'Project version:  {periomodel.__version__}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2986ffd-8576-4149-8e0b-82fd2ed82da2",
   "metadata": {},
   "source": [
    "### Some useful functions for this notebook ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5593cf75-fdd5-4c54-9aca-7833120ec160",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_and_extract(url, download_dir):\n",
    "    output_file_path = FileOP().download_from_url(url=url, download_dir=data_dir)\n",
    "    if os.path.isfile(output_file_path):\n",
    "        extract_dir = os.path.join(data_dir, \n",
    "                              os.path.splitext(os.path.basename(output_file_path))[0])\n",
    "        Path(extract_dir).mkdir(parents=True, exist_ok=True)\n",
    "        with tarfile.open(output_file_path) as tar:\n",
    "            tar.extractall(extract_dir)\n",
    "        print(f'Data downloaded and extracted: {output_file_path}')\n",
    "    else:\n",
    "        logger.warning('Download failed.')\n",
    "        extract_dir = None\n",
    "    return extract_dir\n",
    "    \n",
    "def patient_image_stat(data, file_col='file', group_col='group'):\n",
    "    df_n_patients = data[['PatientIDE', group_col]].\\\n",
    "                    drop_duplicates().\\\n",
    "                    groupby(by=group_col).nunique().\\\n",
    "                    reset_index(drop=False).\\\n",
    "                    rename(columns={'PatientIDE': 'n_patients'}).\\\n",
    "                    sort_values(by=group_col, ascending=True).\\\n",
    "                    reset_index(drop=True)\n",
    "\n",
    "    df_n_images = data[[file_col, group_col]].\\\n",
    "                drop_duplicates().\\\n",
    "                groupby(by=group_col).nunique().\\\n",
    "                reset_index(drop=False).\\\n",
    "                rename(columns={file_col: 'n_images'}).\\\n",
    "                sort_values(by=group_col, ascending=True).\\\n",
    "                reset_index(drop=True)\n",
    "\n",
    "    df_stat = df_n_patients.merge(df_n_images, on=group_col, how='inner')\n",
    "    df_stat = df_stat.assign(images_per_patient=np.round(df_stat['n_images']/df_stat['n_patients']).astype(int))\n",
    "\n",
    "    return df_stat"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c217f74d-5bcc-48ca-9c49-9050bc600b26",
   "metadata": {},
   "source": [
    "### Download and extract the test data ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e12a2bcd-85c0-4725-bc2a-7bb66f5c9ac6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test data URL\n",
    "image_url = 'https://dsets.s3.amazonaws.com/classification_datasets/periodata_test.tar.gz'\n",
    "# Model checkpoint\n",
    "model_url = 'https://dsets.s3.amazonaws.com/classification_datasets/periomodel_checkpoint_1400.tar.gz'\n",
    "# Download directory\n",
    "# The DATA_ROOT directory can be set in the .env file (see repository)\n",
    "data_dir = os.environ.get('DATA_ROOT')\n",
    "assert os.path.isdir(data_dir), f'data_dir {data_dir} does not exist.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "167d1d20-fcdd-420b-8796-60a1b3cbf400",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting test data from: https://dsets.s3.amazonaws.com/classification_datasets/periodata_test.tar.gz\n",
      "Extracting from .gz archive.\n",
      "Uncompressed output file exists: /app/data/periodata_test.tar. Skipping.\n",
      "Data downloaded and extracted: /app/data/periodata_test.tar\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PatientIDE</th>\n",
       "      <th>annotation_id</th>\n",
       "      <th>disease</th>\n",
       "      <th>cl3</th>\n",
       "      <th>dset</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a5c42982</td>\n",
       "      <td>829</td>\n",
       "      <td>unstable</td>\n",
       "      <td>2</td>\n",
       "      <td>test</td>\n",
       "      <td>box_a5c42982_20230414_02_00.png</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>956a3988</td>\n",
       "      <td>1758</td>\n",
       "      <td>stable</td>\n",
       "      <td>1</td>\n",
       "      <td>test</td>\n",
       "      <td>box_956a3988_20221108_02_01.png</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>037914b7</td>\n",
       "      <td>1545</td>\n",
       "      <td>unstable</td>\n",
       "      <td>2</td>\n",
       "      <td>test</td>\n",
       "      <td>box_037914b7_20230331_21_03.png</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c0b47799</td>\n",
       "      <td>5703</td>\n",
       "      <td>very unstable</td>\n",
       "      <td>2</td>\n",
       "      <td>test</td>\n",
       "      <td>box_c0b47799_20230208_05_05.png</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2cfc437a</td>\n",
       "      <td>3334</td>\n",
       "      <td>stable</td>\n",
       "      <td>1</td>\n",
       "      <td>test</td>\n",
       "      <td>box_2cfc437a_20230515_11_02.png</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  PatientIDE  annotation_id        disease  cl3  dset  \\\n",
       "0   a5c42982            829       unstable    2  test   \n",
       "1   956a3988           1758         stable    1  test   \n",
       "2   037914b7           1545       unstable    2  test   \n",
       "3   c0b47799           5703  very unstable    2  test   \n",
       "4   2cfc437a           3334         stable    1  test   \n",
       "\n",
       "                             image  \n",
       "0  box_a5c42982_20230414_02_00.png  \n",
       "1  box_956a3988_20221108_02_01.png  \n",
       "2  box_037914b7_20230331_21_03.png  \n",
       "3  box_c0b47799_20230208_05_05.png  \n",
       "4  box_2cfc437a_20230515_11_02.png  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "614 samples in data frame\n",
      "614 images verified in /app/data/periodata_test/test\n",
      "\n",
      "Extracting model data from: https://dsets.s3.amazonaws.com/classification_datasets/periomodel_checkpoint_1400.tar.gz\n",
      "Extracting from .gz archive.\n",
      "Uncompressed output file exists: /app/data/periomodel_checkpoint_1400.tar. Skipping.\n",
      "Data downloaded and extracted: /app/data/periomodel_checkpoint_1400.tar\n",
      "Model checkpoint file: /app/data/periomodel_checkpoint_1400/periomodel_checkpoint_1400.ckpt\n",
      "Training log file:     /app/data/periomodel_checkpoint_1400/train.log\n"
     ]
    }
   ],
   "source": [
    "# Extract and verify the test data sets\n",
    "print(f'Extracting test data from: {image_url}')\n",
    "image_extract_dir = download_and_extract(url=image_url, download_dir=data_dir)\n",
    "if image_extract_dir is not None:\n",
    "    image_dir = os.path.join(image_extract_dir, 'test')\n",
    "    df_file = glob.glob(os.path.join(image_dir, '*.parquet'))\n",
    "    if len(df_file) > 0:\n",
    "        df_file = df_file[0]\n",
    "        file_list = glob.glob(os.path.join(image_dir, '*.png'))\n",
    "        file_list_verified = [file for file in file_list if is_image(file)]\n",
    "        test_df = pd.read_parquet(df_file)\n",
    "        display(test_df.head())\n",
    "        n_files = len(test_df[\"image\"].unique())\n",
    "        print(f'{n_files} samples in data frame')\n",
    "        print(f'{len(file_list_verified)} images verified in {image_dir}')\n",
    "        assert n_files == len(file_list) == len(file_list_verified), 'WARNING, could not verify all files'\n",
    "    else:\n",
    "        logger.warning(f'Missing .parquet data file in: {image_dir}')\n",
    "\n",
    "# Download and extract the model weights\n",
    "print()\n",
    "print(f'Extracting model data from: {model_url}')\n",
    "model_dir = download_and_extract(url=model_url, download_dir=data_dir)\n",
    "if model_dir is not None:\n",
    "    try:\n",
    "        checkpoint_file = glob.glob(os.path.join(model_dir, '*.ckpt'))[0]\n",
    "        log_file = glob.glob(os.path.join(model_dir, '*.log'))[0]\n",
    "    except IndexError:\n",
    "        logger.warning('.ckpt or .log file not found.')\n",
    "    print(f'Model checkpoint file: {checkpoint_file}')\n",
    "    print(f'Training log file:     {log_file}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d878449a-73fe-4956-92d8-7548d753474b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cl3': {np.int64(0): ('healthy',),\n",
       "  np.int64(1): ('stable',),\n",
       "  np.int64(2): ('unstable', 'very unstable')}}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'cl3': {'healthy': 0, 'stable': 1, 'unstable': 2, 'very unstable': 2}}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Label assignments in data set\n",
    "label_dict = {}\n",
    "label_dict_inv = {}\n",
    "label_col_list = ['cl3']\n",
    "for label_col in label_col_list:\n",
    "    cl_list = sorted(list(test_df.get(label_col).unique()))\n",
    "    ds_list = [tuple(test_df.loc[test_df[label_col]==cl, 'disease'].unique()) for cl in cl_list]\n",
    "    label_cl_dict = dict(zip(cl_list, ds_list))\n",
    "    label_dict.update({label_col: label_cl_dict})\n",
    "    \n",
    "    ds_list_inv = sorted(list(test_df.get('disease').unique()))\n",
    "    cl_list_inv = [int(test_df.loc[test_df['disease'] == disease, label_col].\\\n",
    "                       values[0]) for disease in ds_list_inv]\n",
    "    label_cl_dict_inv = dict(zip(ds_list_inv, cl_list_inv))\n",
    "    label_dict_inv.update({label_col: label_cl_dict_inv})\n",
    "display(label_dict)\n",
    "print()\n",
    "display(label_dict_inv)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08fd3587-e1cf-40e1-ae09-8f4834ac80f3",
   "metadata": {},
   "source": [
    "### Summary statistics for the test data set ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3f69af99-0f01-4e16-974c-f23ec28542b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TEST: 614 IMAGES\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>disease</th>\n",
       "      <th>n_patients</th>\n",
       "      <th>n_images</th>\n",
       "      <th>images_per_patient</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>healthy</td>\n",
       "      <td>20</td>\n",
       "      <td>120</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>stable</td>\n",
       "      <td>25</td>\n",
       "      <td>283</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unstable</td>\n",
       "      <td>16</td>\n",
       "      <td>133</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>very unstable</td>\n",
       "      <td>4</td>\n",
       "      <td>78</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         disease  n_patients  n_images  images_per_patient\n",
       "0        healthy          20       120                   6\n",
       "1         stable          25       283                  11\n",
       "2       unstable          16       133                   8\n",
       "3  very unstable           4        78                  20"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TEST IMAGES FOR CL3 MODEL\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cl3</th>\n",
       "      <th>n_patients</th>\n",
       "      <th>n_images</th>\n",
       "      <th>images_per_patient</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>120</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>283</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>211</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cl3  n_patients  n_images  images_per_patient\n",
       "0    0          20       120                   6\n",
       "1    1          25       283                  11\n",
       "2    2          16       211                  13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Image data used for these models\n",
    "dset_list = sorted(list(test_df.get('dset').unique()))\n",
    "for dset in dset_list:\n",
    "    df_dset = test_df.loc[test_df['dset'] == dset]\n",
    "    dset_n_images = len(df_dset['image'].unique())\n",
    "    print(f'{dset.upper()}: {dset_n_images} IMAGES')\n",
    "    disease_stat = patient_image_stat(df_dset, file_col='image', group_col='disease')\n",
    "    display(disease_stat)\n",
    "    for label_col in ['cl3']:\n",
    "        cl_stat = patient_image_stat(df_dset, file_col='image', group_col=label_col)\n",
    "        print(f'{dset.upper()} IMAGES FOR {label_col.upper()} MODEL')\n",
    "        display(cl_stat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13058ac7-88d5-4f64-84c4-e5fc6c6907bd",
   "metadata": {},
   "source": [
    "### Summary output for this notebook ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "37aed0ac-8a2a-41eb-8208-6cad04a6662a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image files verified: 614\n",
      "Image file location:  /app/data/periodata_test/test\n",
      "Label file:           /app/data/periodata_test/test/perimodel_labels_test.parquet\n",
      "Checkpoint:     /app/data/periomodel_checkpoint_1400/periomodel_checkpoint_1400.ckpt\n",
      "Log file:       /app/data/periomodel_checkpoint_1400/train.log\n"
     ]
    }
   ],
   "source": [
    "# This cell should run without errors.\n",
    "if image_extract_dir is not None:\n",
    "    print(f'Image files verified: {len(file_list_verified)}')\n",
    "    print(f'Image file location:  {image_dir}')\n",
    "    print(f'Label file:           {df_file}')\n",
    "if model_dir is not None:\n",
    "    print(f'Checkpoint:     {checkpoint_file}')\n",
    "    print(f'Log file:       {log_file}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a29ba55f-6d19-45a3-a361-d782a8d11c21",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}