1_xval_model_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "861406e6",
   "metadata": {},
   "source": [
    "# This notebook trains a model with cross-val on the entire dataset\n",
    "- Gets pred_probs on holdout and saves results as numpy files\n",
    "- Make sure you run ``preprocess_data`` and ``create_labels_df`` on local and push/pull newest ``cifar10_test_consensus_dataset`` first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd34358d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load_ext autoreload\n",
    "# %autoreload 2\n",
    "\n",
    "import sys\n",
    "\n",
    "sys.path.insert(0, \"../\")\n",
    "\n",
    "from autogluon.vision import ImagePredictor, ImageDataset\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import datetime\n",
    "from pathlib import Path\n",
    "import cleanlab\n",
    "from utils.cross_validation_autogluon import cross_val_predict_autogluon_image_dataset\n",
    "\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f4af41e",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "52251f3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  image\n",
       "0      0     15\n",
       "1      1     15\n",
       "2      2     15\n",
       "3      3     15\n",
       "4      4     15\n",
       "5      5     15\n",
       "6      6     15\n",
       "7      7     15\n",
       "8      8     15\n",
       "9      9     15"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load consensus test files\n",
    "data_filepath = './data/benchmark_data/cifar10_test_consensus_dataset.csv'\n",
    "df = pd.read_csv(data_filepath)\n",
    "\n",
    "# Create mini train dataset for testing\n",
    "num_from_each_group = 15\n",
    "mini_df = df.groupby(\"label\").head(num_from_each_group)\n",
    "mini_df.groupby(\"label\")[\"image\"].count().reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a1e1497",
   "metadata": {},
   "source": [
    "**Model and data saving params**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7b09f51a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save/load folders\n",
    "model_folder = './data/cifar10_consensus_worst_25_coin20' # + [model_type]\n",
    "\n",
    "# generate cross-validated predicted probabilities for various models\n",
    "models = [\n",
    "    \"resnet18\",\n",
    "    \"swin_base_patch4_window7_224\"\n",
    "]\n",
    "\n",
    "# xvalidation parameters\n",
    "num_cv_folds = 5 # number K in stratified K-folds cross-validation\n",
    "verbose = 1 # verbose for dataloading duing crossval to numpy save\n",
    "\n",
    "# shared model parameters\n",
    "epochs = 100 #100\n",
    "holdout_frac = 0.2\n",
    "time_limit = 21600 #21600\n",
    "random_state = 123"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b541971a",
   "metadata": {},
   "source": [
    "## Run cross validation on `models`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e793c24",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "modified configs(<old> != <new>): {\n",
      "root.img_cls.model   resnet101 != resnet18\n",
      "root.train.early_stop_patience -1 != 10\n",
      "root.train.epochs    200 != 100\n",
      "root.train.early_stop_baseline 0.0 != -inf\n",
      "root.train.batch_size 32 != 16\n",
      "root.train.early_stop_max_value 1.0 != inf\n",
      "root.misc.seed       42 != 141\n",
      "root.misc.num_workers 4 != 64\n",
      "}\n",
      "Saved config to /datasets/uly/multiannotator_benchmarks/74424aa5/.trial_0/config.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----\n",
      "Running cross-validation for model: resnet18\n",
      "----\n",
      "Running Cross-Validation on Split: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/resnet18-5c106cde.pth\" to /home/uly/.cache/torch/hub/checkpoints/resnet18-5c106cde.pth\n",
      "Model resnet18 created, param count:                                         11181642\n",
      "AMP not enabled. Training in float32.\n",
      "Disable EMA as it is not supported for now.\n",
      "Start training from [Epoch 0]\n",
      "[Epoch 0] training: accuracy=0.104167\n",
      "[Epoch 0] speed: 25 samples/sec\ttime cost: 3.108615\n",
      "[Epoch 0] validation: top1=0.083333 top5=0.500000\n",
      "[Epoch 0] Current best top-1: 0.083333 vs previous -inf, saved to /datasets/uly/multiannotator_benchmarks/74424aa5/.trial_0/best_checkpoint.pkl\n",
      "[Epoch 1] training: accuracy=0.072917\n",
      "[Epoch 1] speed: 452 samples/sec\ttime cost: 0.177527\n",
      "[Epoch 1] validation: top1=0.083333 top5=0.416667\n",
      "[Epoch 2] training: accuracy=0.239583\n",
      "[Epoch 2] speed: 480 samples/sec\ttime cost: 0.167176\n",
      "[Epoch 2] validation: top1=0.083333 top5=0.500000\n",
      "[Epoch 3] training: accuracy=0.333333\n",
      "[Epoch 3] speed: 497 samples/sec\ttime cost: 0.161430\n",
      "[Epoch 3] validation: top1=0.166667 top5=0.583333\n",
      "[Epoch 3] Current best top-1: 0.166667 vs previous 0.083333, saved to /datasets/uly/multiannotator_benchmarks/74424aa5/.trial_0/best_checkpoint.pkl\n",
      "[Epoch 4] training: accuracy=0.354167\n",
      "[Epoch 4] speed: 497 samples/sec\ttime cost: 0.161474\n",
      "[Epoch 4] validation: top1=0.166667 top5=0.833333\n",
      "[Epoch 5] training: accuracy=0.447917\n",
      "[Epoch 5] speed: 501 samples/sec\ttime cost: 0.160310\n",
      "[Epoch 5] validation: top1=0.333333 top5=0.500000\n",
      "[Epoch 5] Current best top-1: 0.333333 vs previous 0.166667, saved to /datasets/uly/multiannotator_benchmarks/74424aa5/.trial_0/best_checkpoint.pkl\n",
      "[Epoch 6] training: accuracy=0.437500\n",
      "[Epoch 6] speed: 498 samples/sec\ttime cost: 0.161233\n",
      "[Epoch 6] validation: top1=0.166667 top5=0.583333\n",
      "[Epoch 7] training: accuracy=0.489583\n",
      "[Epoch 7] speed: 502 samples/sec\ttime cost: 0.159921\n",
      "[Epoch 7] validation: top1=0.250000 top5=0.750000\n",
      "[Epoch 8] training: accuracy=0.427083\n",
      "[Epoch 8] speed: 486 samples/sec\ttime cost: 0.165002\n",
      "[Epoch 8] validation: top1=0.166667 top5=0.750000\n",
      "[Epoch 9] training: accuracy=0.500000\n",
      "[Epoch 9] speed: 495 samples/sec\ttime cost: 0.162138\n",
      "[Epoch 9] validation: top1=0.250000 top5=0.583333\n",
      "[Epoch 10] training: accuracy=0.500000\n",
      "[Epoch 10] speed: 496 samples/sec\ttime cost: 0.161626\n",
      "[Epoch 10] validation: top1=0.166667 top5=0.750000\n",
      "[Epoch 11] training: accuracy=0.458333\n",
      "[Epoch 11] speed: 495 samples/sec\ttime cost: 0.162042\n",
      "[Epoch 11] validation: top1=0.000000 top5=0.500000\n",
      "[Epoch 12] training: accuracy=0.437500\n",
      "[Epoch 12] speed: 490 samples/sec\ttime cost: 0.163909\n",
      "[Epoch 12] validation: top1=0.166667 top5=0.666667\n",
      "[Epoch 13] training: accuracy=0.447917\n",
      "[Epoch 13] speed: 498 samples/sec\ttime cost: 0.161087\n",
      "[Epoch 13] validation: top1=0.083333 top5=0.583333\n",
      "[Epoch 14] training: accuracy=0.437500\n",
      "[Epoch 14] speed: 494 samples/sec\ttime cost: 0.162871\n",
      "[Epoch 14] validation: top1=0.333333 top5=0.750000\n",
      "[Epoch 15] training: accuracy=0.614583\n",
      "[Epoch 15] speed: 497 samples/sec\ttime cost: 0.161709\n",
      "[Epoch 15] validation: top1=0.250000 top5=0.750000\n",
      "[Epoch 16] EarlyStop after 10 epochs: no better than 0.3333333432674408\n",
      "Applying the state from the best checkpoint...\n",
      "modified configs(<old> != <new>): {\n",
      "root.img_cls.model   resnet101 != resnet18\n",
      "root.train.early_stop_patience -1 != 10\n",
      "root.train.epochs    200 != 100\n",
      "root.train.early_stop_baseline 0.0 != -inf\n",
      "root.train.batch_size 32 != 16\n",
      "root.train.early_stop_max_value 1.0 != inf\n",
      "root.misc.seed       42 != 662\n",
      "root.misc.num_workers 4 != 64\n",
      "}\n",
      "Saved config to /datasets/uly/multiannotator_benchmarks/f4127b7b/.trial_0/config.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving ./data/cifar10_consensus_worst_25_coin20_resnet18/split_0/_test_pred_probs_split_0\n",
      "Saving ./data/cifar10_consensus_worst_25_coin20_resnet18/split_0/_test_pred_features_split_0\n",
      "Saving ./data/cifar10_consensus_worst_25_coin20_resnet18/split_0/_test_labels_split_0\n",
      "Saving ./data/cifar10_consensus_worst_25_coin20_resnet18/split_0/_test_image_files_split_0\n",
      "Saving ./data/cifar10_consensus_worst_25_coin20_resnet18/split_0/_test_indices_split_0\n",
      "----\n",
      "Running Cross-Validation on Split: 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model resnet18 created, param count:                                         11181642\n",
      "AMP not enabled. Training in float32.\n",
      "Disable EMA as it is not supported for now.\n",
      "Start training from [Epoch 0]\n",
      "[Epoch 0] training: accuracy=0.093750\n",
      "[Epoch 0] speed: 31 samples/sec\ttime cost: 2.520188\n",
      "[Epoch 0] validation: top1=0.083333 top5=0.666667\n",
      "[Epoch 0] Current best top-1: 0.083333 vs previous -inf, saved to /datasets/uly/multiannotator_benchmarks/f4127b7b/.trial_0/best_checkpoint.pkl\n",
      "[Epoch 1] training: accuracy=0.135417\n",
      "[Epoch 1] speed: 439 samples/sec\ttime cost: 0.183105\n",
      "[Epoch 1] validation: top1=0.250000 top5=0.750000\n",
      "[Epoch 1] Current best top-1: 0.250000 vs previous 0.083333, saved to /datasets/uly/multiannotator_benchmarks/f4127b7b/.trial_0/best_checkpoint.pkl\n",
      "[Epoch 2] training: accuracy=0.166667\n",
      "[Epoch 2] speed: 494 samples/sec\ttime cost: 0.162673\n",
      "[Epoch 2] validation: top1=0.333333 top5=0.583333\n",
      "[Epoch 2] Current best top-1: 0.333333 vs previous 0.250000, saved to /datasets/uly/multiannotator_benchmarks/f4127b7b/.trial_0/best_checkpoint.pkl\n",
      "[Epoch 3] training: accuracy=0.354167\n",
      "[Epoch 3] speed: 492 samples/sec\ttime cost: 0.163362\n",
      "[Epoch 3] validation: top1=0.333333 top5=0.750000\n",
      "[Epoch 4] training: accuracy=0.416667\n",
      "[Epoch 4] speed: 491 samples/sec\ttime cost: 0.163802\n",
      "[Epoch 4] validation: top1=0.166667 top5=0.916667\n",
      "[Epoch 5] training: accuracy=0.364583\n",
      "[Epoch 5] speed: 494 samples/sec\ttime cost: 0.162792\n",
      "[Epoch 5] validation: top1=0.166667 top5=0.750000\n",
      "[Epoch 6] training: accuracy=0.406250\n",
      "[Epoch 6] speed: 499 samples/sec\ttime cost: 0.161210\n",
      "[Epoch 6] validation: top1=0.083333 top5=0.666667\n",
      "[Epoch 7] training: accuracy=0.427083\n",
      "[Epoch 7] speed: 497 samples/sec\ttime cost: 0.161878\n",
      "[Epoch 7] validation: top1=0.166667 top5=0.750000\n",
      "[Epoch 8] training: accuracy=0.510417\n",
      "[Epoch 8] speed: 501 samples/sec\ttime cost: 0.160716\n",
      "[Epoch 8] validation: top1=0.166667 top5=0.916667\n",
      "[Epoch 9] training: accuracy=0.437500\n",
      "[Epoch 9] speed: 504 samples/sec\ttime cost: 0.159387\n",
      "[Epoch 9] validation: top1=0.250000 top5=0.750000\n",
      "[Epoch 10] training: accuracy=0.520833\n",
      "[Epoch 10] speed: 502 samples/sec\ttime cost: 0.160050\n",
      "[Epoch 10] validation: top1=0.333333 top5=0.750000\n",
      "[Epoch 11] training: accuracy=0.468750\n",
      "[Epoch 11] speed: 497 samples/sec\ttime cost: 0.161579\n",
      "[Epoch 11] validation: top1=0.333333 top5=0.666667\n",
      "[Epoch 12] training: accuracy=0.385417\n",
      "[Epoch 12] speed: 492 samples/sec\ttime cost: 0.163337\n",
      "[Epoch 12] validation: top1=0.500000 top5=0.916667\n",
      "[Epoch 12] Current best top-1: 0.500000 vs previous 0.333333, saved to /datasets/uly/multiannotator_benchmarks/f4127b7b/.trial_0/best_checkpoint.pkl\n",
      "[Epoch 13] training: accuracy=0.510417\n",
      "[Epoch 13] speed: 493 samples/sec\ttime cost: 0.163016\n",
      "[Epoch 13] validation: top1=0.583333 top5=0.833333\n",
      "[Epoch 13] Current best top-1: 0.583333 vs previous 0.500000, saved to /datasets/uly/multiannotator_benchmarks/f4127b7b/.trial_0/best_checkpoint.pkl\n",
      "[Epoch 14] training: accuracy=0.625000\n",
      "[Epoch 14] speed: 482 samples/sec\ttime cost: 0.166786\n",
      "[Epoch 14] validation: top1=0.416667 top5=0.666667\n",
      "[Epoch 15] training: accuracy=0.604167\n",
      "[Epoch 15] speed: 504 samples/sec\ttime cost: 0.159501\n",
      "[Epoch 15] validation: top1=0.333333 top5=0.750000\n",
      "[Epoch 16] training: accuracy=0.625000\n",
      "[Epoch 16] speed: 489 samples/sec\ttime cost: 0.164305\n",
      "[Epoch 16] validation: top1=0.416667 top5=0.833333\n",
      "[Epoch 17] training: accuracy=0.531250\n",
      "[Epoch 17] speed: 496 samples/sec\ttime cost: 0.162074\n",
      "[Epoch 17] validation: top1=0.583333 top5=0.833333\n",
      "[Epoch 18] training: accuracy=0.666667\n",
      "[Epoch 18] speed: 500 samples/sec\ttime cost: 0.160681\n",
      "[Epoch 18] validation: top1=0.583333 top5=0.833333\n",
      "[Epoch 19] training: accuracy=0.666667\n",
      "[Epoch 19] speed: 493 samples/sec\ttime cost: 0.162902\n",
      "[Epoch 19] validation: top1=0.333333 top5=0.833333\n",
      "[Epoch 20] training: accuracy=0.583333\n",
      "[Epoch 20] speed: 495 samples/sec\ttime cost: 0.162221\n",
      "[Epoch 20] validation: top1=0.416667 top5=0.916667\n",
      "[Epoch 21] training: accuracy=0.677083\n",
      "[Epoch 21] speed: 488 samples/sec\ttime cost: 0.164585\n",
      "[Epoch 21] validation: top1=0.583333 top5=0.916667\n",
      "[Epoch 22] training: accuracy=0.729167\n",
      "[Epoch 22] speed: 489 samples/sec\ttime cost: 0.164450\n",
      "[Epoch 22] validation: top1=0.333333 top5=0.833333\n",
      "[Epoch 23] training: accuracy=0.697917\n",
      "[Epoch 23] speed: 488 samples/sec\ttime cost: 0.164830\n",
      "[Epoch 23] validation: top1=0.333333 top5=0.833333\n",
      "[Epoch 24] EarlyStop after 10 epochs: no better than 0.5833333730697632\n",
      "Applying the state from the best checkpoint...\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# run cross-validation for each model\n",
    "for model in models:\n",
    "    \n",
    "    print(\"----\")\n",
    "    print(f\"Running cross-validation for model: {model}\")\n",
    "\n",
    "    MODEL_PARAMS = {\n",
    "        \"model\": model,\n",
    "        \"epochs\": epochs,\n",
    "        \"holdout_frac\": holdout_frac,\n",
    "    }\n",
    "\n",
    "    # results of cross-validation will be saved to pickle files for each model/fold\n",
    "    _ = \\\n",
    "        cross_val_predict_autogluon_image_dataset(\n",
    "            dataset=mini_df,\n",
    "            out_folder=f\"{model_folder}_{model}/\", # save results of cross-validation in pickle files for each fold\n",
    "            n_splits=num_cv_folds,\n",
    "            model_params=MODEL_PARAMS,\n",
    "            time_limit=time_limit,\n",
    "            random_state=random_state,\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc171378",
   "metadata": {},
   "source": [
    "## Read per-fold pickle files from xvalidation and save data as numpy arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23da39c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load pickle file util\n",
    "def load_pickle(pickle_file_name, verbose=1):\n",
    "    \"\"\"Load pickle file\"\"\"\n",
    "\n",
    "    if verbose:\n",
    "        print(f\"Loading {pickle_file_name}\")\n",
    "\n",
    "    with open(pickle_file_name, 'rb') as handle:\n",
    "        out = pickle.load(handle)\n",
    "        \n",
    "    return out\n",
    "\n",
    "# get the original label from file path (aka \"true labels\" y)\n",
    "get_orig_label_idx_from_file_path = np.vectorize(lambda f: label_name_to_idx_map[Path(f).parts[-2]])\n",
    "\n",
    "# get original label name to idx mapping\n",
    "label_name_to_idx_map = {'airplane': 0,\n",
    "                         'automobile': 1,\n",
    "                         'bird': 2,\n",
    "                         'cat': 3,\n",
    "                         'deer': 4,\n",
    "                         'dog': 5,\n",
    "                         'frog': 6,\n",
    "                         'horse': 7,\n",
    "                         'ship': 8,\n",
    "                         'truck': 9}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba7e7837",
   "metadata": {},
   "source": [
    "#### Save pickle files per fold as single files per model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6102283",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_list = []\n",
    "\n",
    "for model in models:\n",
    "\n",
    "    pred_probs = []\n",
    "    labels = []\n",
    "    images = []\n",
    "    \n",
    "    for split_num in range(num_cv_folds):\n",
    "\n",
    "        out_subfolder = f\"{model_folder}_{model}/split_{split_num}/\"\n",
    "        \n",
    "        # pickle file name to read\n",
    "        get_pickle_file_name = (\n",
    "            lambda object_name: f\"{out_subfolder}_{object_name}_split_{split_num}\"\n",
    "        )\n",
    "\n",
    "        # NOTE: the \"test_\" prefix in the pickle name correspond to the \"test\" split during cross-validation.\n",
    "        pred_probs_split = load_pickle(get_pickle_file_name(\"test_pred_probs\"), verbose=verbose)\n",
    "        labels_split = load_pickle(get_pickle_file_name(\"test_labels\"), verbose=verbose)\n",
    "        images_split = load_pickle(get_pickle_file_name(\"test_image_files\"), verbose=verbose)\n",
    "        indices_split = load_pickle(get_pickle_file_name(\"test_indices\"), verbose=verbose)\n",
    "\n",
    "        # append to list so we can combine data from all the splits\n",
    "        pred_probs.append(pred_probs_split)\n",
    "        labels.append(labels_split)\n",
    "        images.append(images_split)    \n",
    "\n",
    "    # convert list to array\n",
    "    pred_probs = np.vstack(pred_probs)\n",
    "    labels = np.hstack(labels) # remember that this is the noisy labels (s)\n",
    "    images = np.hstack(images)\n",
    "    \n",
    "    # get the true labels (y) from the original file path\n",
    "    true_labels = get_orig_label_idx_from_file_path(images)\n",
    "    \n",
    "    # save to Numpy files\n",
    "    numpy_out_folder = f\"{model_folder}_{model}/\"\n",
    "    \n",
    "    print(f\"Saving to numpy files in this folder: {numpy_out_folder}\")\n",
    "    \n",
    "    np.save(numpy_out_folder + \"pred_probs\", pred_probs)\n",
    "    np.save(numpy_out_folder + \"labels\", labels)\n",
    "    np.save(numpy_out_folder + \"images\", images)\n",
    "    np.save(numpy_out_folder + \"true_labels\", true_labels)\n",
    "\n",
    "    # check the accuracy\n",
    "    acc_labels = (pred_probs.argmax(axis=1) == labels).mean() # noisy labels (s)\n",
    "    acc_true_labels = (pred_probs.argmax(axis=1) == true_labels).mean() # true labels (y)    \n",
    "    acc_noisy_vs_true_labels = (labels == true_labels).mean()\n",
    "    \n",
    "    print(f\"Model: {model}\")\n",
    "    print(f\"  Accuracy (argmax pred vs labels): {acc_labels}\")\n",
    "    print(f\"  Accuracy (argmax pred vs true labels) : {acc_true_labels}\")\n",
    "    print(f\"  Accuracy (labels vs true labels)       : {acc_noisy_vs_true_labels}\")\n",
    "    \n",
    "    results = {\n",
    "        \"model\": model,\n",
    "        \"Accuracy (argmax pred vs noisy labels)\": acc_labels,\n",
    "        \"Accuracy (argmax pred vs true labels)\": acc_true_labels,\n",
    "        \"Accuracy (noisy vs true labels)\": acc_noisy_vs_true_labels\n",
    "    }\n",
    "    \n",
    "    results_list.append(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c8d3b40",
   "metadata": {},
   "outputs": [],
   "source": [
    "for model in models:\n",
    "    numpy_out_folder = f\"{model_folder}_{model}/\"\n",
    "\n",
    "    pred_probs = np.load(numpy_out_folder + 'pred_probs.npy')\n",
    "    labels = np.load(numpy_out_folder + 'labels.npy')\n",
    "    true_labels =  np.load(numpy_out_folder + 'true_labels.npy')\n",
    "    print(f'{model}\\n pred_probs[{pred_probs.shape}],labels[{labels.shape}], true_labels[{true_labels.shape}]\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.6 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}