models_training_best_UNet_complete.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QoT5Per3WBaG",
        "outputId": "16ffbd98-8279-45d6-b1fb-c48c70ccaf20"
      },
      "outputs": [],
      "source": [
        "# === colab configuration ===\n",
        "# p.s. when training on colab, weights are saved on Drive (directory DSCAS/weights).\n",
        "# p.p.s. skip this cell if running demo file locally!\n",
        "\n",
        "! pip install torch-summary\n",
        "! pip install ray\n",
        "! pip install tensorboardX==2.5.1\n",
        "! pip install tensorboard==2.11.2\n",
        "from google.colab import drive, files\n",
        "import sys\n",
        "\n",
        "# setting paths\n",
        "repository_path = '/content/deep-seasonal-color-analysis-system/'\n",
        "dataset_path = repository_path + 'headsegmentation_dataset_ccncsa/'\n",
        "dataset_path_drive = '/content/drive/MyDrive/DSCAS/headsegmentation_dataset_ccncsa/'\n",
        "weights_path = repository_path + 'models/weights/'\n",
        "weights_path_drive = '/content/drive/MyDrive/DSCAS/weights/'\n",
        "checkpoint_path = repository_path + 'models/training_best/'\n",
        "checkpoint_archive = '/content/' + checkpoint_path.split('/')[-2] + '.zip'\n",
        "file_path = '/content/deep-seasonal-color-analysis-system/models_training_or_hpo.py'\n",
        "\n",
        "sys.path.insert(0, repository_path)\n",
        "\n",
        "# cloning project repository and downloading dataset\n",
        "drive.mount('/content/drive')\n",
        "! test ! -d $repository_path && git clone https://github.com/mrcmich/deep-seasonal-color-analysis-system.git\n",
        "! test ! -d $dataset_path && cp -R $dataset_path_drive $dataset_path\n",
        "%cd $repository_path\n",
        "\n",
        "# setting branch and pulling updates\n",
        "branch = 'main'\n",
        "! git checkout $branch\n",
        "! git pull origin $branch\n",
        "\n",
        "executing_on_colab = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "aE0AIafCWBaQ",
        "outputId": "88259487-9e3d-4e5d-e0d0-ae32ca02c5b3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "==========================================================================================\n",
            "Layer (type:depth-idx)                   Output Shape              Param #\n",
            "==========================================================================================\n",
            "├─Sequential: 1-1                        [16, 32, 256, 256]        --\n",
            "|    └─Conv2d: 2-1                       [16, 32, 256, 256]        864\n",
            "|    └─BatchNorm2d: 2-2                  [16, 32, 256, 256]        64\n",
            "|    └─ReLU: 2-3                         [16, 32, 256, 256]        --\n",
            "|    └─Conv2d: 2-4                       [16, 32, 256, 256]        9,216\n",
            "|    └─BatchNorm2d: 2-5                  [16, 32, 256, 256]        64\n",
            "|    └─ReLU: 2-6                         [16, 32, 256, 256]        --\n",
            "├─MaxPool2d: 1-2                         [16, 32, 128, 128]        --\n",
            "├─Sequential: 1-3                        [16, 64, 128, 128]        --\n",
            "|    └─Conv2d: 2-7                       [16, 64, 128, 128]        18,432\n",
            "|    └─BatchNorm2d: 2-8                  [16, 64, 128, 128]        128\n",
            "|    └─ReLU: 2-9                         [16, 64, 128, 128]        --\n",
            "|    └─Conv2d: 2-10                      [16, 64, 128, 128]        36,864\n",
            "|    └─BatchNorm2d: 2-11                 [16, 64, 128, 128]        128\n",
            "|    └─ReLU: 2-12                        [16, 64, 128, 128]        --\n",
            "├─MaxPool2d: 1-4                         [16, 64, 64, 64]          --\n",
            "├─Sequential: 1-5                        [16, 128, 64, 64]         --\n",
            "|    └─Conv2d: 2-13                      [16, 128, 64, 64]         73,728\n",
            "|    └─BatchNorm2d: 2-14                 [16, 128, 64, 64]         256\n",
            "|    └─ReLU: 2-15                        [16, 128, 64, 64]         --\n",
            "|    └─Conv2d: 2-16                      [16, 128, 64, 64]         147,456\n",
            "|    └─BatchNorm2d: 2-17                 [16, 128, 64, 64]         256\n",
            "|    └─ReLU: 2-18                        [16, 128, 64, 64]         --\n",
            "├─MaxPool2d: 1-6                         [16, 128, 32, 32]         --\n",
            "├─Sequential: 1-7                        [16, 256, 32, 32]         --\n",
            "|    └─Conv2d: 2-19                      [16, 256, 32, 32]         294,912\n",
            "|    └─BatchNorm2d: 2-20                 [16, 256, 32, 32]         512\n",
            "|    └─ReLU: 2-21                        [16, 256, 32, 32]         --\n",
            "|    └─Conv2d: 2-22                      [16, 256, 32, 32]         589,824\n",
            "|    └─BatchNorm2d: 2-23                 [16, 256, 32, 32]         512\n",
            "|    └─ReLU: 2-24                        [16, 256, 32, 32]         --\n",
            "├─MaxPool2d: 1-8                         [16, 256, 16, 16]         --\n",
            "├─Sequential: 1-9                        [16, 512, 16, 16]         --\n",
            "|    └─Conv2d: 2-25                      [16, 512, 16, 16]         1,179,648\n",
            "|    └─BatchNorm2d: 2-26                 [16, 512, 16, 16]         1,024\n",
            "|    └─ReLU: 2-27                        [16, 512, 16, 16]         --\n",
            "|    └─Conv2d: 2-28                      [16, 512, 16, 16]         2,359,296\n",
            "|    └─BatchNorm2d: 2-29                 [16, 512, 16, 16]         1,024\n",
            "|    └─ReLU: 2-30                        [16, 512, 16, 16]         --\n",
            "├─ConvTranspose2d: 1-10                  [16, 256, 32, 32]         524,544\n",
            "├─Sequential: 1-11                       [16, 256, 32, 32]         --\n",
            "|    └─Conv2d: 2-31                      [16, 256, 32, 32]         1,179,648\n",
            "|    └─BatchNorm2d: 2-32                 [16, 256, 32, 32]         512\n",
            "|    └─ReLU: 2-33                        [16, 256, 32, 32]         --\n",
            "|    └─Conv2d: 2-34                      [16, 256, 32, 32]         589,824\n",
            "|    └─BatchNorm2d: 2-35                 [16, 256, 32, 32]         512\n",
            "|    └─ReLU: 2-36                        [16, 256, 32, 32]         --\n",
            "├─ConvTranspose2d: 1-12                  [16, 128, 64, 64]         131,200\n",
            "├─Sequential: 1-13                       [16, 128, 64, 64]         --\n",
            "|    └─Conv2d: 2-37                      [16, 128, 64, 64]         294,912\n",
            "|    └─BatchNorm2d: 2-38                 [16, 128, 64, 64]         256\n",
            "|    └─ReLU: 2-39                        [16, 128, 64, 64]         --\n",
            "|    └─Conv2d: 2-40                      [16, 128, 64, 64]         147,456\n",
            "|    └─BatchNorm2d: 2-41                 [16, 128, 64, 64]         256\n",
            "|    └─ReLU: 2-42                        [16, 128, 64, 64]         --\n",
            "├─ConvTranspose2d: 1-14                  [16, 64, 128, 128]        32,832\n",
            "├─Sequential: 1-15                       [16, 64, 128, 128]        --\n",
            "|    └─Conv2d: 2-43                      [16, 64, 128, 128]        73,728\n",
            "|    └─BatchNorm2d: 2-44                 [16, 64, 128, 128]        128\n",
            "|    └─ReLU: 2-45                        [16, 64, 128, 128]        --\n",
            "|    └─Conv2d: 2-46                      [16, 64, 128, 128]        36,864\n",
            "|    └─BatchNorm2d: 2-47                 [16, 64, 128, 128]        128\n",
            "|    └─ReLU: 2-48                        [16, 64, 128, 128]        --\n",
            "├─ConvTranspose2d: 1-16                  [16, 32, 256, 256]        8,224\n",
            "├─Sequential: 1-17                       [16, 32, 256, 256]        --\n",
            "|    └─Conv2d: 2-49                      [16, 32, 256, 256]        18,432\n",
            "|    └─BatchNorm2d: 2-50                 [16, 32, 256, 256]        64\n",
            "|    └─ReLU: 2-51                        [16, 32, 256, 256]        --\n",
            "|    └─Conv2d: 2-52                      [16, 32, 256, 256]        9,216\n",
            "|    └─BatchNorm2d: 2-53                 [16, 32, 256, 256]        64\n",
            "|    └─ReLU: 2-54                        [16, 32, 256, 256]        --\n",
            "├─Conv2d: 1-18                           [16, 11, 256, 256]        363\n",
            "==========================================================================================\n",
            "Total params: 7,763,371\n",
            "Trainable params: 7,763,371\n",
            "Non-trainable params: 0\n",
            "Total mult-adds (G): 13.71\n",
            "==========================================================================================\n",
            "Input size (MB): 12.00\n",
            "Forward/backward pass size (MB): 4472.00\n",
            "Params size (MB): 29.61\n",
            "Estimated Total Size (MB): 4513.61\n",
            "==========================================================================================\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2023-02-03 20:49:17,151\tWARNING worker.py:1851 -- Warning: The actor ImplicitFunc is very large (32 MiB). Check that its definition is not implicitly capturing a large array or other object in scope. Tip: use ray.put() to put large objects in the Ray object store.\n",
            "2023-02-03 20:49:17,291\tWARNING util.py:244 -- The `start_trial` operation took 0.924 s, which may be a performance bottleneck.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "== Status ==\n",
            "Current time: 2023-02-03 20:49:17 (running for 00:00:01.40)\n",
            "Memory usage on this node: 5.0/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+\n",
            "| Trial name              | status   | loc                |\n",
            "|-------------------------+----------+--------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |\n",
            "+-------------------------+----------+--------------------+\n",
            "\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div class=\"trialProgress\">\n",
              "  <h3>Trial Progress</h3>\n",
              "  <table>\n",
              "<thead>\n",
              "<tr><th>Trial name             </th><th>date               </th><th>done  </th><th>episodes_total  </th><th>experiment_id                   </th><th style=\"text-align: right;\">  experiment_tag</th><th>hostname    </th><th style=\"text-align: right;\">  iterations_since_restore</th><th>node_ip    </th><th style=\"text-align: right;\">   pid</th><th style=\"text-align: right;\">  time_since_restore</th><th style=\"text-align: right;\">  time_this_iter_s</th><th style=\"text-align: right;\">  time_total_s</th><th style=\"text-align: right;\">  timestamp</th><th style=\"text-align: right;\">  timesteps_since_restore</th><th>timesteps_total  </th><th style=\"text-align: right;\">  train_loss</th><th style=\"text-align: right;\">  train_score</th><th style=\"text-align: right;\">  training_iteration</th><th>trial_id   </th><th style=\"text-align: right;\">  warmup_time</th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td>train_model_38d98_00000</td><td>2023-02-03_23-12-47</td><td>True  </td><td>                </td><td>9afa47d003cc477eb88a67be546bf89a</td><td style=\"text-align: right;\">               0</td><td>81b7dd2326f6</td><td style=\"text-align: right;\">                        20</td><td>172.28.0.12</td><td style=\"text-align: right;\">103716</td><td style=\"text-align: right;\">             8603.41</td><td style=\"text-align: right;\">           433.336</td><td style=\"text-align: right;\">       8603.41</td><td style=\"text-align: right;\"> 1675465967</td><td style=\"text-align: right;\">                        0</td><td>                 </td><td style=\"text-align: right;\">   0.0350107</td><td style=\"text-align: right;\">     0.748056</td><td style=\"text-align: right;\">                  20</td><td>38d98_00000</td><td style=\"text-align: right;\">    0.0036993</td></tr>\n",
              "</tbody>\n",
              "</table>\n",
              "</div>\n",
              "<style>\n",
              ".trialProgress {\n",
              "  display: flex;\n",
              "  flex-direction: column;\n",
              "  color: var(--jp-ui-font-color1);\n",
              "}\n",
              ".trialProgress h3 {\n",
              "  font-weight: bold;\n",
              "}\n",
              ".trialProgress td {\n",
              "  white-space: nowrap;\n",
              "}\n",
              "</style>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "== Status ==\n",
            "Current time: 2023-02-03 20:59:19 (running for 00:10:03.23)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |     0.731562 |      0.286399 |                    1 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 21:09:22 (running for 00:20:06.69)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |     0.342586 |      0.371383 |                    2 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 21:19:24 (running for 00:30:08.63)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |     0.160029 |      0.555751 |                    4 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 21:29:26 (running for 00:40:10.92)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |     0.127633 |      0.611699 |                    5 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 21:39:28 (running for 00:50:12.80)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |    0.0931739 |      0.656297 |                    7 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 21:49:32 (running for 01:00:16.64)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |     0.082338 |      0.670691 |                    8 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 21:59:36 (running for 01:10:20.85)\n",
            "Memory usage on this node: 7.9/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |    0.0749698 |      0.683169 |                    9 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 22:09:41 (running for 01:20:25.53)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |    0.0635546 |      0.700686 |                   11 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 22:19:44 (running for 01:30:29.02)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |     0.059825 |      0.705316 |                   12 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 22:29:48 (running for 01:40:32.24)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |    0.0511521 |      0.719284 |                   14 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 22:39:49 (running for 01:50:33.67)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |    0.0477474 |      0.725029 |                   15 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 22:49:51 (running for 02:00:36.09)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |    0.0453529 |      0.729973 |                   16 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 22:59:52 (running for 02:10:36.79)\n",
            "Memory usage on this node: 7.9/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |     0.039689 |      0.739084 |                   18 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Current time: 2023-02-03 23:09:54 (running for 02:20:38.83)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 1.0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 RUNNING)\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status   | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+----------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | RUNNING  | 172.28.0.12:103716 |    0.0379577 |       0.74245 |                   19 |\n",
            "+-------------------------+----------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2023-02-03 23:12:47,749\tINFO tune.py:762 -- Total run time: 8612.58 seconds (8611.73 seconds for the tuning loop).\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "== Status ==\n",
            "Current time: 2023-02-03 23:12:47 (running for 02:23:31.74)\n",
            "Memory usage on this node: 7.8/12.7 GiB \n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.41 GiB heap, 0.0/3.7 GiB objects\n",
            "Result logdir: /content/deep-seasonal-color-analysis-system/models/training_best/UNet/validationUNet/validationUNet/complete/train_model_2023-02-03_20-49-15\n",
            "Number of trials: 1/1 (1 TERMINATED)\n",
            "+-------------------------+------------+--------------------+--------------+---------------+----------------------+\n",
            "| Trial name              | status     | loc                |   train_loss |   train_score |   training_iteration |\n",
            "|-------------------------+------------+--------------------+--------------+---------------+----------------------|\n",
            "| train_model_38d98_00000 | TERMINATED | 172.28.0.12:103716 |    0.0350107 |      0.748056 |                   20 |\n",
            "+-------------------------+------------+--------------------+--------------+---------------+----------------------+\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "%run $file_path --config=best --model_name=unet --evaluate=False --n_epochs=20"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 376
        },
        "id": "dToeE-J_fwP6",
        "outputId": "bf7768f6-2b61-4439-e1b5-5cf2297449b9"
      },
      "outputs": [],
      "source": [
        "# download training results\n",
        "! zip -r $checkpoint_archive $checkpoint_path\n",
        "files.download(checkpoint_archive)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "cv",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.6"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "bff6c79cdac020ac98a972002b5dbe7766c5a0fccfcd9b44550d7582f6eef17b"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}