3dunet-minimal.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "\n",
    "from datasets.hdf5 import get_train_loaders\n",
    "from unet3d.config import load_config\n",
    "from unet3d.losses import get_loss_criterion\n",
    "from unet3d.metrics import get_evaluation_metric\n",
    "from unet3d.model import get_model\n",
    "from unet3d.trainer import UNet3DTrainer\n",
    "from unet3d.utils import get_logger\n",
    "from unet3d.utils import get_number_of_learnable_parameters\n",
    "\n",
    "import argparse\n",
    "\n",
    "import os\n",
    "import yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-07-31 15:36:09,135 [MainThread] INFO UNet3DTrainer - {'manual_seed': 0, 'model': {'name': 'UNet3D', 'in_channels': 1, 'out_channels': 2, 'layer_order': 'crg', 'f_maps': 32, 'num_groups': 8, 'final_sigmoid': False}, 'trainer': {'checkpoint_dir': '3dunet', 'resume': None, 'validate_after_iters': 20, 'log_after_iters': 20, 'epochs': 600, 'iters': 100000, 'eval_score_higher_is_better': True}, 'optimizer': {'learning_rate': 0.0002, 'weight_decay': 0.0001}, 'loss': {'name': 'CrossEntropyLoss', 'loss_weight': None, 'ignore_index': None}, 'eval_metric': {'name': 'MeanIoU', 'ignore_index': None}, 'lr_scheduler': {'name': 'MultiStepLR', 'milestones': [10, 30, 60], 'gamma': 0.2}, 'loaders': {'train_patch': [128, 128, 128], 'train_stride': [64, 64, 64], 'val_patch': [128, 128, 128], 'val_stride': [128, 128, 128], 'raw_internal_path': 'raw', 'label_internal_path': 'label', 'train_path': ['../suki_fractals/test.h5'], 'val_path': ['../suki_fractals/test.h5'], 'num_workers': 12, 'transformer': {'train': {'raw': [{'name': 'Normalize'}, {'name': 'ToTensor', 'expand_dims': True}], 'label': [{'name': 'ToTensor', 'expand_dims': False, 'dtype': 'long'}]}, 'test': {'raw': [{'name': 'Normalize'}, {'name': 'ToTensor', 'expand_dims': True}], 'label': [{'name': 'ToTensor', 'expand_dims': False, 'dtype': 'long'}]}}}, 'device': device(type='cuda', index=0)}\n",
      "2019-07-31 15:36:09,135 [MainThread] INFO UNet3DTrainer - {'manual_seed': 0, 'model': {'name': 'UNet3D', 'in_channels': 1, 'out_channels': 2, 'layer_order': 'crg', 'f_maps': 32, 'num_groups': 8, 'final_sigmoid': False}, 'trainer': {'checkpoint_dir': '3dunet', 'resume': None, 'validate_after_iters': 20, 'log_after_iters': 20, 'epochs': 600, 'iters': 100000, 'eval_score_higher_is_better': True}, 'optimizer': {'learning_rate': 0.0002, 'weight_decay': 0.0001}, 'loss': {'name': 'CrossEntropyLoss', 'loss_weight': None, 'ignore_index': None}, 'eval_metric': {'name': 'MeanIoU', 'ignore_index': None}, 'lr_scheduler': {'name': 'MultiStepLR', 'milestones': [10, 30, 60], 'gamma': 0.2}, 'loaders': {'train_patch': [128, 128, 128], 'train_stride': [64, 64, 64], 'val_patch': [128, 128, 128], 'val_stride': [128, 128, 128], 'raw_internal_path': 'raw', 'label_internal_path': 'label', 'train_path': ['../suki_fractals/test.h5'], 'val_path': ['../suki_fractals/test.h5'], 'num_workers': 12, 'transformer': {'train': {'raw': [{'name': 'Normalize'}, {'name': 'ToTensor', 'expand_dims': True}], 'label': [{'name': 'ToTensor', 'expand_dims': False, 'dtype': 'long'}]}, 'test': {'raw': [{'name': 'Normalize'}, {'name': 'ToTensor', 'expand_dims': True}], 'label': [{'name': 'ToTensor', 'expand_dims': False, 'dtype': 'long'}]}}}, 'device': device(type='cuda', index=0)}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gheimber/miniconda3/envs/dev2/lib/python3.7/site-packages/ipykernel_launcher.py:15: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "# set config and device parameters\n",
    "\n",
    "logger = get_logger('UNet3DTrainer')\n",
    "\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    DEFAULT_DEVICE = 'cuda:0'\n",
    "else:\n",
    "    DEFAULT_DEVICE = 'cpu'\n",
    "\n",
    "# config file\n",
    "config_file = 'resources/train_config_ce.yaml'\n",
    "#config_file = 'resources/train_config_dice.yaml'\n",
    "\n",
    "config = yaml.load(open(config_file, 'r'))\n",
    "# Get a device to train on\n",
    "device = config.get('device', DEFAULT_DEVICE)\n",
    "config['device'] = torch.device(device)\n",
    "\n",
    "logger.info(config)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _create_trainer(config, model, optimizer, lr_scheduler, loss_criterion, eval_criterion, loaders, logger):\n",
    "    assert 'trainer' in config, 'Could not find trainer configuration'\n",
    "    trainer_config = config['trainer']\n",
    "\n",
    "    resume = trainer_config.get('resume', None)\n",
    "    pre_trained = trainer_config.get('pre_trained', None)\n",
    "\n",
    "    return UNet3DTrainer(model, optimizer, lr_scheduler, loss_criterion, eval_criterion,\n",
    "                         config['device'], loaders, trainer_config['checkpoint_dir'],\n",
    "                         max_num_epochs=trainer_config['epochs'],\n",
    "                         max_num_iterations=trainer_config['iters'],\n",
    "                         validate_after_iters=trainer_config['validate_after_iters'],\n",
    "                         log_after_iters=trainer_config['log_after_iters'],\n",
    "                         eval_score_higher_is_better=trainer_config['eval_score_higher_is_better'],\n",
    "                         logger=logger)\n",
    "\n",
    "\n",
    "def _create_optimizer(config, model):\n",
    "    assert 'optimizer' in config, 'Cannot find optimizer configuration'\n",
    "    optimizer_config = config['optimizer']\n",
    "    learning_rate = optimizer_config['learning_rate']\n",
    "    weight_decay = optimizer_config['weight_decay']\n",
    "    optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n",
    "    return optimizer\n",
    "\n",
    "\n",
    "def _create_lr_scheduler(config, optimizer):\n",
    "    lr_config = config.get('lr_scheduler', None)\n",
    "    if lr_config is None:\n",
    "        # use ReduceLROnPlateau as a default scheduler\n",
    "        return ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=20, verbose=True)\n",
    "    else:\n",
    "        class_name = lr_config.pop('name')\n",
    "        m = importlib.import_module('torch.optim.lr_scheduler')\n",
    "        clazz = getattr(m, class_name)\n",
    "        # add optimizer to the config\n",
    "        lr_config['optimizer'] = optimizer\n",
    "        return clazz(**lr_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-07-31 15:36:09,419 [MainThread] INFO UNet3DTrainer - Seed the RNG for all devices with 0\n",
      "2019-07-31 15:36:09,419 [MainThread] INFO UNet3DTrainer - Seed the RNG for all devices with 0\n",
      "2019-07-31 15:36:09,460 [MainThread] INFO UNet3DTrainer - Sending the model to 'cuda:0'\n",
      "2019-07-31 15:36:09,460 [MainThread] INFO UNet3DTrainer - Sending the model to 'cuda:0'\n",
      "2019-07-31 15:36:09,469 [MainThread] INFO UNet3DTrainer - Number of learnable params 4080914\n",
      "2019-07-31 15:36:09,469 [MainThread] INFO UNet3DTrainer - Number of learnable params 4080914\n",
      "2019-07-31 15:36:09,470 [MainThread] INFO HDF5Dataset - Creating training and validation set loaders...\n",
      "2019-07-31 15:36:09,470 [MainThread] INFO HDF5Dataset - Creating training and validation set loaders...\n",
      "2019-07-31 15:36:09,471 [MainThread] INFO HDF5Dataset - Slice builder class: SliceBuilder\n",
      "2019-07-31 15:36:09,471 [MainThread] INFO HDF5Dataset - Slice builder class: SliceBuilder\n",
      "2019-07-31 15:36:09,472 [MainThread] INFO HDF5Dataset - Loading training set from: ../suki_fractals/test.h5...\n",
      "2019-07-31 15:36:09,472 [MainThread] INFO HDF5Dataset - Loading training set from: ../suki_fractals/test.h5...\n",
      "2019-07-31 15:36:09,582 [MainThread] INFO HDF5Dataset - Loading validation set from: ../suki_fractals/test.h5...\n",
      "2019-07-31 15:36:09,582 [MainThread] INFO HDF5Dataset - Loading validation set from: ../suki_fractals/test.h5...\n",
      "2019-07-31 15:36:09,698 [MainThread] INFO HDF5Dataset - Number of workers for train/val datasets: 12\n",
      "2019-07-31 15:36:09,698 [MainThread] INFO HDF5Dataset - Number of workers for train/val datasets: 12\n",
      "2019-07-31 15:36:09,701 [MainThread] INFO UNet3DTrainer - UNet3D(\n",
      "  (encoders): ModuleList(\n",
      "    (0): Encoder(\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(1, 16, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 16, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(16, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 32, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (1): Encoder(\n",
      "      (pooling): MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=0, dilation=1, ceil_mode=False)\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 32, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(32, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 64, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (2): Encoder(\n",
      "      (pooling): MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=0, dilation=1, ceil_mode=False)\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 64, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(64, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 128, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (3): Encoder(\n",
      "      (pooling): MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=0, dilation=1, ceil_mode=False)\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 128, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(128, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 256, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (decoders): ModuleList(\n",
      "    (0): Decoder(\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(384, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 128, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 128, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (1): Decoder(\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(192, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 64, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 64, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (2): Decoder(\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(96, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 32, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 32, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (final_conv): Conv3d(32, 2, kernel_size=(1, 1, 1), stride=(1, 1, 1))\n",
      "  (final_activation): Softmax()\n",
      ")\n",
      "2019-07-31 15:36:09,701 [MainThread] INFO UNet3DTrainer - UNet3D(\n",
      "  (encoders): ModuleList(\n",
      "    (0): Encoder(\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(1, 16, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 16, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(16, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 32, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (1): Encoder(\n",
      "      (pooling): MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=0, dilation=1, ceil_mode=False)\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 32, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(32, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 64, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (2): Encoder(\n",
      "      (pooling): MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=0, dilation=1, ceil_mode=False)\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 64, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(64, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 128, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (3): Encoder(\n",
      "      (pooling): MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=0, dilation=1, ceil_mode=False)\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 128, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(128, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 256, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (decoders): ModuleList(\n",
      "    (0): Decoder(\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(384, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 128, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(128, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 128, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (1): Decoder(\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(192, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 64, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(64, 64, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 64, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (2): Decoder(\n",
      "      (basic_module): DoubleConv(\n",
      "        (SingleConv1): SingleConv(\n",
      "          (conv): Conv3d(96, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 32, eps=1e-05, affine=True)\n",
      "        )\n",
      "        (SingleConv2): SingleConv(\n",
      "          (conv): Conv3d(32, 32, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "          (ReLU): ReLU(inplace)\n",
      "          (groupnorm): GroupNorm(8, 32, eps=1e-05, affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (final_conv): Conv3d(32, 2, kernel_size=(1, 1, 1), stride=(1, 1, 1))\n",
      "  (final_activation): Softmax()\n",
      ")\n",
      "2019-07-31 15:36:09,703 [MainThread] INFO UNet3DTrainer - eval_score_higher_is_better: True\n",
      "2019-07-31 15:36:09,703 [MainThread] INFO UNet3DTrainer - eval_score_higher_is_better: True\n",
      "2019-07-31 15:36:10,101 [MainThread] INFO UNet3DTrainer - Training iteration 1. Batch 0. Epoch [0/599]\n",
      "2019-07-31 15:36:10,101 [MainThread] INFO UNet3DTrainer - Training iteration 1. Batch 0. Epoch [0/599]\n",
      "2019-07-31 15:36:10,829 [MainThread] INFO UNet3DTrainer - Training iteration 2. Batch 0. Epoch [1/599]\n",
      "2019-07-31 15:36:10,829 [MainThread] INFO UNet3DTrainer - Training iteration 2. Batch 0. Epoch [1/599]\n",
      "2019-07-31 15:36:12,072 [MainThread] INFO UNet3DTrainer - Training iteration 3. Batch 0. Epoch [2/599]\n",
      "2019-07-31 15:36:12,072 [MainThread] INFO UNet3DTrainer - Training iteration 3. Batch 0. Epoch [2/599]\n",
      "2019-07-31 15:36:13,329 [MainThread] INFO UNet3DTrainer - Training iteration 4. Batch 0. Epoch [3/599]\n",
      "2019-07-31 15:36:13,329 [MainThread] INFO UNet3DTrainer - Training iteration 4. Batch 0. Epoch [3/599]\n",
      "2019-07-31 15:36:14,586 [MainThread] INFO UNet3DTrainer - Training iteration 5. Batch 0. Epoch [4/599]\n",
      "2019-07-31 15:36:14,586 [MainThread] INFO UNet3DTrainer - Training iteration 5. Batch 0. Epoch [4/599]\n",
      "2019-07-31 15:36:15,842 [MainThread] INFO UNet3DTrainer - Training iteration 6. Batch 0. Epoch [5/599]\n",
      "2019-07-31 15:36:15,842 [MainThread] INFO UNet3DTrainer - Training iteration 6. Batch 0. Epoch [5/599]\n",
      "2019-07-31 15:36:17,096 [MainThread] INFO UNet3DTrainer - Training iteration 7. Batch 0. Epoch [6/599]\n",
      "2019-07-31 15:36:17,096 [MainThread] INFO UNet3DTrainer - Training iteration 7. Batch 0. Epoch [6/599]\n",
      "2019-07-31 15:36:18,331 [MainThread] INFO UNet3DTrainer - Training iteration 8. Batch 0. Epoch [7/599]\n",
      "2019-07-31 15:36:18,331 [MainThread] INFO UNet3DTrainer - Training iteration 8. Batch 0. Epoch [7/599]\n",
      "2019-07-31 15:36:19,638 [MainThread] INFO UNet3DTrainer - Training iteration 9. Batch 0. Epoch [8/599]\n",
      "2019-07-31 15:36:19,638 [MainThread] INFO UNet3DTrainer - Training iteration 9. Batch 0. Epoch [8/599]\n",
      "2019-07-31 15:36:20,922 [MainThread] INFO UNet3DTrainer - Training iteration 10. Batch 0. Epoch [9/599]\n",
      "2019-07-31 15:36:20,922 [MainThread] INFO UNet3DTrainer - Training iteration 10. Batch 0. Epoch [9/599]\n",
      "2019-07-31 15:36:22,090 [MainThread] INFO UNet3DTrainer - Training iteration 11. Batch 0. Epoch [10/599]\n",
      "2019-07-31 15:36:22,090 [MainThread] INFO UNet3DTrainer - Training iteration 11. Batch 0. Epoch [10/599]\n",
      "2019-07-31 15:36:23,329 [MainThread] INFO UNet3DTrainer - Training iteration 12. Batch 0. Epoch [11/599]\n",
      "2019-07-31 15:36:23,329 [MainThread] INFO UNet3DTrainer - Training iteration 12. Batch 0. Epoch [11/599]\n",
      "2019-07-31 15:36:24,582 [MainThread] INFO UNet3DTrainer - Training iteration 13. Batch 0. Epoch [12/599]\n",
      "2019-07-31 15:36:24,582 [MainThread] INFO UNet3DTrainer - Training iteration 13. Batch 0. Epoch [12/599]\n",
      "2019-07-31 15:36:25,821 [MainThread] INFO UNet3DTrainer - Training iteration 14. Batch 0. Epoch [13/599]\n",
      "2019-07-31 15:36:25,821 [MainThread] INFO UNet3DTrainer - Training iteration 14. Batch 0. Epoch [13/599]\n",
      "2019-07-31 15:36:27,054 [MainThread] INFO UNet3DTrainer - Training iteration 15. Batch 0. Epoch [14/599]\n",
      "2019-07-31 15:36:27,054 [MainThread] INFO UNet3DTrainer - Training iteration 15. Batch 0. Epoch [14/599]\n",
      "2019-07-31 15:36:28,305 [MainThread] INFO UNet3DTrainer - Training iteration 16. Batch 0. Epoch [15/599]\n",
      "2019-07-31 15:36:28,305 [MainThread] INFO UNet3DTrainer - Training iteration 16. Batch 0. Epoch [15/599]\n",
      "2019-07-31 15:36:29,594 [MainThread] INFO UNet3DTrainer - Training iteration 17. Batch 0. Epoch [16/599]\n",
      "2019-07-31 15:36:29,594 [MainThread] INFO UNet3DTrainer - Training iteration 17. Batch 0. Epoch [16/599]\n",
      "2019-07-31 15:36:30,824 [MainThread] INFO UNet3DTrainer - Training iteration 18. Batch 0. Epoch [17/599]\n",
      "2019-07-31 15:36:30,824 [MainThread] INFO UNet3DTrainer - Training iteration 18. Batch 0. Epoch [17/599]\n",
      "2019-07-31 15:36:32,086 [MainThread] INFO UNet3DTrainer - Training iteration 19. Batch 0. Epoch [18/599]\n",
      "2019-07-31 15:36:32,086 [MainThread] INFO UNet3DTrainer - Training iteration 19. Batch 0. Epoch [18/599]\n",
      "2019-07-31 15:36:33,295 [MainThread] INFO UNet3DTrainer - Training iteration 20. Batch 0. Epoch [19/599]\n",
      "2019-07-31 15:36:33,295 [MainThread] INFO UNet3DTrainer - Training iteration 20. Batch 0. Epoch [19/599]\n",
      "2019-07-31 15:36:34,129 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:36:34,129 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:36:35,453 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.37070760130882263. Evaluation score: 0.4996824264526367\n",
      "2019-07-31 15:36:35,453 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.37070760130882263. Evaluation score: 0.4996824264526367\n",
      "2019-07-31 15:36:35,459 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.4996824264526367\n",
      "2019-07-31 15:36:35,459 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.4996824264526367\n",
      "2019-07-31 15:36:35,461 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:36:35,461 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:36:35,522 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:36:35,522 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:36:35,575 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.10625074803829193. Evaluation score: 0.499680757522583\n",
      "2019-07-31 15:36:35,575 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.10625074803829193. Evaluation score: 0.499680757522583\n",
      "2019-07-31 15:36:35,576 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:36:35,576 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:36:37,371 [MainThread] INFO UNet3DTrainer - Training iteration 21. Batch 0. Epoch [20/599]\n",
      "2019-07-31 15:36:37,371 [MainThread] INFO UNet3DTrainer - Training iteration 21. Batch 0. Epoch [20/599]\n",
      "2019-07-31 15:36:38,082 [MainThread] INFO UNet3DTrainer - Training iteration 22. Batch 0. Epoch [21/599]\n",
      "2019-07-31 15:36:38,082 [MainThread] INFO UNet3DTrainer - Training iteration 22. Batch 0. Epoch [21/599]\n",
      "2019-07-31 15:36:39,292 [MainThread] INFO UNet3DTrainer - Training iteration 23. Batch 0. Epoch [22/599]\n",
      "2019-07-31 15:36:39,292 [MainThread] INFO UNet3DTrainer - Training iteration 23. Batch 0. Epoch [22/599]\n",
      "2019-07-31 15:36:40,559 [MainThread] INFO UNet3DTrainer - Training iteration 24. Batch 0. Epoch [23/599]\n",
      "2019-07-31 15:36:40,559 [MainThread] INFO UNet3DTrainer - Training iteration 24. Batch 0. Epoch [23/599]\n",
      "2019-07-31 15:36:41,808 [MainThread] INFO UNet3DTrainer - Training iteration 25. Batch 0. Epoch [24/599]\n",
      "2019-07-31 15:36:41,808 [MainThread] INFO UNet3DTrainer - Training iteration 25. Batch 0. Epoch [24/599]\n",
      "2019-07-31 15:36:43,050 [MainThread] INFO UNet3DTrainer - Training iteration 26. Batch 0. Epoch [25/599]\n",
      "2019-07-31 15:36:43,050 [MainThread] INFO UNet3DTrainer - Training iteration 26. Batch 0. Epoch [25/599]\n",
      "2019-07-31 15:36:44,319 [MainThread] INFO UNet3DTrainer - Training iteration 27. Batch 0. Epoch [26/599]\n",
      "2019-07-31 15:36:44,319 [MainThread] INFO UNet3DTrainer - Training iteration 27. Batch 0. Epoch [26/599]\n",
      "2019-07-31 15:36:45,589 [MainThread] INFO UNet3DTrainer - Training iteration 28. Batch 0. Epoch [27/599]\n",
      "2019-07-31 15:36:45,589 [MainThread] INFO UNet3DTrainer - Training iteration 28. Batch 0. Epoch [27/599]\n",
      "2019-07-31 15:36:46,809 [MainThread] INFO UNet3DTrainer - Training iteration 29. Batch 0. Epoch [28/599]\n",
      "2019-07-31 15:36:46,809 [MainThread] INFO UNet3DTrainer - Training iteration 29. Batch 0. Epoch [28/599]\n",
      "2019-07-31 15:36:48,096 [MainThread] INFO UNet3DTrainer - Training iteration 30. Batch 0. Epoch [29/599]\n",
      "2019-07-31 15:36:48,096 [MainThread] INFO UNet3DTrainer - Training iteration 30. Batch 0. Epoch [29/599]\n",
      "2019-07-31 15:36:49,326 [MainThread] INFO UNet3DTrainer - Training iteration 31. Batch 0. Epoch [30/599]\n",
      "2019-07-31 15:36:49,326 [MainThread] INFO UNet3DTrainer - Training iteration 31. Batch 0. Epoch [30/599]\n",
      "2019-07-31 15:36:50,568 [MainThread] INFO UNet3DTrainer - Training iteration 32. Batch 0. Epoch [31/599]\n",
      "2019-07-31 15:36:50,568 [MainThread] INFO UNet3DTrainer - Training iteration 32. Batch 0. Epoch [31/599]\n",
      "2019-07-31 15:36:51,818 [MainThread] INFO UNet3DTrainer - Training iteration 33. Batch 0. Epoch [32/599]\n",
      "2019-07-31 15:36:51,818 [MainThread] INFO UNet3DTrainer - Training iteration 33. Batch 0. Epoch [32/599]\n",
      "2019-07-31 15:36:53,077 [MainThread] INFO UNet3DTrainer - Training iteration 34. Batch 0. Epoch [33/599]\n",
      "2019-07-31 15:36:53,077 [MainThread] INFO UNet3DTrainer - Training iteration 34. Batch 0. Epoch [33/599]\n",
      "2019-07-31 15:36:54,344 [MainThread] INFO UNet3DTrainer - Training iteration 35. Batch 0. Epoch [34/599]\n",
      "2019-07-31 15:36:54,344 [MainThread] INFO UNet3DTrainer - Training iteration 35. Batch 0. Epoch [34/599]\n",
      "2019-07-31 15:36:55,590 [MainThread] INFO UNet3DTrainer - Training iteration 36. Batch 0. Epoch [35/599]\n",
      "2019-07-31 15:36:55,590 [MainThread] INFO UNet3DTrainer - Training iteration 36. Batch 0. Epoch [35/599]\n",
      "2019-07-31 15:36:56,833 [MainThread] INFO UNet3DTrainer - Training iteration 37. Batch 0. Epoch [36/599]\n",
      "2019-07-31 15:36:56,833 [MainThread] INFO UNet3DTrainer - Training iteration 37. Batch 0. Epoch [36/599]\n",
      "2019-07-31 15:36:58,095 [MainThread] INFO UNet3DTrainer - Training iteration 38. Batch 0. Epoch [37/599]\n",
      "2019-07-31 15:36:58,095 [MainThread] INFO UNet3DTrainer - Training iteration 38. Batch 0. Epoch [37/599]\n",
      "2019-07-31 15:36:59,347 [MainThread] INFO UNet3DTrainer - Training iteration 39. Batch 0. Epoch [38/599]\n",
      "2019-07-31 15:36:59,347 [MainThread] INFO UNet3DTrainer - Training iteration 39. Batch 0. Epoch [38/599]\n",
      "2019-07-31 15:37:00,609 [MainThread] INFO UNet3DTrainer - Training iteration 40. Batch 0. Epoch [39/599]\n",
      "2019-07-31 15:37:00,609 [MainThread] INFO UNet3DTrainer - Training iteration 40. Batch 0. Epoch [39/599]\n",
      "2019-07-31 15:37:01,407 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:37:01,407 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:37:02,723 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3572764992713928. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:02,723 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3572764992713928. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:02,728 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.49968528747558594\n",
      "2019-07-31 15:37:02,728 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.49968528747558594\n",
      "2019-07-31 15:37:02,730 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:37:02,730 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:37:03,006 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:37:03,006 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:37:03,222 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.08204380422830582. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:03,222 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.08204380422830582. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:03,224 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:37:03,224 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:37:04,993 [MainThread] INFO UNet3DTrainer - Training iteration 41. Batch 0. Epoch [40/599]\n",
      "2019-07-31 15:37:04,993 [MainThread] INFO UNet3DTrainer - Training iteration 41. Batch 0. Epoch [40/599]\n",
      "2019-07-31 15:37:05,693 [MainThread] INFO UNet3DTrainer - Training iteration 42. Batch 0. Epoch [41/599]\n",
      "2019-07-31 15:37:05,693 [MainThread] INFO UNet3DTrainer - Training iteration 42. Batch 0. Epoch [41/599]\n",
      "2019-07-31 15:37:06,923 [MainThread] INFO UNet3DTrainer - Training iteration 43. Batch 0. Epoch [42/599]\n",
      "2019-07-31 15:37:06,923 [MainThread] INFO UNet3DTrainer - Training iteration 43. Batch 0. Epoch [42/599]\n",
      "2019-07-31 15:37:08,174 [MainThread] INFO UNet3DTrainer - Training iteration 44. Batch 0. Epoch [43/599]\n",
      "2019-07-31 15:37:08,174 [MainThread] INFO UNet3DTrainer - Training iteration 44. Batch 0. Epoch [43/599]\n",
      "2019-07-31 15:37:09,421 [MainThread] INFO UNet3DTrainer - Training iteration 45. Batch 0. Epoch [44/599]\n",
      "2019-07-31 15:37:09,421 [MainThread] INFO UNet3DTrainer - Training iteration 45. Batch 0. Epoch [44/599]\n",
      "2019-07-31 15:37:10,667 [MainThread] INFO UNet3DTrainer - Training iteration 46. Batch 0. Epoch [45/599]\n",
      "2019-07-31 15:37:10,667 [MainThread] INFO UNet3DTrainer - Training iteration 46. Batch 0. Epoch [45/599]\n",
      "2019-07-31 15:37:11,915 [MainThread] INFO UNet3DTrainer - Training iteration 47. Batch 0. Epoch [46/599]\n",
      "2019-07-31 15:37:11,915 [MainThread] INFO UNet3DTrainer - Training iteration 47. Batch 0. Epoch [46/599]\n",
      "2019-07-31 15:37:13,180 [MainThread] INFO UNet3DTrainer - Training iteration 48. Batch 0. Epoch [47/599]\n",
      "2019-07-31 15:37:13,180 [MainThread] INFO UNet3DTrainer - Training iteration 48. Batch 0. Epoch [47/599]\n",
      "2019-07-31 15:37:14,422 [MainThread] INFO UNet3DTrainer - Training iteration 49. Batch 0. Epoch [48/599]\n",
      "2019-07-31 15:37:14,422 [MainThread] INFO UNet3DTrainer - Training iteration 49. Batch 0. Epoch [48/599]\n",
      "2019-07-31 15:37:15,680 [MainThread] INFO UNet3DTrainer - Training iteration 50. Batch 0. Epoch [49/599]\n",
      "2019-07-31 15:37:15,680 [MainThread] INFO UNet3DTrainer - Training iteration 50. Batch 0. Epoch [49/599]\n",
      "2019-07-31 15:37:16,915 [MainThread] INFO UNet3DTrainer - Training iteration 51. Batch 0. Epoch [50/599]\n",
      "2019-07-31 15:37:16,915 [MainThread] INFO UNet3DTrainer - Training iteration 51. Batch 0. Epoch [50/599]\n",
      "2019-07-31 15:37:18,184 [MainThread] INFO UNet3DTrainer - Training iteration 52. Batch 0. Epoch [51/599]\n",
      "2019-07-31 15:37:18,184 [MainThread] INFO UNet3DTrainer - Training iteration 52. Batch 0. Epoch [51/599]\n",
      "2019-07-31 15:37:19,451 [MainThread] INFO UNet3DTrainer - Training iteration 53. Batch 0. Epoch [52/599]\n",
      "2019-07-31 15:37:19,451 [MainThread] INFO UNet3DTrainer - Training iteration 53. Batch 0. Epoch [52/599]\n",
      "2019-07-31 15:37:20,713 [MainThread] INFO UNet3DTrainer - Training iteration 54. Batch 0. Epoch [53/599]\n",
      "2019-07-31 15:37:20,713 [MainThread] INFO UNet3DTrainer - Training iteration 54. Batch 0. Epoch [53/599]\n",
      "2019-07-31 15:37:21,959 [MainThread] INFO UNet3DTrainer - Training iteration 55. Batch 0. Epoch [54/599]\n",
      "2019-07-31 15:37:21,959 [MainThread] INFO UNet3DTrainer - Training iteration 55. Batch 0. Epoch [54/599]\n",
      "2019-07-31 15:37:23,211 [MainThread] INFO UNet3DTrainer - Training iteration 56. Batch 0. Epoch [55/599]\n",
      "2019-07-31 15:37:23,211 [MainThread] INFO UNet3DTrainer - Training iteration 56. Batch 0. Epoch [55/599]\n",
      "2019-07-31 15:37:24,457 [MainThread] INFO UNet3DTrainer - Training iteration 57. Batch 0. Epoch [56/599]\n",
      "2019-07-31 15:37:24,457 [MainThread] INFO UNet3DTrainer - Training iteration 57. Batch 0. Epoch [56/599]\n",
      "2019-07-31 15:37:25,728 [MainThread] INFO UNet3DTrainer - Training iteration 58. Batch 0. Epoch [57/599]\n",
      "2019-07-31 15:37:25,728 [MainThread] INFO UNet3DTrainer - Training iteration 58. Batch 0. Epoch [57/599]\n",
      "2019-07-31 15:37:26,922 [MainThread] INFO UNet3DTrainer - Training iteration 59. Batch 0. Epoch [58/599]\n",
      "2019-07-31 15:37:26,922 [MainThread] INFO UNet3DTrainer - Training iteration 59. Batch 0. Epoch [58/599]\n",
      "2019-07-31 15:37:28,205 [MainThread] INFO UNet3DTrainer - Training iteration 60. Batch 0. Epoch [59/599]\n",
      "2019-07-31 15:37:28,205 [MainThread] INFO UNet3DTrainer - Training iteration 60. Batch 0. Epoch [59/599]\n",
      "2019-07-31 15:37:29,013 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:37:29,013 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:37:30,340 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3393518924713135. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:30,340 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3393518924713135. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:30,346 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:37:30,346 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:37:30,612 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.04907771199941635. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:30,612 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.04907771199941635. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:30,614 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:37:30,614 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:37:32,066 [MainThread] INFO UNet3DTrainer - Training iteration 61. Batch 0. Epoch [60/599]\n",
      "2019-07-31 15:37:32,066 [MainThread] INFO UNet3DTrainer - Training iteration 61. Batch 0. Epoch [60/599]\n",
      "2019-07-31 15:37:32,813 [MainThread] INFO UNet3DTrainer - Training iteration 62. Batch 0. Epoch [61/599]\n",
      "2019-07-31 15:37:32,813 [MainThread] INFO UNet3DTrainer - Training iteration 62. Batch 0. Epoch [61/599]\n",
      "2019-07-31 15:37:34,006 [MainThread] INFO UNet3DTrainer - Training iteration 63. Batch 0. Epoch [62/599]\n",
      "2019-07-31 15:37:34,006 [MainThread] INFO UNet3DTrainer - Training iteration 63. Batch 0. Epoch [62/599]\n",
      "2019-07-31 15:37:35,246 [MainThread] INFO UNet3DTrainer - Training iteration 64. Batch 0. Epoch [63/599]\n",
      "2019-07-31 15:37:35,246 [MainThread] INFO UNet3DTrainer - Training iteration 64. Batch 0. Epoch [63/599]\n",
      "2019-07-31 15:37:36,507 [MainThread] INFO UNet3DTrainer - Training iteration 65. Batch 0. Epoch [64/599]\n",
      "2019-07-31 15:37:36,507 [MainThread] INFO UNet3DTrainer - Training iteration 65. Batch 0. Epoch [64/599]\n",
      "2019-07-31 15:37:37,744 [MainThread] INFO UNet3DTrainer - Training iteration 66. Batch 0. Epoch [65/599]\n",
      "2019-07-31 15:37:37,744 [MainThread] INFO UNet3DTrainer - Training iteration 66. Batch 0. Epoch [65/599]\n",
      "2019-07-31 15:37:38,990 [MainThread] INFO UNet3DTrainer - Training iteration 67. Batch 0. Epoch [66/599]\n",
      "2019-07-31 15:37:38,990 [MainThread] INFO UNet3DTrainer - Training iteration 67. Batch 0. Epoch [66/599]\n",
      "2019-07-31 15:37:40,243 [MainThread] INFO UNet3DTrainer - Training iteration 68. Batch 0. Epoch [67/599]\n",
      "2019-07-31 15:37:40,243 [MainThread] INFO UNet3DTrainer - Training iteration 68. Batch 0. Epoch [67/599]\n",
      "2019-07-31 15:37:41,499 [MainThread] INFO UNet3DTrainer - Training iteration 69. Batch 0. Epoch [68/599]\n",
      "2019-07-31 15:37:41,499 [MainThread] INFO UNet3DTrainer - Training iteration 69. Batch 0. Epoch [68/599]\n",
      "2019-07-31 15:37:42,737 [MainThread] INFO UNet3DTrainer - Training iteration 70. Batch 0. Epoch [69/599]\n",
      "2019-07-31 15:37:42,737 [MainThread] INFO UNet3DTrainer - Training iteration 70. Batch 0. Epoch [69/599]\n",
      "2019-07-31 15:37:44,062 [MainThread] INFO UNet3DTrainer - Training iteration 71. Batch 0. Epoch [70/599]\n",
      "2019-07-31 15:37:44,062 [MainThread] INFO UNet3DTrainer - Training iteration 71. Batch 0. Epoch [70/599]\n",
      "2019-07-31 15:37:45,246 [MainThread] INFO UNet3DTrainer - Training iteration 72. Batch 0. Epoch [71/599]\n",
      "2019-07-31 15:37:45,246 [MainThread] INFO UNet3DTrainer - Training iteration 72. Batch 0. Epoch [71/599]\n",
      "2019-07-31 15:37:46,525 [MainThread] INFO UNet3DTrainer - Training iteration 73. Batch 0. Epoch [72/599]\n",
      "2019-07-31 15:37:46,525 [MainThread] INFO UNet3DTrainer - Training iteration 73. Batch 0. Epoch [72/599]\n",
      "2019-07-31 15:37:47,785 [MainThread] INFO UNet3DTrainer - Training iteration 74. Batch 0. Epoch [73/599]\n",
      "2019-07-31 15:37:47,785 [MainThread] INFO UNet3DTrainer - Training iteration 74. Batch 0. Epoch [73/599]\n",
      "2019-07-31 15:37:49,029 [MainThread] INFO UNet3DTrainer - Training iteration 75. Batch 0. Epoch [74/599]\n",
      "2019-07-31 15:37:49,029 [MainThread] INFO UNet3DTrainer - Training iteration 75. Batch 0. Epoch [74/599]\n",
      "2019-07-31 15:37:50,321 [MainThread] INFO UNet3DTrainer - Training iteration 76. Batch 0. Epoch [75/599]\n",
      "2019-07-31 15:37:50,321 [MainThread] INFO UNet3DTrainer - Training iteration 76. Batch 0. Epoch [75/599]\n",
      "2019-07-31 15:37:51,534 [MainThread] INFO UNet3DTrainer - Training iteration 77. Batch 0. Epoch [76/599]\n",
      "2019-07-31 15:37:51,534 [MainThread] INFO UNet3DTrainer - Training iteration 77. Batch 0. Epoch [76/599]\n",
      "2019-07-31 15:37:52,793 [MainThread] INFO UNet3DTrainer - Training iteration 78. Batch 0. Epoch [77/599]\n",
      "2019-07-31 15:37:52,793 [MainThread] INFO UNet3DTrainer - Training iteration 78. Batch 0. Epoch [77/599]\n",
      "2019-07-31 15:37:54,036 [MainThread] INFO UNet3DTrainer - Training iteration 79. Batch 0. Epoch [78/599]\n",
      "2019-07-31 15:37:54,036 [MainThread] INFO UNet3DTrainer - Training iteration 79. Batch 0. Epoch [78/599]\n",
      "2019-07-31 15:37:55,296 [MainThread] INFO UNet3DTrainer - Training iteration 80. Batch 0. Epoch [79/599]\n",
      "2019-07-31 15:37:55,296 [MainThread] INFO UNet3DTrainer - Training iteration 80. Batch 0. Epoch [79/599]\n",
      "2019-07-31 15:37:56,084 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:37:56,084 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:37:57,415 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3322862982749939. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:57,415 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3322862982749939. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:57,421 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:37:57,421 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:37:57,701 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.035957079380750656. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:57,701 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.035957079380750656. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:37:57,702 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:37:57,702 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:37:59,151 [MainThread] INFO UNet3DTrainer - Training iteration 81. Batch 0. Epoch [80/599]\n",
      "2019-07-31 15:37:59,151 [MainThread] INFO UNet3DTrainer - Training iteration 81. Batch 0. Epoch [80/599]\n",
      "2019-07-31 15:37:59,855 [MainThread] INFO UNet3DTrainer - Training iteration 82. Batch 0. Epoch [81/599]\n",
      "2019-07-31 15:37:59,855 [MainThread] INFO UNet3DTrainer - Training iteration 82. Batch 0. Epoch [81/599]\n",
      "2019-07-31 15:38:01,086 [MainThread] INFO UNet3DTrainer - Training iteration 83. Batch 0. Epoch [82/599]\n",
      "2019-07-31 15:38:01,086 [MainThread] INFO UNet3DTrainer - Training iteration 83. Batch 0. Epoch [82/599]\n",
      "2019-07-31 15:38:02,340 [MainThread] INFO UNet3DTrainer - Training iteration 84. Batch 0. Epoch [83/599]\n",
      "2019-07-31 15:38:02,340 [MainThread] INFO UNet3DTrainer - Training iteration 84. Batch 0. Epoch [83/599]\n",
      "2019-07-31 15:38:03,587 [MainThread] INFO UNet3DTrainer - Training iteration 85. Batch 0. Epoch [84/599]\n",
      "2019-07-31 15:38:03,587 [MainThread] INFO UNet3DTrainer - Training iteration 85. Batch 0. Epoch [84/599]\n",
      "2019-07-31 15:38:04,837 [MainThread] INFO UNet3DTrainer - Training iteration 86. Batch 0. Epoch [85/599]\n",
      "2019-07-31 15:38:04,837 [MainThread] INFO UNet3DTrainer - Training iteration 86. Batch 0. Epoch [85/599]\n",
      "2019-07-31 15:38:06,080 [MainThread] INFO UNet3DTrainer - Training iteration 87. Batch 0. Epoch [86/599]\n",
      "2019-07-31 15:38:06,080 [MainThread] INFO UNet3DTrainer - Training iteration 87. Batch 0. Epoch [86/599]\n",
      "2019-07-31 15:38:07,327 [MainThread] INFO UNet3DTrainer - Training iteration 88. Batch 0. Epoch [87/599]\n",
      "2019-07-31 15:38:07,327 [MainThread] INFO UNet3DTrainer - Training iteration 88. Batch 0. Epoch [87/599]\n",
      "2019-07-31 15:38:08,576 [MainThread] INFO UNet3DTrainer - Training iteration 89. Batch 0. Epoch [88/599]\n",
      "2019-07-31 15:38:08,576 [MainThread] INFO UNet3DTrainer - Training iteration 89. Batch 0. Epoch [88/599]\n",
      "2019-07-31 15:38:09,827 [MainThread] INFO UNet3DTrainer - Training iteration 90. Batch 0. Epoch [89/599]\n",
      "2019-07-31 15:38:09,827 [MainThread] INFO UNet3DTrainer - Training iteration 90. Batch 0. Epoch [89/599]\n",
      "2019-07-31 15:38:11,081 [MainThread] INFO UNet3DTrainer - Training iteration 91. Batch 0. Epoch [90/599]\n",
      "2019-07-31 15:38:11,081 [MainThread] INFO UNet3DTrainer - Training iteration 91. Batch 0. Epoch [90/599]\n",
      "2019-07-31 15:38:12,332 [MainThread] INFO UNet3DTrainer - Training iteration 92. Batch 0. Epoch [91/599]\n",
      "2019-07-31 15:38:12,332 [MainThread] INFO UNet3DTrainer - Training iteration 92. Batch 0. Epoch [91/599]\n",
      "2019-07-31 15:38:13,598 [MainThread] INFO UNet3DTrainer - Training iteration 93. Batch 0. Epoch [92/599]\n",
      "2019-07-31 15:38:13,598 [MainThread] INFO UNet3DTrainer - Training iteration 93. Batch 0. Epoch [92/599]\n",
      "2019-07-31 15:38:14,865 [MainThread] INFO UNet3DTrainer - Training iteration 94. Batch 0. Epoch [93/599]\n",
      "2019-07-31 15:38:14,865 [MainThread] INFO UNet3DTrainer - Training iteration 94. Batch 0. Epoch [93/599]\n",
      "2019-07-31 15:38:16,125 [MainThread] INFO UNet3DTrainer - Training iteration 95. Batch 0. Epoch [94/599]\n",
      "2019-07-31 15:38:16,125 [MainThread] INFO UNet3DTrainer - Training iteration 95. Batch 0. Epoch [94/599]\n",
      "2019-07-31 15:38:17,366 [MainThread] INFO UNet3DTrainer - Training iteration 96. Batch 0. Epoch [95/599]\n",
      "2019-07-31 15:38:17,366 [MainThread] INFO UNet3DTrainer - Training iteration 96. Batch 0. Epoch [95/599]\n",
      "2019-07-31 15:38:18,638 [MainThread] INFO UNet3DTrainer - Training iteration 97. Batch 0. Epoch [96/599]\n",
      "2019-07-31 15:38:18,638 [MainThread] INFO UNet3DTrainer - Training iteration 97. Batch 0. Epoch [96/599]\n",
      "2019-07-31 15:38:19,857 [MainThread] INFO UNet3DTrainer - Training iteration 98. Batch 0. Epoch [97/599]\n",
      "2019-07-31 15:38:19,857 [MainThread] INFO UNet3DTrainer - Training iteration 98. Batch 0. Epoch [97/599]\n",
      "2019-07-31 15:38:21,128 [MainThread] INFO UNet3DTrainer - Training iteration 99. Batch 0. Epoch [98/599]\n",
      "2019-07-31 15:38:21,128 [MainThread] INFO UNet3DTrainer - Training iteration 99. Batch 0. Epoch [98/599]\n",
      "2019-07-31 15:38:22,372 [MainThread] INFO UNet3DTrainer - Training iteration 100. Batch 0. Epoch [99/599]\n",
      "2019-07-31 15:38:22,372 [MainThread] INFO UNet3DTrainer - Training iteration 100. Batch 0. Epoch [99/599]\n",
      "2019-07-31 15:38:23,160 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:38:23,160 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:38:24,492 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3293929994106293. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:38:24,492 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3293929994106293. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:38:24,498 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:38:24,498 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:38:24,769 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.030686603859066963. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:38:24,769 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.030686603859066963. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:38:24,770 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:38:24,770 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:38:26,172 [MainThread] INFO UNet3DTrainer - Training iteration 101. Batch 0. Epoch [100/599]\n",
      "2019-07-31 15:38:26,172 [MainThread] INFO UNet3DTrainer - Training iteration 101. Batch 0. Epoch [100/599]\n",
      "2019-07-31 15:38:26,909 [MainThread] INFO UNet3DTrainer - Training iteration 102. Batch 0. Epoch [101/599]\n",
      "2019-07-31 15:38:26,909 [MainThread] INFO UNet3DTrainer - Training iteration 102. Batch 0. Epoch [101/599]\n",
      "2019-07-31 15:38:28,139 [MainThread] INFO UNet3DTrainer - Training iteration 103. Batch 0. Epoch [102/599]\n",
      "2019-07-31 15:38:28,139 [MainThread] INFO UNet3DTrainer - Training iteration 103. Batch 0. Epoch [102/599]\n",
      "2019-07-31 15:38:29,367 [MainThread] INFO UNet3DTrainer - Training iteration 104. Batch 0. Epoch [103/599]\n",
      "2019-07-31 15:38:29,367 [MainThread] INFO UNet3DTrainer - Training iteration 104. Batch 0. Epoch [103/599]\n",
      "2019-07-31 15:38:30,622 [MainThread] INFO UNet3DTrainer - Training iteration 105. Batch 0. Epoch [104/599]\n",
      "2019-07-31 15:38:30,622 [MainThread] INFO UNet3DTrainer - Training iteration 105. Batch 0. Epoch [104/599]\n",
      "2019-07-31 15:38:31,865 [MainThread] INFO UNet3DTrainer - Training iteration 106. Batch 0. Epoch [105/599]\n",
      "2019-07-31 15:38:31,865 [MainThread] INFO UNet3DTrainer - Training iteration 106. Batch 0. Epoch [105/599]\n",
      "2019-07-31 15:38:33,116 [MainThread] INFO UNet3DTrainer - Training iteration 107. Batch 0. Epoch [106/599]\n",
      "2019-07-31 15:38:33,116 [MainThread] INFO UNet3DTrainer - Training iteration 107. Batch 0. Epoch [106/599]\n",
      "2019-07-31 15:38:34,366 [MainThread] INFO UNet3DTrainer - Training iteration 108. Batch 0. Epoch [107/599]\n",
      "2019-07-31 15:38:34,366 [MainThread] INFO UNet3DTrainer - Training iteration 108. Batch 0. Epoch [107/599]\n",
      "2019-07-31 15:38:35,629 [MainThread] INFO UNet3DTrainer - Training iteration 109. Batch 0. Epoch [108/599]\n",
      "2019-07-31 15:38:35,629 [MainThread] INFO UNet3DTrainer - Training iteration 109. Batch 0. Epoch [108/599]\n",
      "2019-07-31 15:38:36,869 [MainThread] INFO UNet3DTrainer - Training iteration 110. Batch 0. Epoch [109/599]\n",
      "2019-07-31 15:38:36,869 [MainThread] INFO UNet3DTrainer - Training iteration 110. Batch 0. Epoch [109/599]\n",
      "2019-07-31 15:38:38,167 [MainThread] INFO UNet3DTrainer - Training iteration 111. Batch 0. Epoch [110/599]\n",
      "2019-07-31 15:38:38,167 [MainThread] INFO UNet3DTrainer - Training iteration 111. Batch 0. Epoch [110/599]\n",
      "2019-07-31 15:38:39,418 [MainThread] INFO UNet3DTrainer - Training iteration 112. Batch 0. Epoch [111/599]\n",
      "2019-07-31 15:38:39,418 [MainThread] INFO UNet3DTrainer - Training iteration 112. Batch 0. Epoch [111/599]\n",
      "2019-07-31 15:38:40,634 [MainThread] INFO UNet3DTrainer - Training iteration 113. Batch 0. Epoch [112/599]\n",
      "2019-07-31 15:38:40,634 [MainThread] INFO UNet3DTrainer - Training iteration 113. Batch 0. Epoch [112/599]\n",
      "2019-07-31 15:38:41,909 [MainThread] INFO UNet3DTrainer - Training iteration 114. Batch 0. Epoch [113/599]\n",
      "2019-07-31 15:38:41,909 [MainThread] INFO UNet3DTrainer - Training iteration 114. Batch 0. Epoch [113/599]\n",
      "2019-07-31 15:38:43,160 [MainThread] INFO UNet3DTrainer - Training iteration 115. Batch 0. Epoch [114/599]\n",
      "2019-07-31 15:38:43,160 [MainThread] INFO UNet3DTrainer - Training iteration 115. Batch 0. Epoch [114/599]\n",
      "2019-07-31 15:38:44,410 [MainThread] INFO UNet3DTrainer - Training iteration 116. Batch 0. Epoch [115/599]\n",
      "2019-07-31 15:38:44,410 [MainThread] INFO UNet3DTrainer - Training iteration 116. Batch 0. Epoch [115/599]\n",
      "2019-07-31 15:38:45,639 [MainThread] INFO UNet3DTrainer - Training iteration 117. Batch 0. Epoch [116/599]\n",
      "2019-07-31 15:38:45,639 [MainThread] INFO UNet3DTrainer - Training iteration 117. Batch 0. Epoch [116/599]\n",
      "2019-07-31 15:38:46,927 [MainThread] INFO UNet3DTrainer - Training iteration 118. Batch 0. Epoch [117/599]\n",
      "2019-07-31 15:38:46,927 [MainThread] INFO UNet3DTrainer - Training iteration 118. Batch 0. Epoch [117/599]\n",
      "2019-07-31 15:38:48,147 [MainThread] INFO UNet3DTrainer - Training iteration 119. Batch 0. Epoch [118/599]\n",
      "2019-07-31 15:38:48,147 [MainThread] INFO UNet3DTrainer - Training iteration 119. Batch 0. Epoch [118/599]\n",
      "2019-07-31 15:38:49,423 [MainThread] INFO UNet3DTrainer - Training iteration 120. Batch 0. Epoch [119/599]\n",
      "2019-07-31 15:38:49,423 [MainThread] INFO UNet3DTrainer - Training iteration 120. Batch 0. Epoch [119/599]\n",
      "2019-07-31 15:38:50,202 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:38:50,202 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:38:51,514 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.32726627588272095. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:38:51,514 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.32726627588272095. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:38:51,520 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:38:51,520 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:38:51,776 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.02677687257528305. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:38:51,776 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.02677687257528305. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:38:51,778 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:38:51,778 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:38:53,242 [MainThread] INFO UNet3DTrainer - Training iteration 121. Batch 0. Epoch [120/599]\n",
      "2019-07-31 15:38:53,242 [MainThread] INFO UNet3DTrainer - Training iteration 121. Batch 0. Epoch [120/599]\n",
      "2019-07-31 15:38:53,991 [MainThread] INFO UNet3DTrainer - Training iteration 122. Batch 0. Epoch [121/599]\n",
      "2019-07-31 15:38:53,991 [MainThread] INFO UNet3DTrainer - Training iteration 122. Batch 0. Epoch [121/599]\n",
      "2019-07-31 15:38:55,192 [MainThread] INFO UNet3DTrainer - Training iteration 123. Batch 0. Epoch [122/599]\n",
      "2019-07-31 15:38:55,192 [MainThread] INFO UNet3DTrainer - Training iteration 123. Batch 0. Epoch [122/599]\n",
      "2019-07-31 15:38:56,446 [MainThread] INFO UNet3DTrainer - Training iteration 124. Batch 0. Epoch [123/599]\n",
      "2019-07-31 15:38:56,446 [MainThread] INFO UNet3DTrainer - Training iteration 124. Batch 0. Epoch [123/599]\n",
      "2019-07-31 15:38:57,681 [MainThread] INFO UNet3DTrainer - Training iteration 125. Batch 0. Epoch [124/599]\n",
      "2019-07-31 15:38:57,681 [MainThread] INFO UNet3DTrainer - Training iteration 125. Batch 0. Epoch [124/599]\n",
      "2019-07-31 15:38:58,941 [MainThread] INFO UNet3DTrainer - Training iteration 126. Batch 0. Epoch [125/599]\n",
      "2019-07-31 15:38:58,941 [MainThread] INFO UNet3DTrainer - Training iteration 126. Batch 0. Epoch [125/599]\n",
      "2019-07-31 15:39:00,173 [MainThread] INFO UNet3DTrainer - Training iteration 127. Batch 0. Epoch [126/599]\n",
      "2019-07-31 15:39:00,173 [MainThread] INFO UNet3DTrainer - Training iteration 127. Batch 0. Epoch [126/599]\n",
      "2019-07-31 15:39:01,440 [MainThread] INFO UNet3DTrainer - Training iteration 128. Batch 0. Epoch [127/599]\n",
      "2019-07-31 15:39:01,440 [MainThread] INFO UNet3DTrainer - Training iteration 128. Batch 0. Epoch [127/599]\n",
      "2019-07-31 15:39:02,686 [MainThread] INFO UNet3DTrainer - Training iteration 129. Batch 0. Epoch [128/599]\n",
      "2019-07-31 15:39:02,686 [MainThread] INFO UNet3DTrainer - Training iteration 129. Batch 0. Epoch [128/599]\n",
      "2019-07-31 15:39:03,938 [MainThread] INFO UNet3DTrainer - Training iteration 130. Batch 0. Epoch [129/599]\n",
      "2019-07-31 15:39:03,938 [MainThread] INFO UNet3DTrainer - Training iteration 130. Batch 0. Epoch [129/599]\n",
      "2019-07-31 15:39:05,182 [MainThread] INFO UNet3DTrainer - Training iteration 131. Batch 0. Epoch [130/599]\n",
      "2019-07-31 15:39:05,182 [MainThread] INFO UNet3DTrainer - Training iteration 131. Batch 0. Epoch [130/599]\n",
      "2019-07-31 15:39:06,444 [MainThread] INFO UNet3DTrainer - Training iteration 132. Batch 0. Epoch [131/599]\n",
      "2019-07-31 15:39:06,444 [MainThread] INFO UNet3DTrainer - Training iteration 132. Batch 0. Epoch [131/599]\n",
      "2019-07-31 15:39:07,693 [MainThread] INFO UNet3DTrainer - Training iteration 133. Batch 0. Epoch [132/599]\n",
      "2019-07-31 15:39:07,693 [MainThread] INFO UNet3DTrainer - Training iteration 133. Batch 0. Epoch [132/599]\n",
      "2019-07-31 15:39:08,983 [MainThread] INFO UNet3DTrainer - Training iteration 134. Batch 0. Epoch [133/599]\n",
      "2019-07-31 15:39:08,983 [MainThread] INFO UNet3DTrainer - Training iteration 134. Batch 0. Epoch [133/599]\n",
      "2019-07-31 15:39:10,205 [MainThread] INFO UNet3DTrainer - Training iteration 135. Batch 0. Epoch [134/599]\n",
      "2019-07-31 15:39:10,205 [MainThread] INFO UNet3DTrainer - Training iteration 135. Batch 0. Epoch [134/599]\n",
      "2019-07-31 15:39:11,484 [MainThread] INFO UNet3DTrainer - Training iteration 136. Batch 0. Epoch [135/599]\n",
      "2019-07-31 15:39:11,484 [MainThread] INFO UNet3DTrainer - Training iteration 136. Batch 0. Epoch [135/599]\n",
      "2019-07-31 15:39:12,716 [MainThread] INFO UNet3DTrainer - Training iteration 137. Batch 0. Epoch [136/599]\n",
      "2019-07-31 15:39:12,716 [MainThread] INFO UNet3DTrainer - Training iteration 137. Batch 0. Epoch [136/599]\n",
      "2019-07-31 15:39:13,956 [MainThread] INFO UNet3DTrainer - Training iteration 138. Batch 0. Epoch [137/599]\n",
      "2019-07-31 15:39:13,956 [MainThread] INFO UNet3DTrainer - Training iteration 138. Batch 0. Epoch [137/599]\n",
      "2019-07-31 15:39:15,243 [MainThread] INFO UNet3DTrainer - Training iteration 139. Batch 0. Epoch [138/599]\n",
      "2019-07-31 15:39:15,243 [MainThread] INFO UNet3DTrainer - Training iteration 139. Batch 0. Epoch [138/599]\n",
      "2019-07-31 15:39:16,468 [MainThread] INFO UNet3DTrainer - Training iteration 140. Batch 0. Epoch [139/599]\n",
      "2019-07-31 15:39:16,468 [MainThread] INFO UNet3DTrainer - Training iteration 140. Batch 0. Epoch [139/599]\n",
      "2019-07-31 15:39:17,279 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:39:17,279 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:39:18,609 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3257049322128296. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:39:18,609 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3257049322128296. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:39:18,615 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:39:18,615 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:39:18,899 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.023782847449183464. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:39:18,899 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.023782847449183464. Evaluation score: 0.49968528747558594\n",
      "2019-07-31 15:39:18,901 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:39:18,901 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:39:20,329 [MainThread] INFO UNet3DTrainer - Training iteration 141. Batch 0. Epoch [140/599]\n",
      "2019-07-31 15:39:20,329 [MainThread] INFO UNet3DTrainer - Training iteration 141. Batch 0. Epoch [140/599]\n",
      "2019-07-31 15:39:21,061 [MainThread] INFO UNet3DTrainer - Training iteration 142. Batch 0. Epoch [141/599]\n",
      "2019-07-31 15:39:21,061 [MainThread] INFO UNet3DTrainer - Training iteration 142. Batch 0. Epoch [141/599]\n",
      "2019-07-31 15:39:22,276 [MainThread] INFO UNet3DTrainer - Training iteration 143. Batch 0. Epoch [142/599]\n",
      "2019-07-31 15:39:22,276 [MainThread] INFO UNet3DTrainer - Training iteration 143. Batch 0. Epoch [142/599]\n",
      "2019-07-31 15:39:23,545 [MainThread] INFO UNet3DTrainer - Training iteration 144. Batch 0. Epoch [143/599]\n",
      "2019-07-31 15:39:23,545 [MainThread] INFO UNet3DTrainer - Training iteration 144. Batch 0. Epoch [143/599]\n",
      "2019-07-31 15:39:24,773 [MainThread] INFO UNet3DTrainer - Training iteration 145. Batch 0. Epoch [144/599]\n",
      "2019-07-31 15:39:24,773 [MainThread] INFO UNet3DTrainer - Training iteration 145. Batch 0. Epoch [144/599]\n",
      "2019-07-31 15:39:26,028 [MainThread] INFO UNet3DTrainer - Training iteration 146. Batch 0. Epoch [145/599]\n",
      "2019-07-31 15:39:26,028 [MainThread] INFO UNet3DTrainer - Training iteration 146. Batch 0. Epoch [145/599]\n",
      "2019-07-31 15:39:27,269 [MainThread] INFO UNet3DTrainer - Training iteration 147. Batch 0. Epoch [146/599]\n",
      "2019-07-31 15:39:27,269 [MainThread] INFO UNet3DTrainer - Training iteration 147. Batch 0. Epoch [146/599]\n",
      "2019-07-31 15:39:28,528 [MainThread] INFO UNet3DTrainer - Training iteration 148. Batch 0. Epoch [147/599]\n",
      "2019-07-31 15:39:28,528 [MainThread] INFO UNet3DTrainer - Training iteration 148. Batch 0. Epoch [147/599]\n",
      "2019-07-31 15:39:29,780 [MainThread] INFO UNet3DTrainer - Training iteration 149. Batch 0. Epoch [148/599]\n",
      "2019-07-31 15:39:29,780 [MainThread] INFO UNet3DTrainer - Training iteration 149. Batch 0. Epoch [148/599]\n",
      "2019-07-31 15:39:31,032 [MainThread] INFO UNet3DTrainer - Training iteration 150. Batch 0. Epoch [149/599]\n",
      "2019-07-31 15:39:31,032 [MainThread] INFO UNet3DTrainer - Training iteration 150. Batch 0. Epoch [149/599]\n",
      "2019-07-31 15:39:32,325 [MainThread] INFO UNet3DTrainer - Training iteration 151. Batch 0. Epoch [150/599]\n",
      "2019-07-31 15:39:32,325 [MainThread] INFO UNet3DTrainer - Training iteration 151. Batch 0. Epoch [150/599]\n",
      "2019-07-31 15:39:33,535 [MainThread] INFO UNet3DTrainer - Training iteration 152. Batch 0. Epoch [151/599]\n",
      "2019-07-31 15:39:33,535 [MainThread] INFO UNet3DTrainer - Training iteration 152. Batch 0. Epoch [151/599]\n",
      "2019-07-31 15:39:34,807 [MainThread] INFO UNet3DTrainer - Training iteration 153. Batch 0. Epoch [152/599]\n",
      "2019-07-31 15:39:34,807 [MainThread] INFO UNet3DTrainer - Training iteration 153. Batch 0. Epoch [152/599]\n",
      "2019-07-31 15:39:36,070 [MainThread] INFO UNet3DTrainer - Training iteration 154. Batch 0. Epoch [153/599]\n",
      "2019-07-31 15:39:36,070 [MainThread] INFO UNet3DTrainer - Training iteration 154. Batch 0. Epoch [153/599]\n",
      "2019-07-31 15:39:37,315 [MainThread] INFO UNet3DTrainer - Training iteration 155. Batch 0. Epoch [154/599]\n",
      "2019-07-31 15:39:37,315 [MainThread] INFO UNet3DTrainer - Training iteration 155. Batch 0. Epoch [154/599]\n",
      "2019-07-31 15:39:38,538 [MainThread] INFO UNet3DTrainer - Training iteration 156. Batch 0. Epoch [155/599]\n",
      "2019-07-31 15:39:38,538 [MainThread] INFO UNet3DTrainer - Training iteration 156. Batch 0. Epoch [155/599]\n",
      "2019-07-31 15:39:39,857 [MainThread] INFO UNet3DTrainer - Training iteration 157. Batch 0. Epoch [156/599]\n",
      "2019-07-31 15:39:39,857 [MainThread] INFO UNet3DTrainer - Training iteration 157. Batch 0. Epoch [156/599]\n",
      "2019-07-31 15:39:41,057 [MainThread] INFO UNet3DTrainer - Training iteration 158. Batch 0. Epoch [157/599]\n",
      "2019-07-31 15:39:41,057 [MainThread] INFO UNet3DTrainer - Training iteration 158. Batch 0. Epoch [157/599]\n",
      "2019-07-31 15:39:42,332 [MainThread] INFO UNet3DTrainer - Training iteration 159. Batch 0. Epoch [158/599]\n",
      "2019-07-31 15:39:42,332 [MainThread] INFO UNet3DTrainer - Training iteration 159. Batch 0. Epoch [158/599]\n",
      "2019-07-31 15:39:43,558 [MainThread] INFO UNet3DTrainer - Training iteration 160. Batch 0. Epoch [159/599]\n",
      "2019-07-31 15:39:43,558 [MainThread] INFO UNet3DTrainer - Training iteration 160. Batch 0. Epoch [159/599]\n",
      "2019-07-31 15:39:44,366 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:39:44,366 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:39:45,683 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3243502080440521. Evaluation score: 0.5000637769699097\n",
      "2019-07-31 15:39:45,683 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3243502080440521. Evaluation score: 0.5000637769699097\n",
      "2019-07-31 15:39:45,689 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5000637769699097\n",
      "2019-07-31 15:39:45,689 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5000637769699097\n",
      "2019-07-31 15:39:45,691 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:39:45,691 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:39:45,966 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:39:45,966 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:39:46,183 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.02123168110847473. Evaluation score: 0.5000637769699097\n",
      "2019-07-31 15:39:46,183 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.02123168110847473. Evaluation score: 0.5000637769699097\n",
      "2019-07-31 15:39:46,185 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:39:46,185 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:39:47,622 [MainThread] INFO UNet3DTrainer - Training iteration 161. Batch 0. Epoch [160/599]\n",
      "2019-07-31 15:39:47,622 [MainThread] INFO UNet3DTrainer - Training iteration 161. Batch 0. Epoch [160/599]\n",
      "2019-07-31 15:39:48,327 [MainThread] INFO UNet3DTrainer - Training iteration 162. Batch 0. Epoch [161/599]\n",
      "2019-07-31 15:39:48,327 [MainThread] INFO UNet3DTrainer - Training iteration 162. Batch 0. Epoch [161/599]\n",
      "2019-07-31 15:39:49,561 [MainThread] INFO UNet3DTrainer - Training iteration 163. Batch 0. Epoch [162/599]\n",
      "2019-07-31 15:39:49,561 [MainThread] INFO UNet3DTrainer - Training iteration 163. Batch 0. Epoch [162/599]\n",
      "2019-07-31 15:39:50,811 [MainThread] INFO UNet3DTrainer - Training iteration 164. Batch 0. Epoch [163/599]\n",
      "2019-07-31 15:39:50,811 [MainThread] INFO UNet3DTrainer - Training iteration 164. Batch 0. Epoch [163/599]\n",
      "2019-07-31 15:39:52,053 [MainThread] INFO UNet3DTrainer - Training iteration 165. Batch 0. Epoch [164/599]\n",
      "2019-07-31 15:39:52,053 [MainThread] INFO UNet3DTrainer - Training iteration 165. Batch 0. Epoch [164/599]\n",
      "2019-07-31 15:39:53,318 [MainThread] INFO UNet3DTrainer - Training iteration 166. Batch 0. Epoch [165/599]\n",
      "2019-07-31 15:39:53,318 [MainThread] INFO UNet3DTrainer - Training iteration 166. Batch 0. Epoch [165/599]\n",
      "2019-07-31 15:39:54,578 [MainThread] INFO UNet3DTrainer - Training iteration 167. Batch 0. Epoch [166/599]\n",
      "2019-07-31 15:39:54,578 [MainThread] INFO UNet3DTrainer - Training iteration 167. Batch 0. Epoch [166/599]\n",
      "2019-07-31 15:39:55,824 [MainThread] INFO UNet3DTrainer - Training iteration 168. Batch 0. Epoch [167/599]\n",
      "2019-07-31 15:39:55,824 [MainThread] INFO UNet3DTrainer - Training iteration 168. Batch 0. Epoch [167/599]\n",
      "2019-07-31 15:39:57,066 [MainThread] INFO UNet3DTrainer - Training iteration 169. Batch 0. Epoch [168/599]\n",
      "2019-07-31 15:39:57,066 [MainThread] INFO UNet3DTrainer - Training iteration 169. Batch 0. Epoch [168/599]\n",
      "2019-07-31 15:39:58,328 [MainThread] INFO UNet3DTrainer - Training iteration 170. Batch 0. Epoch [169/599]\n",
      "2019-07-31 15:39:58,328 [MainThread] INFO UNet3DTrainer - Training iteration 170. Batch 0. Epoch [169/599]\n",
      "2019-07-31 15:39:59,609 [MainThread] INFO UNet3DTrainer - Training iteration 171. Batch 0. Epoch [170/599]\n",
      "2019-07-31 15:39:59,609 [MainThread] INFO UNet3DTrainer - Training iteration 171. Batch 0. Epoch [170/599]\n",
      "2019-07-31 15:40:00,824 [MainThread] INFO UNet3DTrainer - Training iteration 172. Batch 0. Epoch [171/599]\n",
      "2019-07-31 15:40:00,824 [MainThread] INFO UNet3DTrainer - Training iteration 172. Batch 0. Epoch [171/599]\n",
      "2019-07-31 15:40:02,095 [MainThread] INFO UNet3DTrainer - Training iteration 173. Batch 0. Epoch [172/599]\n",
      "2019-07-31 15:40:02,095 [MainThread] INFO UNet3DTrainer - Training iteration 173. Batch 0. Epoch [172/599]\n",
      "2019-07-31 15:40:03,381 [MainThread] INFO UNet3DTrainer - Training iteration 174. Batch 0. Epoch [173/599]\n",
      "2019-07-31 15:40:03,381 [MainThread] INFO UNet3DTrainer - Training iteration 174. Batch 0. Epoch [173/599]\n",
      "2019-07-31 15:40:04,602 [MainThread] INFO UNet3DTrainer - Training iteration 175. Batch 0. Epoch [174/599]\n",
      "2019-07-31 15:40:04,602 [MainThread] INFO UNet3DTrainer - Training iteration 175. Batch 0. Epoch [174/599]\n",
      "2019-07-31 15:40:05,848 [MainThread] INFO UNet3DTrainer - Training iteration 176. Batch 0. Epoch [175/599]\n",
      "2019-07-31 15:40:05,848 [MainThread] INFO UNet3DTrainer - Training iteration 176. Batch 0. Epoch [175/599]\n",
      "2019-07-31 15:40:07,086 [MainThread] INFO UNet3DTrainer - Training iteration 177. Batch 0. Epoch [176/599]\n",
      "2019-07-31 15:40:07,086 [MainThread] INFO UNet3DTrainer - Training iteration 177. Batch 0. Epoch [176/599]\n",
      "2019-07-31 15:40:08,354 [MainThread] INFO UNet3DTrainer - Training iteration 178. Batch 0. Epoch [177/599]\n",
      "2019-07-31 15:40:08,354 [MainThread] INFO UNet3DTrainer - Training iteration 178. Batch 0. Epoch [177/599]\n",
      "2019-07-31 15:40:09,582 [MainThread] INFO UNet3DTrainer - Training iteration 179. Batch 0. Epoch [178/599]\n",
      "2019-07-31 15:40:09,582 [MainThread] INFO UNet3DTrainer - Training iteration 179. Batch 0. Epoch [178/599]\n",
      "2019-07-31 15:40:10,865 [MainThread] INFO UNet3DTrainer - Training iteration 180. Batch 0. Epoch [179/599]\n",
      "2019-07-31 15:40:10,865 [MainThread] INFO UNet3DTrainer - Training iteration 180. Batch 0. Epoch [179/599]\n",
      "2019-07-31 15:40:11,664 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:40:11,664 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:40:12,996 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3232162892818451. Evaluation score: 0.512443482875824\n",
      "2019-07-31 15:40:12,996 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3232162892818451. Evaluation score: 0.512443482875824\n",
      "2019-07-31 15:40:13,002 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.512443482875824\n",
      "2019-07-31 15:40:13,002 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.512443482875824\n",
      "2019-07-31 15:40:13,005 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:40:13,005 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:40:13,231 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:40:13,231 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:40:13,549 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.019081877544522285. Evaluation score: 0.5120872259140015\n",
      "2019-07-31 15:40:13,549 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.019081877544522285. Evaluation score: 0.5120872259140015\n",
      "2019-07-31 15:40:13,551 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:40:13,551 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:40:15,030 [MainThread] INFO UNet3DTrainer - Training iteration 181. Batch 0. Epoch [180/599]\n",
      "2019-07-31 15:40:15,030 [MainThread] INFO UNet3DTrainer - Training iteration 181. Batch 0. Epoch [180/599]\n",
      "2019-07-31 15:40:15,741 [MainThread] INFO UNet3DTrainer - Training iteration 182. Batch 0. Epoch [181/599]\n",
      "2019-07-31 15:40:15,741 [MainThread] INFO UNet3DTrainer - Training iteration 182. Batch 0. Epoch [181/599]\n",
      "2019-07-31 15:40:16,978 [MainThread] INFO UNet3DTrainer - Training iteration 183. Batch 0. Epoch [182/599]\n",
      "2019-07-31 15:40:16,978 [MainThread] INFO UNet3DTrainer - Training iteration 183. Batch 0. Epoch [182/599]\n",
      "2019-07-31 15:40:18,222 [MainThread] INFO UNet3DTrainer - Training iteration 184. Batch 0. Epoch [183/599]\n",
      "2019-07-31 15:40:18,222 [MainThread] INFO UNet3DTrainer - Training iteration 184. Batch 0. Epoch [183/599]\n",
      "2019-07-31 15:40:19,475 [MainThread] INFO UNet3DTrainer - Training iteration 185. Batch 0. Epoch [184/599]\n",
      "2019-07-31 15:40:19,475 [MainThread] INFO UNet3DTrainer - Training iteration 185. Batch 0. Epoch [184/599]\n",
      "2019-07-31 15:40:20,722 [MainThread] INFO UNet3DTrainer - Training iteration 186. Batch 0. Epoch [185/599]\n",
      "2019-07-31 15:40:20,722 [MainThread] INFO UNet3DTrainer - Training iteration 186. Batch 0. Epoch [185/599]\n",
      "2019-07-31 15:40:21,982 [MainThread] INFO UNet3DTrainer - Training iteration 187. Batch 0. Epoch [186/599]\n",
      "2019-07-31 15:40:21,982 [MainThread] INFO UNet3DTrainer - Training iteration 187. Batch 0. Epoch [186/599]\n",
      "2019-07-31 15:40:23,223 [MainThread] INFO UNet3DTrainer - Training iteration 188. Batch 0. Epoch [187/599]\n",
      "2019-07-31 15:40:23,223 [MainThread] INFO UNet3DTrainer - Training iteration 188. Batch 0. Epoch [187/599]\n",
      "2019-07-31 15:40:24,470 [MainThread] INFO UNet3DTrainer - Training iteration 189. Batch 0. Epoch [188/599]\n",
      "2019-07-31 15:40:24,470 [MainThread] INFO UNet3DTrainer - Training iteration 189. Batch 0. Epoch [188/599]\n",
      "2019-07-31 15:40:25,726 [MainThread] INFO UNet3DTrainer - Training iteration 190. Batch 0. Epoch [189/599]\n",
      "2019-07-31 15:40:25,726 [MainThread] INFO UNet3DTrainer - Training iteration 190. Batch 0. Epoch [189/599]\n",
      "2019-07-31 15:40:26,992 [MainThread] INFO UNet3DTrainer - Training iteration 191. Batch 0. Epoch [190/599]\n",
      "2019-07-31 15:40:26,992 [MainThread] INFO UNet3DTrainer - Training iteration 191. Batch 0. Epoch [190/599]\n",
      "2019-07-31 15:40:28,223 [MainThread] INFO UNet3DTrainer - Training iteration 192. Batch 0. Epoch [191/599]\n",
      "2019-07-31 15:40:28,223 [MainThread] INFO UNet3DTrainer - Training iteration 192. Batch 0. Epoch [191/599]\n",
      "2019-07-31 15:40:29,483 [MainThread] INFO UNet3DTrainer - Training iteration 193. Batch 0. Epoch [192/599]\n",
      "2019-07-31 15:40:29,483 [MainThread] INFO UNet3DTrainer - Training iteration 193. Batch 0. Epoch [192/599]\n",
      "2019-07-31 15:40:30,743 [MainThread] INFO UNet3DTrainer - Training iteration 194. Batch 0. Epoch [193/599]\n",
      "2019-07-31 15:40:30,743 [MainThread] INFO UNet3DTrainer - Training iteration 194. Batch 0. Epoch [193/599]\n",
      "2019-07-31 15:40:31,984 [MainThread] INFO UNet3DTrainer - Training iteration 195. Batch 0. Epoch [194/599]\n",
      "2019-07-31 15:40:31,984 [MainThread] INFO UNet3DTrainer - Training iteration 195. Batch 0. Epoch [194/599]\n",
      "2019-07-31 15:40:33,264 [MainThread] INFO UNet3DTrainer - Training iteration 196. Batch 0. Epoch [195/599]\n",
      "2019-07-31 15:40:33,264 [MainThread] INFO UNet3DTrainer - Training iteration 196. Batch 0. Epoch [195/599]\n",
      "2019-07-31 15:40:34,512 [MainThread] INFO UNet3DTrainer - Training iteration 197. Batch 0. Epoch [196/599]\n",
      "2019-07-31 15:40:34,512 [MainThread] INFO UNet3DTrainer - Training iteration 197. Batch 0. Epoch [196/599]\n",
      "2019-07-31 15:40:35,781 [MainThread] INFO UNet3DTrainer - Training iteration 198. Batch 0. Epoch [197/599]\n",
      "2019-07-31 15:40:35,781 [MainThread] INFO UNet3DTrainer - Training iteration 198. Batch 0. Epoch [197/599]\n",
      "2019-07-31 15:40:37,030 [MainThread] INFO UNet3DTrainer - Training iteration 199. Batch 0. Epoch [198/599]\n",
      "2019-07-31 15:40:37,030 [MainThread] INFO UNet3DTrainer - Training iteration 199. Batch 0. Epoch [198/599]\n",
      "2019-07-31 15:40:38,321 [MainThread] INFO UNet3DTrainer - Training iteration 200. Batch 0. Epoch [199/599]\n",
      "2019-07-31 15:40:38,321 [MainThread] INFO UNet3DTrainer - Training iteration 200. Batch 0. Epoch [199/599]\n",
      "2019-07-31 15:40:39,074 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:40:39,074 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:40:40,399 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3222411274909973. Evaluation score: 0.5300747752189636\n",
      "2019-07-31 15:40:40,399 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3222411274909973. Evaluation score: 0.5300747752189636\n",
      "2019-07-31 15:40:40,404 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5300747752189636\n",
      "2019-07-31 15:40:40,404 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5300747752189636\n",
      "2019-07-31 15:40:40,406 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:40:40,406 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:40:40,681 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:40:40,681 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:40:40,899 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.01725924015045166. Evaluation score: 0.5300973057746887\n",
      "2019-07-31 15:40:40,899 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.01725924015045166. Evaluation score: 0.5300973057746887\n",
      "2019-07-31 15:40:40,900 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:40:40,900 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:40:42,813 [MainThread] INFO UNet3DTrainer - Training iteration 201. Batch 0. Epoch [200/599]\n",
      "2019-07-31 15:40:42,813 [MainThread] INFO UNet3DTrainer - Training iteration 201. Batch 0. Epoch [200/599]\n",
      "2019-07-31 15:40:43,523 [MainThread] INFO UNet3DTrainer - Training iteration 202. Batch 0. Epoch [201/599]\n",
      "2019-07-31 15:40:43,523 [MainThread] INFO UNet3DTrainer - Training iteration 202. Batch 0. Epoch [201/599]\n",
      "2019-07-31 15:40:44,765 [MainThread] INFO UNet3DTrainer - Training iteration 203. Batch 0. Epoch [202/599]\n",
      "2019-07-31 15:40:44,765 [MainThread] INFO UNet3DTrainer - Training iteration 203. Batch 0. Epoch [202/599]\n",
      "2019-07-31 15:40:46,014 [MainThread] INFO UNet3DTrainer - Training iteration 204. Batch 0. Epoch [203/599]\n",
      "2019-07-31 15:40:46,014 [MainThread] INFO UNet3DTrainer - Training iteration 204. Batch 0. Epoch [203/599]\n",
      "2019-07-31 15:40:47,251 [MainThread] INFO UNet3DTrainer - Training iteration 205. Batch 0. Epoch [204/599]\n",
      "2019-07-31 15:40:47,251 [MainThread] INFO UNet3DTrainer - Training iteration 205. Batch 0. Epoch [204/599]\n",
      "2019-07-31 15:40:48,507 [MainThread] INFO UNet3DTrainer - Training iteration 206. Batch 0. Epoch [205/599]\n",
      "2019-07-31 15:40:48,507 [MainThread] INFO UNet3DTrainer - Training iteration 206. Batch 0. Epoch [205/599]\n",
      "2019-07-31 15:40:49,751 [MainThread] INFO UNet3DTrainer - Training iteration 207. Batch 0. Epoch [206/599]\n",
      "2019-07-31 15:40:49,751 [MainThread] INFO UNet3DTrainer - Training iteration 207. Batch 0. Epoch [206/599]\n",
      "2019-07-31 15:40:51,003 [MainThread] INFO UNet3DTrainer - Training iteration 208. Batch 0. Epoch [207/599]\n",
      "2019-07-31 15:40:51,003 [MainThread] INFO UNet3DTrainer - Training iteration 208. Batch 0. Epoch [207/599]\n",
      "2019-07-31 15:40:52,251 [MainThread] INFO UNet3DTrainer - Training iteration 209. Batch 0. Epoch [208/599]\n",
      "2019-07-31 15:40:52,251 [MainThread] INFO UNet3DTrainer - Training iteration 209. Batch 0. Epoch [208/599]\n",
      "2019-07-31 15:40:53,512 [MainThread] INFO UNet3DTrainer - Training iteration 210. Batch 0. Epoch [209/599]\n",
      "2019-07-31 15:40:53,512 [MainThread] INFO UNet3DTrainer - Training iteration 210. Batch 0. Epoch [209/599]\n",
      "2019-07-31 15:40:54,754 [MainThread] INFO UNet3DTrainer - Training iteration 211. Batch 0. Epoch [210/599]\n",
      "2019-07-31 15:40:54,754 [MainThread] INFO UNet3DTrainer - Training iteration 211. Batch 0. Epoch [210/599]\n",
      "2019-07-31 15:40:56,018 [MainThread] INFO UNet3DTrainer - Training iteration 212. Batch 0. Epoch [211/599]\n",
      "2019-07-31 15:40:56,018 [MainThread] INFO UNet3DTrainer - Training iteration 212. Batch 0. Epoch [211/599]\n",
      "2019-07-31 15:40:57,264 [MainThread] INFO UNet3DTrainer - Training iteration 213. Batch 0. Epoch [212/599]\n",
      "2019-07-31 15:40:57,264 [MainThread] INFO UNet3DTrainer - Training iteration 213. Batch 0. Epoch [212/599]\n",
      "2019-07-31 15:40:58,512 [MainThread] INFO UNet3DTrainer - Training iteration 214. Batch 0. Epoch [213/599]\n",
      "2019-07-31 15:40:58,512 [MainThread] INFO UNet3DTrainer - Training iteration 214. Batch 0. Epoch [213/599]\n",
      "2019-07-31 15:40:59,769 [MainThread] INFO UNet3DTrainer - Training iteration 215. Batch 0. Epoch [214/599]\n",
      "2019-07-31 15:40:59,769 [MainThread] INFO UNet3DTrainer - Training iteration 215. Batch 0. Epoch [214/599]\n",
      "2019-07-31 15:41:01,031 [MainThread] INFO UNet3DTrainer - Training iteration 216. Batch 0. Epoch [215/599]\n",
      "2019-07-31 15:41:01,031 [MainThread] INFO UNet3DTrainer - Training iteration 216. Batch 0. Epoch [215/599]\n",
      "2019-07-31 15:41:02,302 [MainThread] INFO UNet3DTrainer - Training iteration 217. Batch 0. Epoch [216/599]\n",
      "2019-07-31 15:41:02,302 [MainThread] INFO UNet3DTrainer - Training iteration 217. Batch 0. Epoch [216/599]\n",
      "2019-07-31 15:41:03,560 [MainThread] INFO UNet3DTrainer - Training iteration 218. Batch 0. Epoch [217/599]\n",
      "2019-07-31 15:41:03,560 [MainThread] INFO UNet3DTrainer - Training iteration 218. Batch 0. Epoch [217/599]\n",
      "2019-07-31 15:41:04,784 [MainThread] INFO UNet3DTrainer - Training iteration 219. Batch 0. Epoch [218/599]\n",
      "2019-07-31 15:41:04,784 [MainThread] INFO UNet3DTrainer - Training iteration 219. Batch 0. Epoch [218/599]\n",
      "2019-07-31 15:41:06,057 [MainThread] INFO UNet3DTrainer - Training iteration 220. Batch 0. Epoch [219/599]\n",
      "2019-07-31 15:41:06,057 [MainThread] INFO UNet3DTrainer - Training iteration 220. Batch 0. Epoch [219/599]\n",
      "2019-07-31 15:41:06,853 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:41:06,853 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:41:08,180 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3220398426055908. Evaluation score: 0.5318823456764221\n",
      "2019-07-31 15:41:08,180 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3220398426055908. Evaluation score: 0.5318823456764221\n",
      "2019-07-31 15:41:08,185 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5318823456764221\n",
      "2019-07-31 15:41:08,185 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5318823456764221\n",
      "2019-07-31 15:41:08,188 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:41:08,188 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:41:08,458 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:41:08,458 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:41:08,685 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.016839412972331047. Evaluation score: 0.531173586845398\n",
      "2019-07-31 15:41:08,685 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.016839412972331047. Evaluation score: 0.531173586845398\n",
      "2019-07-31 15:41:08,687 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:41:08,687 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:41:10,176 [MainThread] INFO UNet3DTrainer - Training iteration 221. Batch 0. Epoch [220/599]\n",
      "2019-07-31 15:41:10,176 [MainThread] INFO UNet3DTrainer - Training iteration 221. Batch 0. Epoch [220/599]\n",
      "2019-07-31 15:41:10,912 [MainThread] INFO UNet3DTrainer - Training iteration 222. Batch 0. Epoch [221/599]\n",
      "2019-07-31 15:41:10,912 [MainThread] INFO UNet3DTrainer - Training iteration 222. Batch 0. Epoch [221/599]\n",
      "2019-07-31 15:41:12,124 [MainThread] INFO UNet3DTrainer - Training iteration 223. Batch 0. Epoch [222/599]\n",
      "2019-07-31 15:41:12,124 [MainThread] INFO UNet3DTrainer - Training iteration 223. Batch 0. Epoch [222/599]\n",
      "2019-07-31 15:41:13,365 [MainThread] INFO UNet3DTrainer - Training iteration 224. Batch 0. Epoch [223/599]\n",
      "2019-07-31 15:41:13,365 [MainThread] INFO UNet3DTrainer - Training iteration 224. Batch 0. Epoch [223/599]\n",
      "2019-07-31 15:41:14,609 [MainThread] INFO UNet3DTrainer - Training iteration 225. Batch 0. Epoch [224/599]\n",
      "2019-07-31 15:41:14,609 [MainThread] INFO UNet3DTrainer - Training iteration 225. Batch 0. Epoch [224/599]\n",
      "2019-07-31 15:41:15,886 [MainThread] INFO UNet3DTrainer - Training iteration 226. Batch 0. Epoch [225/599]\n",
      "2019-07-31 15:41:15,886 [MainThread] INFO UNet3DTrainer - Training iteration 226. Batch 0. Epoch [225/599]\n",
      "2019-07-31 15:41:17,123 [MainThread] INFO UNet3DTrainer - Training iteration 227. Batch 0. Epoch [226/599]\n",
      "2019-07-31 15:41:17,123 [MainThread] INFO UNet3DTrainer - Training iteration 227. Batch 0. Epoch [226/599]\n",
      "2019-07-31 15:41:18,375 [MainThread] INFO UNet3DTrainer - Training iteration 228. Batch 0. Epoch [227/599]\n",
      "2019-07-31 15:41:18,375 [MainThread] INFO UNet3DTrainer - Training iteration 228. Batch 0. Epoch [227/599]\n",
      "2019-07-31 15:41:19,610 [MainThread] INFO UNet3DTrainer - Training iteration 229. Batch 0. Epoch [228/599]\n",
      "2019-07-31 15:41:19,610 [MainThread] INFO UNet3DTrainer - Training iteration 229. Batch 0. Epoch [228/599]\n",
      "2019-07-31 15:41:20,902 [MainThread] INFO UNet3DTrainer - Training iteration 230. Batch 0. Epoch [229/599]\n",
      "2019-07-31 15:41:20,902 [MainThread] INFO UNet3DTrainer - Training iteration 230. Batch 0. Epoch [229/599]\n",
      "2019-07-31 15:41:22,149 [MainThread] INFO UNet3DTrainer - Training iteration 231. Batch 0. Epoch [230/599]\n",
      "2019-07-31 15:41:22,149 [MainThread] INFO UNet3DTrainer - Training iteration 231. Batch 0. Epoch [230/599]\n",
      "2019-07-31 15:41:23,392 [MainThread] INFO UNet3DTrainer - Training iteration 232. Batch 0. Epoch [231/599]\n",
      "2019-07-31 15:41:23,392 [MainThread] INFO UNet3DTrainer - Training iteration 232. Batch 0. Epoch [231/599]\n",
      "2019-07-31 15:41:24,660 [MainThread] INFO UNet3DTrainer - Training iteration 233. Batch 0. Epoch [232/599]\n",
      "2019-07-31 15:41:24,660 [MainThread] INFO UNet3DTrainer - Training iteration 233. Batch 0. Epoch [232/599]\n",
      "2019-07-31 15:41:25,942 [MainThread] INFO UNet3DTrainer - Training iteration 234. Batch 0. Epoch [233/599]\n",
      "2019-07-31 15:41:25,942 [MainThread] INFO UNet3DTrainer - Training iteration 234. Batch 0. Epoch [233/599]\n",
      "2019-07-31 15:41:27,183 [MainThread] INFO UNet3DTrainer - Training iteration 235. Batch 0. Epoch [234/599]\n",
      "2019-07-31 15:41:27,183 [MainThread] INFO UNet3DTrainer - Training iteration 235. Batch 0. Epoch [234/599]\n",
      "2019-07-31 15:41:28,379 [MainThread] INFO UNet3DTrainer - Training iteration 236. Batch 0. Epoch [235/599]\n",
      "2019-07-31 15:41:28,379 [MainThread] INFO UNet3DTrainer - Training iteration 236. Batch 0. Epoch [235/599]\n",
      "2019-07-31 15:41:29,657 [MainThread] INFO UNet3DTrainer - Training iteration 237. Batch 0. Epoch [236/599]\n",
      "2019-07-31 15:41:29,657 [MainThread] INFO UNet3DTrainer - Training iteration 237. Batch 0. Epoch [236/599]\n",
      "2019-07-31 15:41:30,901 [MainThread] INFO UNet3DTrainer - Training iteration 238. Batch 0. Epoch [237/599]\n",
      "2019-07-31 15:41:30,901 [MainThread] INFO UNet3DTrainer - Training iteration 238. Batch 0. Epoch [237/599]\n",
      "2019-07-31 15:41:32,148 [MainThread] INFO UNet3DTrainer - Training iteration 239. Batch 0. Epoch [238/599]\n",
      "2019-07-31 15:41:32,148 [MainThread] INFO UNet3DTrainer - Training iteration 239. Batch 0. Epoch [238/599]\n",
      "2019-07-31 15:41:33,416 [MainThread] INFO UNet3DTrainer - Training iteration 240. Batch 0. Epoch [239/599]\n",
      "2019-07-31 15:41:33,416 [MainThread] INFO UNet3DTrainer - Training iteration 240. Batch 0. Epoch [239/599]\n",
      "2019-07-31 15:41:34,218 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:41:34,218 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:41:35,541 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3218521177768707. Evaluation score: 0.5338764190673828\n",
      "2019-07-31 15:41:35,541 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3218521177768707. Evaluation score: 0.5338764190673828\n",
      "2019-07-31 15:41:35,547 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5338764190673828\n",
      "2019-07-31 15:41:35,547 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5338764190673828\n",
      "2019-07-31 15:41:35,548 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:41:35,548 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:41:35,822 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:41:35,822 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:41:36,050 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.016493385657668114. Evaluation score: 0.5338764190673828\n",
      "2019-07-31 15:41:36,050 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.016493385657668114. Evaluation score: 0.5338764190673828\n",
      "2019-07-31 15:41:36,052 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:41:36,052 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:41:37,575 [MainThread] INFO UNet3DTrainer - Training iteration 241. Batch 0. Epoch [240/599]\n",
      "2019-07-31 15:41:37,575 [MainThread] INFO UNet3DTrainer - Training iteration 241. Batch 0. Epoch [240/599]\n",
      "2019-07-31 15:41:38,290 [MainThread] INFO UNet3DTrainer - Training iteration 242. Batch 0. Epoch [241/599]\n",
      "2019-07-31 15:41:38,290 [MainThread] INFO UNet3DTrainer - Training iteration 242. Batch 0. Epoch [241/599]\n",
      "2019-07-31 15:41:39,520 [MainThread] INFO UNet3DTrainer - Training iteration 243. Batch 0. Epoch [242/599]\n",
      "2019-07-31 15:41:39,520 [MainThread] INFO UNet3DTrainer - Training iteration 243. Batch 0. Epoch [242/599]\n",
      "2019-07-31 15:41:40,757 [MainThread] INFO UNet3DTrainer - Training iteration 244. Batch 0. Epoch [243/599]\n",
      "2019-07-31 15:41:40,757 [MainThread] INFO UNet3DTrainer - Training iteration 244. Batch 0. Epoch [243/599]\n",
      "2019-07-31 15:41:42,022 [MainThread] INFO UNet3DTrainer - Training iteration 245. Batch 0. Epoch [244/599]\n",
      "2019-07-31 15:41:42,022 [MainThread] INFO UNet3DTrainer - Training iteration 245. Batch 0. Epoch [244/599]\n",
      "2019-07-31 15:41:43,297 [MainThread] INFO UNet3DTrainer - Training iteration 246. Batch 0. Epoch [245/599]\n",
      "2019-07-31 15:41:43,297 [MainThread] INFO UNet3DTrainer - Training iteration 246. Batch 0. Epoch [245/599]\n",
      "2019-07-31 15:41:44,509 [MainThread] INFO UNet3DTrainer - Training iteration 247. Batch 0. Epoch [246/599]\n",
      "2019-07-31 15:41:44,509 [MainThread] INFO UNet3DTrainer - Training iteration 247. Batch 0. Epoch [246/599]\n",
      "2019-07-31 15:41:45,774 [MainThread] INFO UNet3DTrainer - Training iteration 248. Batch 0. Epoch [247/599]\n",
      "2019-07-31 15:41:45,774 [MainThread] INFO UNet3DTrainer - Training iteration 248. Batch 0. Epoch [247/599]\n",
      "2019-07-31 15:41:47,028 [MainThread] INFO UNet3DTrainer - Training iteration 249. Batch 0. Epoch [248/599]\n",
      "2019-07-31 15:41:47,028 [MainThread] INFO UNet3DTrainer - Training iteration 249. Batch 0. Epoch [248/599]\n",
      "2019-07-31 15:41:48,271 [MainThread] INFO UNet3DTrainer - Training iteration 250. Batch 0. Epoch [249/599]\n",
      "2019-07-31 15:41:48,271 [MainThread] INFO UNet3DTrainer - Training iteration 250. Batch 0. Epoch [249/599]\n",
      "2019-07-31 15:41:49,580 [MainThread] INFO UNet3DTrainer - Training iteration 251. Batch 0. Epoch [250/599]\n",
      "2019-07-31 15:41:49,580 [MainThread] INFO UNet3DTrainer - Training iteration 251. Batch 0. Epoch [250/599]\n",
      "2019-07-31 15:41:50,810 [MainThread] INFO UNet3DTrainer - Training iteration 252. Batch 0. Epoch [251/599]\n",
      "2019-07-31 15:41:50,810 [MainThread] INFO UNet3DTrainer - Training iteration 252. Batch 0. Epoch [251/599]\n",
      "2019-07-31 15:41:52,056 [MainThread] INFO UNet3DTrainer - Training iteration 253. Batch 0. Epoch [252/599]\n",
      "2019-07-31 15:41:52,056 [MainThread] INFO UNet3DTrainer - Training iteration 253. Batch 0. Epoch [252/599]\n",
      "2019-07-31 15:41:53,283 [MainThread] INFO UNet3DTrainer - Training iteration 254. Batch 0. Epoch [253/599]\n",
      "2019-07-31 15:41:53,283 [MainThread] INFO UNet3DTrainer - Training iteration 254. Batch 0. Epoch [253/599]\n",
      "2019-07-31 15:41:54,539 [MainThread] INFO UNet3DTrainer - Training iteration 255. Batch 0. Epoch [254/599]\n",
      "2019-07-31 15:41:54,539 [MainThread] INFO UNet3DTrainer - Training iteration 255. Batch 0. Epoch [254/599]\n",
      "2019-07-31 15:41:55,791 [MainThread] INFO UNet3DTrainer - Training iteration 256. Batch 0. Epoch [255/599]\n",
      "2019-07-31 15:41:55,791 [MainThread] INFO UNet3DTrainer - Training iteration 256. Batch 0. Epoch [255/599]\n",
      "2019-07-31 15:41:57,056 [MainThread] INFO UNet3DTrainer - Training iteration 257. Batch 0. Epoch [256/599]\n",
      "2019-07-31 15:41:57,056 [MainThread] INFO UNet3DTrainer - Training iteration 257. Batch 0. Epoch [256/599]\n",
      "2019-07-31 15:41:58,337 [MainThread] INFO UNet3DTrainer - Training iteration 258. Batch 0. Epoch [257/599]\n",
      "2019-07-31 15:41:58,337 [MainThread] INFO UNet3DTrainer - Training iteration 258. Batch 0. Epoch [257/599]\n",
      "2019-07-31 15:41:59,569 [MainThread] INFO UNet3DTrainer - Training iteration 259. Batch 0. Epoch [258/599]\n",
      "2019-07-31 15:41:59,569 [MainThread] INFO UNet3DTrainer - Training iteration 259. Batch 0. Epoch [258/599]\n",
      "2019-07-31 15:42:00,827 [MainThread] INFO UNet3DTrainer - Training iteration 260. Batch 0. Epoch [259/599]\n",
      "2019-07-31 15:42:00,827 [MainThread] INFO UNet3DTrainer - Training iteration 260. Batch 0. Epoch [259/599]\n",
      "2019-07-31 15:42:01,619 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:42:01,619 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:42:02,935 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3216814398765564. Evaluation score: 0.5332472324371338\n",
      "2019-07-31 15:42:02,935 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3216814398765564. Evaluation score: 0.5332472324371338\n",
      "2019-07-31 15:42:02,941 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:42:02,941 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:42:03,217 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.0161740705370903. Evaluation score: 0.5332472324371338\n",
      "2019-07-31 15:42:03,217 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.0161740705370903. Evaluation score: 0.5332472324371338\n",
      "2019-07-31 15:42:03,219 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:42:03,219 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:42:04,629 [MainThread] INFO UNet3DTrainer - Training iteration 261. Batch 0. Epoch [260/599]\n",
      "2019-07-31 15:42:04,629 [MainThread] INFO UNet3DTrainer - Training iteration 261. Batch 0. Epoch [260/599]\n",
      "2019-07-31 15:42:05,336 [MainThread] INFO UNet3DTrainer - Training iteration 262. Batch 0. Epoch [261/599]\n",
      "2019-07-31 15:42:05,336 [MainThread] INFO UNet3DTrainer - Training iteration 262. Batch 0. Epoch [261/599]\n",
      "2019-07-31 15:42:06,564 [MainThread] INFO UNet3DTrainer - Training iteration 263. Batch 0. Epoch [262/599]\n",
      "2019-07-31 15:42:06,564 [MainThread] INFO UNet3DTrainer - Training iteration 263. Batch 0. Epoch [262/599]\n",
      "2019-07-31 15:42:07,815 [MainThread] INFO UNet3DTrainer - Training iteration 264. Batch 0. Epoch [263/599]\n",
      "2019-07-31 15:42:07,815 [MainThread] INFO UNet3DTrainer - Training iteration 264. Batch 0. Epoch [263/599]\n",
      "2019-07-31 15:42:09,072 [MainThread] INFO UNet3DTrainer - Training iteration 265. Batch 0. Epoch [264/599]\n",
      "2019-07-31 15:42:09,072 [MainThread] INFO UNet3DTrainer - Training iteration 265. Batch 0. Epoch [264/599]\n",
      "2019-07-31 15:42:10,311 [MainThread] INFO UNet3DTrainer - Training iteration 266. Batch 0. Epoch [265/599]\n",
      "2019-07-31 15:42:10,311 [MainThread] INFO UNet3DTrainer - Training iteration 266. Batch 0. Epoch [265/599]\n",
      "2019-07-31 15:42:11,581 [MainThread] INFO UNet3DTrainer - Training iteration 267. Batch 0. Epoch [266/599]\n",
      "2019-07-31 15:42:11,581 [MainThread] INFO UNet3DTrainer - Training iteration 267. Batch 0. Epoch [266/599]\n",
      "2019-07-31 15:42:12,836 [MainThread] INFO UNet3DTrainer - Training iteration 268. Batch 0. Epoch [267/599]\n",
      "2019-07-31 15:42:12,836 [MainThread] INFO UNet3DTrainer - Training iteration 268. Batch 0. Epoch [267/599]\n",
      "2019-07-31 15:42:14,075 [MainThread] INFO UNet3DTrainer - Training iteration 269. Batch 0. Epoch [268/599]\n",
      "2019-07-31 15:42:14,075 [MainThread] INFO UNet3DTrainer - Training iteration 269. Batch 0. Epoch [268/599]\n",
      "2019-07-31 15:42:15,318 [MainThread] INFO UNet3DTrainer - Training iteration 270. Batch 0. Epoch [269/599]\n",
      "2019-07-31 15:42:15,318 [MainThread] INFO UNet3DTrainer - Training iteration 270. Batch 0. Epoch [269/599]\n",
      "2019-07-31 15:42:16,573 [MainThread] INFO UNet3DTrainer - Training iteration 271. Batch 0. Epoch [270/599]\n",
      "2019-07-31 15:42:16,573 [MainThread] INFO UNet3DTrainer - Training iteration 271. Batch 0. Epoch [270/599]\n",
      "2019-07-31 15:42:17,820 [MainThread] INFO UNet3DTrainer - Training iteration 272. Batch 0. Epoch [271/599]\n",
      "2019-07-31 15:42:17,820 [MainThread] INFO UNet3DTrainer - Training iteration 272. Batch 0. Epoch [271/599]\n",
      "2019-07-31 15:42:19,101 [MainThread] INFO UNet3DTrainer - Training iteration 273. Batch 0. Epoch [272/599]\n",
      "2019-07-31 15:42:19,101 [MainThread] INFO UNet3DTrainer - Training iteration 273. Batch 0. Epoch [272/599]\n",
      "2019-07-31 15:42:20,331 [MainThread] INFO UNet3DTrainer - Training iteration 274. Batch 0. Epoch [273/599]\n",
      "2019-07-31 15:42:20,331 [MainThread] INFO UNet3DTrainer - Training iteration 274. Batch 0. Epoch [273/599]\n",
      "2019-07-31 15:42:21,576 [MainThread] INFO UNet3DTrainer - Training iteration 275. Batch 0. Epoch [274/599]\n",
      "2019-07-31 15:42:21,576 [MainThread] INFO UNet3DTrainer - Training iteration 275. Batch 0. Epoch [274/599]\n",
      "2019-07-31 15:42:22,880 [MainThread] INFO UNet3DTrainer - Training iteration 276. Batch 0. Epoch [275/599]\n",
      "2019-07-31 15:42:22,880 [MainThread] INFO UNet3DTrainer - Training iteration 276. Batch 0. Epoch [275/599]\n",
      "2019-07-31 15:42:24,098 [MainThread] INFO UNet3DTrainer - Training iteration 277. Batch 0. Epoch [276/599]\n",
      "2019-07-31 15:42:24,098 [MainThread] INFO UNet3DTrainer - Training iteration 277. Batch 0. Epoch [276/599]\n",
      "2019-07-31 15:42:25,389 [MainThread] INFO UNet3DTrainer - Training iteration 278. Batch 0. Epoch [277/599]\n",
      "2019-07-31 15:42:25,389 [MainThread] INFO UNet3DTrainer - Training iteration 278. Batch 0. Epoch [277/599]\n",
      "2019-07-31 15:42:26,628 [MainThread] INFO UNet3DTrainer - Training iteration 279. Batch 0. Epoch [278/599]\n",
      "2019-07-31 15:42:26,628 [MainThread] INFO UNet3DTrainer - Training iteration 279. Batch 0. Epoch [278/599]\n",
      "2019-07-31 15:42:27,865 [MainThread] INFO UNet3DTrainer - Training iteration 280. Batch 0. Epoch [279/599]\n",
      "2019-07-31 15:42:27,865 [MainThread] INFO UNet3DTrainer - Training iteration 280. Batch 0. Epoch [279/599]\n",
      "2019-07-31 15:42:28,669 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:42:28,669 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:42:29,997 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.32151514291763306. Evaluation score: 0.5353319048881531\n",
      "2019-07-31 15:42:29,997 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.32151514291763306. Evaluation score: 0.5353319048881531\n",
      "2019-07-31 15:42:30,003 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5353319048881531\n",
      "2019-07-31 15:42:30,003 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5353319048881531\n",
      "2019-07-31 15:42:30,005 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:42:30,005 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:42:30,239 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:42:30,239 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:42:30,546 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.01586253196001053. Evaluation score: 0.5353319048881531\n",
      "2019-07-31 15:42:30,546 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.01586253196001053. Evaluation score: 0.5353319048881531\n",
      "2019-07-31 15:42:30,547 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:42:30,547 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:42:32,015 [MainThread] INFO UNet3DTrainer - Training iteration 281. Batch 0. Epoch [280/599]\n",
      "2019-07-31 15:42:32,015 [MainThread] INFO UNet3DTrainer - Training iteration 281. Batch 0. Epoch [280/599]\n",
      "2019-07-31 15:42:32,751 [MainThread] INFO UNet3DTrainer - Training iteration 282. Batch 0. Epoch [281/599]\n",
      "2019-07-31 15:42:32,751 [MainThread] INFO UNet3DTrainer - Training iteration 282. Batch 0. Epoch [281/599]\n",
      "2019-07-31 15:42:33,948 [MainThread] INFO UNet3DTrainer - Training iteration 283. Batch 0. Epoch [282/599]\n",
      "2019-07-31 15:42:33,948 [MainThread] INFO UNet3DTrainer - Training iteration 283. Batch 0. Epoch [282/599]\n",
      "2019-07-31 15:42:35,218 [MainThread] INFO UNet3DTrainer - Training iteration 284. Batch 0. Epoch [283/599]\n",
      "2019-07-31 15:42:35,218 [MainThread] INFO UNet3DTrainer - Training iteration 284. Batch 0. Epoch [283/599]\n",
      "2019-07-31 15:42:36,454 [MainThread] INFO UNet3DTrainer - Training iteration 285. Batch 0. Epoch [284/599]\n",
      "2019-07-31 15:42:36,454 [MainThread] INFO UNet3DTrainer - Training iteration 285. Batch 0. Epoch [284/599]\n",
      "2019-07-31 15:42:37,697 [MainThread] INFO UNet3DTrainer - Training iteration 286. Batch 0. Epoch [285/599]\n",
      "2019-07-31 15:42:37,697 [MainThread] INFO UNet3DTrainer - Training iteration 286. Batch 0. Epoch [285/599]\n",
      "2019-07-31 15:42:38,951 [MainThread] INFO UNet3DTrainer - Training iteration 287. Batch 0. Epoch [286/599]\n",
      "2019-07-31 15:42:38,951 [MainThread] INFO UNet3DTrainer - Training iteration 287. Batch 0. Epoch [286/599]\n",
      "2019-07-31 15:42:40,197 [MainThread] INFO UNet3DTrainer - Training iteration 288. Batch 0. Epoch [287/599]\n",
      "2019-07-31 15:42:40,197 [MainThread] INFO UNet3DTrainer - Training iteration 288. Batch 0. Epoch [287/599]\n",
      "2019-07-31 15:42:41,448 [MainThread] INFO UNet3DTrainer - Training iteration 289. Batch 0. Epoch [288/599]\n",
      "2019-07-31 15:42:41,448 [MainThread] INFO UNet3DTrainer - Training iteration 289. Batch 0. Epoch [288/599]\n",
      "2019-07-31 15:42:42,702 [MainThread] INFO UNet3DTrainer - Training iteration 290. Batch 0. Epoch [289/599]\n",
      "2019-07-31 15:42:42,702 [MainThread] INFO UNet3DTrainer - Training iteration 290. Batch 0. Epoch [289/599]\n",
      "2019-07-31 15:42:43,964 [MainThread] INFO UNet3DTrainer - Training iteration 291. Batch 0. Epoch [290/599]\n",
      "2019-07-31 15:42:43,964 [MainThread] INFO UNet3DTrainer - Training iteration 291. Batch 0. Epoch [290/599]\n",
      "2019-07-31 15:42:45,220 [MainThread] INFO UNet3DTrainer - Training iteration 292. Batch 0. Epoch [291/599]\n",
      "2019-07-31 15:42:45,220 [MainThread] INFO UNet3DTrainer - Training iteration 292. Batch 0. Epoch [291/599]\n",
      "2019-07-31 15:42:46,492 [MainThread] INFO UNet3DTrainer - Training iteration 293. Batch 0. Epoch [292/599]\n",
      "2019-07-31 15:42:46,492 [MainThread] INFO UNet3DTrainer - Training iteration 293. Batch 0. Epoch [292/599]\n",
      "2019-07-31 15:42:47,717 [MainThread] INFO UNet3DTrainer - Training iteration 294. Batch 0. Epoch [293/599]\n",
      "2019-07-31 15:42:47,717 [MainThread] INFO UNet3DTrainer - Training iteration 294. Batch 0. Epoch [293/599]\n",
      "2019-07-31 15:42:49,008 [MainThread] INFO UNet3DTrainer - Training iteration 295. Batch 0. Epoch [294/599]\n",
      "2019-07-31 15:42:49,008 [MainThread] INFO UNet3DTrainer - Training iteration 295. Batch 0. Epoch [294/599]\n",
      "2019-07-31 15:42:50,261 [MainThread] INFO UNet3DTrainer - Training iteration 296. Batch 0. Epoch [295/599]\n",
      "2019-07-31 15:42:50,261 [MainThread] INFO UNet3DTrainer - Training iteration 296. Batch 0. Epoch [295/599]\n",
      "2019-07-31 15:42:51,483 [MainThread] INFO UNet3DTrainer - Training iteration 297. Batch 0. Epoch [296/599]\n",
      "2019-07-31 15:42:51,483 [MainThread] INFO UNet3DTrainer - Training iteration 297. Batch 0. Epoch [296/599]\n",
      "2019-07-31 15:42:52,743 [MainThread] INFO UNet3DTrainer - Training iteration 298. Batch 0. Epoch [297/599]\n",
      "2019-07-31 15:42:52,743 [MainThread] INFO UNet3DTrainer - Training iteration 298. Batch 0. Epoch [297/599]\n",
      "2019-07-31 15:42:53,971 [MainThread] INFO UNet3DTrainer - Training iteration 299. Batch 0. Epoch [298/599]\n",
      "2019-07-31 15:42:53,971 [MainThread] INFO UNet3DTrainer - Training iteration 299. Batch 0. Epoch [298/599]\n",
      "2019-07-31 15:42:55,270 [MainThread] INFO UNet3DTrainer - Training iteration 300. Batch 0. Epoch [299/599]\n",
      "2019-07-31 15:42:55,270 [MainThread] INFO UNet3DTrainer - Training iteration 300. Batch 0. Epoch [299/599]\n",
      "2019-07-31 15:42:56,056 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:42:56,056 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:42:57,382 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3213493227958679. Evaluation score: 0.5356430411338806\n",
      "2019-07-31 15:42:57,382 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3213493227958679. Evaluation score: 0.5356430411338806\n",
      "2019-07-31 15:42:57,387 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5356430411338806\n",
      "2019-07-31 15:42:57,387 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5356430411338806\n",
      "2019-07-31 15:42:57,389 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:42:57,389 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:42:57,653 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:42:57,653 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:42:57,879 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.015554497949779034. Evaluation score: 0.5356430411338806\n",
      "2019-07-31 15:42:57,879 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.015554497949779034. Evaluation score: 0.5356430411338806\n",
      "2019-07-31 15:42:57,881 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:42:57,881 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:42:59,665 [MainThread] INFO UNet3DTrainer - Training iteration 301. Batch 0. Epoch [300/599]\n",
      "2019-07-31 15:42:59,665 [MainThread] INFO UNet3DTrainer - Training iteration 301. Batch 0. Epoch [300/599]\n",
      "2019-07-31 15:43:00,346 [MainThread] INFO UNet3DTrainer - Training iteration 302. Batch 0. Epoch [301/599]\n",
      "2019-07-31 15:43:00,346 [MainThread] INFO UNet3DTrainer - Training iteration 302. Batch 0. Epoch [301/599]\n",
      "2019-07-31 15:43:01,597 [MainThread] INFO UNet3DTrainer - Training iteration 303. Batch 0. Epoch [302/599]\n",
      "2019-07-31 15:43:01,597 [MainThread] INFO UNet3DTrainer - Training iteration 303. Batch 0. Epoch [302/599]\n",
      "2019-07-31 15:43:02,862 [MainThread] INFO UNet3DTrainer - Training iteration 304. Batch 0. Epoch [303/599]\n",
      "2019-07-31 15:43:02,862 [MainThread] INFO UNet3DTrainer - Training iteration 304. Batch 0. Epoch [303/599]\n",
      "2019-07-31 15:43:04,105 [MainThread] INFO UNet3DTrainer - Training iteration 305. Batch 0. Epoch [304/599]\n",
      "2019-07-31 15:43:04,105 [MainThread] INFO UNet3DTrainer - Training iteration 305. Batch 0. Epoch [304/599]\n",
      "2019-07-31 15:43:05,356 [MainThread] INFO UNet3DTrainer - Training iteration 306. Batch 0. Epoch [305/599]\n",
      "2019-07-31 15:43:05,356 [MainThread] INFO UNet3DTrainer - Training iteration 306. Batch 0. Epoch [305/599]\n",
      "2019-07-31 15:43:06,615 [MainThread] INFO UNet3DTrainer - Training iteration 307. Batch 0. Epoch [306/599]\n",
      "2019-07-31 15:43:06,615 [MainThread] INFO UNet3DTrainer - Training iteration 307. Batch 0. Epoch [306/599]\n",
      "2019-07-31 15:43:07,844 [MainThread] INFO UNet3DTrainer - Training iteration 308. Batch 0. Epoch [307/599]\n",
      "2019-07-31 15:43:07,844 [MainThread] INFO UNet3DTrainer - Training iteration 308. Batch 0. Epoch [307/599]\n",
      "2019-07-31 15:43:09,102 [MainThread] INFO UNet3DTrainer - Training iteration 309. Batch 0. Epoch [308/599]\n",
      "2019-07-31 15:43:09,102 [MainThread] INFO UNet3DTrainer - Training iteration 309. Batch 0. Epoch [308/599]\n",
      "2019-07-31 15:43:10,347 [MainThread] INFO UNet3DTrainer - Training iteration 310. Batch 0. Epoch [309/599]\n",
      "2019-07-31 15:43:10,347 [MainThread] INFO UNet3DTrainer - Training iteration 310. Batch 0. Epoch [309/599]\n",
      "2019-07-31 15:43:11,620 [MainThread] INFO UNet3DTrainer - Training iteration 311. Batch 0. Epoch [310/599]\n",
      "2019-07-31 15:43:11,620 [MainThread] INFO UNet3DTrainer - Training iteration 311. Batch 0. Epoch [310/599]\n",
      "2019-07-31 15:43:12,911 [MainThread] INFO UNet3DTrainer - Training iteration 312. Batch 0. Epoch [311/599]\n",
      "2019-07-31 15:43:12,911 [MainThread] INFO UNet3DTrainer - Training iteration 312. Batch 0. Epoch [311/599]\n",
      "2019-07-31 15:43:14,114 [MainThread] INFO UNet3DTrainer - Training iteration 313. Batch 0. Epoch [312/599]\n",
      "2019-07-31 15:43:14,114 [MainThread] INFO UNet3DTrainer - Training iteration 313. Batch 0. Epoch [312/599]\n",
      "2019-07-31 15:43:15,361 [MainThread] INFO UNet3DTrainer - Training iteration 314. Batch 0. Epoch [313/599]\n",
      "2019-07-31 15:43:15,361 [MainThread] INFO UNet3DTrainer - Training iteration 314. Batch 0. Epoch [313/599]\n",
      "2019-07-31 15:43:16,655 [MainThread] INFO UNet3DTrainer - Training iteration 315. Batch 0. Epoch [314/599]\n",
      "2019-07-31 15:43:16,655 [MainThread] INFO UNet3DTrainer - Training iteration 315. Batch 0. Epoch [314/599]\n",
      "2019-07-31 15:43:17,927 [MainThread] INFO UNet3DTrainer - Training iteration 316. Batch 0. Epoch [315/599]\n",
      "2019-07-31 15:43:17,927 [MainThread] INFO UNet3DTrainer - Training iteration 316. Batch 0. Epoch [315/599]\n",
      "2019-07-31 15:43:19,161 [MainThread] INFO UNet3DTrainer - Training iteration 317. Batch 0. Epoch [316/599]\n",
      "2019-07-31 15:43:19,161 [MainThread] INFO UNet3DTrainer - Training iteration 317. Batch 0. Epoch [316/599]\n",
      "2019-07-31 15:43:20,406 [MainThread] INFO UNet3DTrainer - Training iteration 318. Batch 0. Epoch [317/599]\n",
      "2019-07-31 15:43:20,406 [MainThread] INFO UNet3DTrainer - Training iteration 318. Batch 0. Epoch [317/599]\n",
      "2019-07-31 15:43:21,652 [MainThread] INFO UNet3DTrainer - Training iteration 319. Batch 0. Epoch [318/599]\n",
      "2019-07-31 15:43:21,652 [MainThread] INFO UNet3DTrainer - Training iteration 319. Batch 0. Epoch [318/599]\n",
      "2019-07-31 15:43:22,890 [MainThread] INFO UNet3DTrainer - Training iteration 320. Batch 0. Epoch [319/599]\n",
      "2019-07-31 15:43:22,890 [MainThread] INFO UNet3DTrainer - Training iteration 320. Batch 0. Epoch [319/599]\n",
      "2019-07-31 15:43:23,707 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:43:23,707 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:43:25,037 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3211858868598938. Evaluation score: 0.5377672910690308\n",
      "2019-07-31 15:43:25,037 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3211858868598938. Evaluation score: 0.5377672910690308\n",
      "2019-07-31 15:43:25,044 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5377672910690308\n",
      "2019-07-31 15:43:25,044 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5377672910690308\n",
      "2019-07-31 15:43:25,046 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:43:25,046 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:43:25,322 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:43:25,322 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:43:25,732 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.015249955467879772. Evaluation score: 0.5370416641235352\n",
      "2019-07-31 15:43:25,732 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.015249955467879772. Evaluation score: 0.5370416641235352\n",
      "2019-07-31 15:43:25,734 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:43:25,734 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:43:27,165 [MainThread] INFO UNet3DTrainer - Training iteration 321. Batch 0. Epoch [320/599]\n",
      "2019-07-31 15:43:27,165 [MainThread] INFO UNet3DTrainer - Training iteration 321. Batch 0. Epoch [320/599]\n",
      "2019-07-31 15:43:27,869 [MainThread] INFO UNet3DTrainer - Training iteration 322. Batch 0. Epoch [321/599]\n",
      "2019-07-31 15:43:27,869 [MainThread] INFO UNet3DTrainer - Training iteration 322. Batch 0. Epoch [321/599]\n",
      "2019-07-31 15:43:29,103 [MainThread] INFO UNet3DTrainer - Training iteration 323. Batch 0. Epoch [322/599]\n",
      "2019-07-31 15:43:29,103 [MainThread] INFO UNet3DTrainer - Training iteration 323. Batch 0. Epoch [322/599]\n",
      "2019-07-31 15:43:30,342 [MainThread] INFO UNet3DTrainer - Training iteration 324. Batch 0. Epoch [323/599]\n",
      "2019-07-31 15:43:30,342 [MainThread] INFO UNet3DTrainer - Training iteration 324. Batch 0. Epoch [323/599]\n",
      "2019-07-31 15:43:31,597 [MainThread] INFO UNet3DTrainer - Training iteration 325. Batch 0. Epoch [324/599]\n",
      "2019-07-31 15:43:31,597 [MainThread] INFO UNet3DTrainer - Training iteration 325. Batch 0. Epoch [324/599]\n",
      "2019-07-31 15:43:32,868 [MainThread] INFO UNet3DTrainer - Training iteration 326. Batch 0. Epoch [325/599]\n",
      "2019-07-31 15:43:32,868 [MainThread] INFO UNet3DTrainer - Training iteration 326. Batch 0. Epoch [325/599]\n",
      "2019-07-31 15:43:34,098 [MainThread] INFO UNet3DTrainer - Training iteration 327. Batch 0. Epoch [326/599]\n",
      "2019-07-31 15:43:34,098 [MainThread] INFO UNet3DTrainer - Training iteration 327. Batch 0. Epoch [326/599]\n",
      "2019-07-31 15:43:35,348 [MainThread] INFO UNet3DTrainer - Training iteration 328. Batch 0. Epoch [327/599]\n",
      "2019-07-31 15:43:35,348 [MainThread] INFO UNet3DTrainer - Training iteration 328. Batch 0. Epoch [327/599]\n",
      "2019-07-31 15:43:36,611 [MainThread] INFO UNet3DTrainer - Training iteration 329. Batch 0. Epoch [328/599]\n",
      "2019-07-31 15:43:36,611 [MainThread] INFO UNet3DTrainer - Training iteration 329. Batch 0. Epoch [328/599]\n",
      "2019-07-31 15:43:37,861 [MainThread] INFO UNet3DTrainer - Training iteration 330. Batch 0. Epoch [329/599]\n",
      "2019-07-31 15:43:37,861 [MainThread] INFO UNet3DTrainer - Training iteration 330. Batch 0. Epoch [329/599]\n",
      "2019-07-31 15:43:39,107 [MainThread] INFO UNet3DTrainer - Training iteration 331. Batch 0. Epoch [330/599]\n",
      "2019-07-31 15:43:39,107 [MainThread] INFO UNet3DTrainer - Training iteration 331. Batch 0. Epoch [330/599]\n",
      "2019-07-31 15:43:40,381 [MainThread] INFO UNet3DTrainer - Training iteration 332. Batch 0. Epoch [331/599]\n",
      "2019-07-31 15:43:40,381 [MainThread] INFO UNet3DTrainer - Training iteration 332. Batch 0. Epoch [331/599]\n",
      "2019-07-31 15:43:41,638 [MainThread] INFO UNet3DTrainer - Training iteration 333. Batch 0. Epoch [332/599]\n",
      "2019-07-31 15:43:41,638 [MainThread] INFO UNet3DTrainer - Training iteration 333. Batch 0. Epoch [332/599]\n",
      "2019-07-31 15:43:42,903 [MainThread] INFO UNet3DTrainer - Training iteration 334. Batch 0. Epoch [333/599]\n",
      "2019-07-31 15:43:42,903 [MainThread] INFO UNet3DTrainer - Training iteration 334. Batch 0. Epoch [333/599]\n",
      "2019-07-31 15:43:44,126 [MainThread] INFO UNet3DTrainer - Training iteration 335. Batch 0. Epoch [334/599]\n",
      "2019-07-31 15:43:44,126 [MainThread] INFO UNet3DTrainer - Training iteration 335. Batch 0. Epoch [334/599]\n",
      "2019-07-31 15:43:45,380 [MainThread] INFO UNet3DTrainer - Training iteration 336. Batch 0. Epoch [335/599]\n",
      "2019-07-31 15:43:45,380 [MainThread] INFO UNet3DTrainer - Training iteration 336. Batch 0. Epoch [335/599]\n",
      "2019-07-31 15:43:46,665 [MainThread] INFO UNet3DTrainer - Training iteration 337. Batch 0. Epoch [336/599]\n",
      "2019-07-31 15:43:46,665 [MainThread] INFO UNet3DTrainer - Training iteration 337. Batch 0. Epoch [336/599]\n",
      "2019-07-31 15:43:47,888 [MainThread] INFO UNet3DTrainer - Training iteration 338. Batch 0. Epoch [337/599]\n",
      "2019-07-31 15:43:47,888 [MainThread] INFO UNet3DTrainer - Training iteration 338. Batch 0. Epoch [337/599]\n",
      "2019-07-31 15:43:49,157 [MainThread] INFO UNet3DTrainer - Training iteration 339. Batch 0. Epoch [338/599]\n",
      "2019-07-31 15:43:49,157 [MainThread] INFO UNet3DTrainer - Training iteration 339. Batch 0. Epoch [338/599]\n",
      "2019-07-31 15:43:50,391 [MainThread] INFO UNet3DTrainer - Training iteration 340. Batch 0. Epoch [339/599]\n",
      "2019-07-31 15:43:50,391 [MainThread] INFO UNet3DTrainer - Training iteration 340. Batch 0. Epoch [339/599]\n",
      "2019-07-31 15:43:51,206 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:43:51,206 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:43:52,533 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.321024090051651. Evaluation score: 0.5381582379341125\n",
      "2019-07-31 15:43:52,533 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.321024090051651. Evaluation score: 0.5381582379341125\n",
      "2019-07-31 15:43:52,539 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5381582379341125\n",
      "2019-07-31 15:43:52,539 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5381582379341125\n",
      "2019-07-31 15:43:52,542 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:43:52,542 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:43:52,807 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:43:52,807 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:43:53,041 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.014949211850762367. Evaluation score: 0.5381582379341125\n",
      "2019-07-31 15:43:53,041 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.014949211850762367. Evaluation score: 0.5381582379341125\n",
      "2019-07-31 15:43:53,043 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:43:53,043 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:43:54,483 [MainThread] INFO UNet3DTrainer - Training iteration 341. Batch 0. Epoch [340/599]\n",
      "2019-07-31 15:43:54,483 [MainThread] INFO UNet3DTrainer - Training iteration 341. Batch 0. Epoch [340/599]\n",
      "2019-07-31 15:43:55,217 [MainThread] INFO UNet3DTrainer - Training iteration 342. Batch 0. Epoch [341/599]\n",
      "2019-07-31 15:43:55,217 [MainThread] INFO UNet3DTrainer - Training iteration 342. Batch 0. Epoch [341/599]\n",
      "2019-07-31 15:43:56,423 [MainThread] INFO UNet3DTrainer - Training iteration 343. Batch 0. Epoch [342/599]\n",
      "2019-07-31 15:43:56,423 [MainThread] INFO UNet3DTrainer - Training iteration 343. Batch 0. Epoch [342/599]\n",
      "2019-07-31 15:43:57,663 [MainThread] INFO UNet3DTrainer - Training iteration 344. Batch 0. Epoch [343/599]\n",
      "2019-07-31 15:43:57,663 [MainThread] INFO UNet3DTrainer - Training iteration 344. Batch 0. Epoch [343/599]\n",
      "2019-07-31 15:43:58,928 [MainThread] INFO UNet3DTrainer - Training iteration 345. Batch 0. Epoch [344/599]\n",
      "2019-07-31 15:43:58,928 [MainThread] INFO UNet3DTrainer - Training iteration 345. Batch 0. Epoch [344/599]\n",
      "2019-07-31 15:44:00,175 [MainThread] INFO UNet3DTrainer - Training iteration 346. Batch 0. Epoch [345/599]\n",
      "2019-07-31 15:44:00,175 [MainThread] INFO UNet3DTrainer - Training iteration 346. Batch 0. Epoch [345/599]\n",
      "2019-07-31 15:44:01,432 [MainThread] INFO UNet3DTrainer - Training iteration 347. Batch 0. Epoch [346/599]\n",
      "2019-07-31 15:44:01,432 [MainThread] INFO UNet3DTrainer - Training iteration 347. Batch 0. Epoch [346/599]\n",
      "2019-07-31 15:44:02,666 [MainThread] INFO UNet3DTrainer - Training iteration 348. Batch 0. Epoch [347/599]\n",
      "2019-07-31 15:44:02,666 [MainThread] INFO UNet3DTrainer - Training iteration 348. Batch 0. Epoch [347/599]\n",
      "2019-07-31 15:44:03,935 [MainThread] INFO UNet3DTrainer - Training iteration 349. Batch 0. Epoch [348/599]\n",
      "2019-07-31 15:44:03,935 [MainThread] INFO UNet3DTrainer - Training iteration 349. Batch 0. Epoch [348/599]\n",
      "2019-07-31 15:44:05,193 [MainThread] INFO UNet3DTrainer - Training iteration 350. Batch 0. Epoch [349/599]\n",
      "2019-07-31 15:44:05,193 [MainThread] INFO UNet3DTrainer - Training iteration 350. Batch 0. Epoch [349/599]\n",
      "2019-07-31 15:44:06,469 [MainThread] INFO UNet3DTrainer - Training iteration 351. Batch 0. Epoch [350/599]\n",
      "2019-07-31 15:44:06,469 [MainThread] INFO UNet3DTrainer - Training iteration 351. Batch 0. Epoch [350/599]\n",
      "2019-07-31 15:44:07,677 [MainThread] INFO UNet3DTrainer - Training iteration 352. Batch 0. Epoch [351/599]\n",
      "2019-07-31 15:44:07,677 [MainThread] INFO UNet3DTrainer - Training iteration 352. Batch 0. Epoch [351/599]\n",
      "2019-07-31 15:44:08,972 [MainThread] INFO UNet3DTrainer - Training iteration 353. Batch 0. Epoch [352/599]\n",
      "2019-07-31 15:44:08,972 [MainThread] INFO UNet3DTrainer - Training iteration 353. Batch 0. Epoch [352/599]\n",
      "2019-07-31 15:44:10,238 [MainThread] INFO UNet3DTrainer - Training iteration 354. Batch 0. Epoch [353/599]\n",
      "2019-07-31 15:44:10,238 [MainThread] INFO UNet3DTrainer - Training iteration 354. Batch 0. Epoch [353/599]\n",
      "2019-07-31 15:44:11,475 [MainThread] INFO UNet3DTrainer - Training iteration 355. Batch 0. Epoch [354/599]\n",
      "2019-07-31 15:44:11,475 [MainThread] INFO UNet3DTrainer - Training iteration 355. Batch 0. Epoch [354/599]\n",
      "2019-07-31 15:44:12,741 [MainThread] INFO UNet3DTrainer - Training iteration 356. Batch 0. Epoch [355/599]\n",
      "2019-07-31 15:44:12,741 [MainThread] INFO UNet3DTrainer - Training iteration 356. Batch 0. Epoch [355/599]\n",
      "2019-07-31 15:44:13,997 [MainThread] INFO UNet3DTrainer - Training iteration 357. Batch 0. Epoch [356/599]\n",
      "2019-07-31 15:44:13,997 [MainThread] INFO UNet3DTrainer - Training iteration 357. Batch 0. Epoch [356/599]\n",
      "2019-07-31 15:44:15,254 [MainThread] INFO UNet3DTrainer - Training iteration 358. Batch 0. Epoch [357/599]\n",
      "2019-07-31 15:44:15,254 [MainThread] INFO UNet3DTrainer - Training iteration 358. Batch 0. Epoch [357/599]\n",
      "2019-07-31 15:44:16,449 [MainThread] INFO UNet3DTrainer - Training iteration 359. Batch 0. Epoch [358/599]\n",
      "2019-07-31 15:44:16,449 [MainThread] INFO UNet3DTrainer - Training iteration 359. Batch 0. Epoch [358/599]\n",
      "2019-07-31 15:44:17,714 [MainThread] INFO UNet3DTrainer - Training iteration 360. Batch 0. Epoch [359/599]\n",
      "2019-07-31 15:44:17,714 [MainThread] INFO UNet3DTrainer - Training iteration 360. Batch 0. Epoch [359/599]\n",
      "2019-07-31 15:44:18,521 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:44:18,521 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:44:19,843 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3208647668361664. Evaluation score: 0.5409723520278931\n",
      "2019-07-31 15:44:19,843 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3208647668361664. Evaluation score: 0.5409723520278931\n",
      "2019-07-31 15:44:19,849 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5409723520278931\n",
      "2019-07-31 15:44:19,849 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5409723520278931\n",
      "2019-07-31 15:44:19,851 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:44:19,851 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:44:20,121 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:44:20,121 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:44:20,542 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.0146526750177145. Evaluation score: 0.5409723520278931\n",
      "2019-07-31 15:44:20,542 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.0146526750177145. Evaluation score: 0.5409723520278931\n",
      "2019-07-31 15:44:20,544 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:44:20,544 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:44:22,031 [MainThread] INFO UNet3DTrainer - Training iteration 361. Batch 0. Epoch [360/599]\n",
      "2019-07-31 15:44:22,031 [MainThread] INFO UNet3DTrainer - Training iteration 361. Batch 0. Epoch [360/599]\n",
      "2019-07-31 15:44:22,770 [MainThread] INFO UNet3DTrainer - Training iteration 362. Batch 0. Epoch [361/599]\n",
      "2019-07-31 15:44:22,770 [MainThread] INFO UNet3DTrainer - Training iteration 362. Batch 0. Epoch [361/599]\n",
      "2019-07-31 15:44:23,982 [MainThread] INFO UNet3DTrainer - Training iteration 363. Batch 0. Epoch [362/599]\n",
      "2019-07-31 15:44:23,982 [MainThread] INFO UNet3DTrainer - Training iteration 363. Batch 0. Epoch [362/599]\n",
      "2019-07-31 15:44:25,233 [MainThread] INFO UNet3DTrainer - Training iteration 364. Batch 0. Epoch [363/599]\n",
      "2019-07-31 15:44:25,233 [MainThread] INFO UNet3DTrainer - Training iteration 364. Batch 0. Epoch [363/599]\n",
      "2019-07-31 15:44:26,474 [MainThread] INFO UNet3DTrainer - Training iteration 365. Batch 0. Epoch [364/599]\n",
      "2019-07-31 15:44:26,474 [MainThread] INFO UNet3DTrainer - Training iteration 365. Batch 0. Epoch [364/599]\n",
      "2019-07-31 15:44:27,729 [MainThread] INFO UNet3DTrainer - Training iteration 366. Batch 0. Epoch [365/599]\n",
      "2019-07-31 15:44:27,729 [MainThread] INFO UNet3DTrainer - Training iteration 366. Batch 0. Epoch [365/599]\n",
      "2019-07-31 15:44:28,976 [MainThread] INFO UNet3DTrainer - Training iteration 367. Batch 0. Epoch [366/599]\n",
      "2019-07-31 15:44:28,976 [MainThread] INFO UNet3DTrainer - Training iteration 367. Batch 0. Epoch [366/599]\n",
      "2019-07-31 15:44:30,227 [MainThread] INFO UNet3DTrainer - Training iteration 368. Batch 0. Epoch [367/599]\n",
      "2019-07-31 15:44:30,227 [MainThread] INFO UNet3DTrainer - Training iteration 368. Batch 0. Epoch [367/599]\n",
      "2019-07-31 15:44:31,476 [MainThread] INFO UNet3DTrainer - Training iteration 369. Batch 0. Epoch [368/599]\n",
      "2019-07-31 15:44:31,476 [MainThread] INFO UNet3DTrainer - Training iteration 369. Batch 0. Epoch [368/599]\n",
      "2019-07-31 15:44:32,715 [MainThread] INFO UNet3DTrainer - Training iteration 370. Batch 0. Epoch [369/599]\n",
      "2019-07-31 15:44:32,715 [MainThread] INFO UNet3DTrainer - Training iteration 370. Batch 0. Epoch [369/599]\n",
      "2019-07-31 15:44:34,045 [MainThread] INFO UNet3DTrainer - Training iteration 371. Batch 0. Epoch [370/599]\n",
      "2019-07-31 15:44:34,045 [MainThread] INFO UNet3DTrainer - Training iteration 371. Batch 0. Epoch [370/599]\n",
      "2019-07-31 15:44:35,266 [MainThread] INFO UNet3DTrainer - Training iteration 372. Batch 0. Epoch [371/599]\n",
      "2019-07-31 15:44:35,266 [MainThread] INFO UNet3DTrainer - Training iteration 372. Batch 0. Epoch [371/599]\n",
      "2019-07-31 15:44:36,495 [MainThread] INFO UNet3DTrainer - Training iteration 373. Batch 0. Epoch [372/599]\n",
      "2019-07-31 15:44:36,495 [MainThread] INFO UNet3DTrainer - Training iteration 373. Batch 0. Epoch [372/599]\n",
      "2019-07-31 15:44:37,752 [MainThread] INFO UNet3DTrainer - Training iteration 374. Batch 0. Epoch [373/599]\n",
      "2019-07-31 15:44:37,752 [MainThread] INFO UNet3DTrainer - Training iteration 374. Batch 0. Epoch [373/599]\n",
      "2019-07-31 15:44:38,999 [MainThread] INFO UNet3DTrainer - Training iteration 375. Batch 0. Epoch [374/599]\n",
      "2019-07-31 15:44:38,999 [MainThread] INFO UNet3DTrainer - Training iteration 375. Batch 0. Epoch [374/599]\n",
      "2019-07-31 15:44:40,289 [MainThread] INFO UNet3DTrainer - Training iteration 376. Batch 0. Epoch [375/599]\n",
      "2019-07-31 15:44:40,289 [MainThread] INFO UNet3DTrainer - Training iteration 376. Batch 0. Epoch [375/599]\n",
      "2019-07-31 15:44:41,534 [MainThread] INFO UNet3DTrainer - Training iteration 377. Batch 0. Epoch [376/599]\n",
      "2019-07-31 15:44:41,534 [MainThread] INFO UNet3DTrainer - Training iteration 377. Batch 0. Epoch [376/599]\n",
      "2019-07-31 15:44:42,751 [MainThread] INFO UNet3DTrainer - Training iteration 378. Batch 0. Epoch [377/599]\n",
      "2019-07-31 15:44:42,751 [MainThread] INFO UNet3DTrainer - Training iteration 378. Batch 0. Epoch [377/599]\n",
      "2019-07-31 15:44:44,028 [MainThread] INFO UNet3DTrainer - Training iteration 379. Batch 0. Epoch [378/599]\n",
      "2019-07-31 15:44:44,028 [MainThread] INFO UNet3DTrainer - Training iteration 379. Batch 0. Epoch [378/599]\n",
      "2019-07-31 15:44:45,263 [MainThread] INFO UNet3DTrainer - Training iteration 380. Batch 0. Epoch [379/599]\n",
      "2019-07-31 15:44:45,263 [MainThread] INFO UNet3DTrainer - Training iteration 380. Batch 0. Epoch [379/599]\n",
      "2019-07-31 15:44:46,073 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:44:46,073 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:44:47,401 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3207078278064728. Evaluation score: 0.5442005395889282\n",
      "2019-07-31 15:44:47,401 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3207078278064728. Evaluation score: 0.5442005395889282\n",
      "2019-07-31 15:44:47,406 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5442005395889282\n",
      "2019-07-31 15:44:47,406 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5442005395889282\n",
      "2019-07-31 15:44:47,408 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:44:47,408 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:44:47,638 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:44:47,638 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:44:47,945 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.014360700733959675. Evaluation score: 0.5442005395889282\n",
      "2019-07-31 15:44:47,945 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.014360700733959675. Evaluation score: 0.5442005395889282\n",
      "2019-07-31 15:44:47,947 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:44:47,947 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:44:49,443 [MainThread] INFO UNet3DTrainer - Training iteration 381. Batch 0. Epoch [380/599]\n",
      "2019-07-31 15:44:49,443 [MainThread] INFO UNet3DTrainer - Training iteration 381. Batch 0. Epoch [380/599]\n",
      "2019-07-31 15:44:50,165 [MainThread] INFO UNet3DTrainer - Training iteration 382. Batch 0. Epoch [381/599]\n",
      "2019-07-31 15:44:50,165 [MainThread] INFO UNet3DTrainer - Training iteration 382. Batch 0. Epoch [381/599]\n",
      "2019-07-31 15:44:51,387 [MainThread] INFO UNet3DTrainer - Training iteration 383. Batch 0. Epoch [382/599]\n",
      "2019-07-31 15:44:51,387 [MainThread] INFO UNet3DTrainer - Training iteration 383. Batch 0. Epoch [382/599]\n",
      "2019-07-31 15:44:52,627 [MainThread] INFO UNet3DTrainer - Training iteration 384. Batch 0. Epoch [383/599]\n",
      "2019-07-31 15:44:52,627 [MainThread] INFO UNet3DTrainer - Training iteration 384. Batch 0. Epoch [383/599]\n",
      "2019-07-31 15:44:53,894 [MainThread] INFO UNet3DTrainer - Training iteration 385. Batch 0. Epoch [384/599]\n",
      "2019-07-31 15:44:53,894 [MainThread] INFO UNet3DTrainer - Training iteration 385. Batch 0. Epoch [384/599]\n",
      "2019-07-31 15:44:55,170 [MainThread] INFO UNet3DTrainer - Training iteration 386. Batch 0. Epoch [385/599]\n",
      "2019-07-31 15:44:55,170 [MainThread] INFO UNet3DTrainer - Training iteration 386. Batch 0. Epoch [385/599]\n",
      "2019-07-31 15:44:56,390 [MainThread] INFO UNet3DTrainer - Training iteration 387. Batch 0. Epoch [386/599]\n",
      "2019-07-31 15:44:56,390 [MainThread] INFO UNet3DTrainer - Training iteration 387. Batch 0. Epoch [386/599]\n",
      "2019-07-31 15:44:57,637 [MainThread] INFO UNet3DTrainer - Training iteration 388. Batch 0. Epoch [387/599]\n",
      "2019-07-31 15:44:57,637 [MainThread] INFO UNet3DTrainer - Training iteration 388. Batch 0. Epoch [387/599]\n",
      "2019-07-31 15:44:58,892 [MainThread] INFO UNet3DTrainer - Training iteration 389. Batch 0. Epoch [388/599]\n",
      "2019-07-31 15:44:58,892 [MainThread] INFO UNet3DTrainer - Training iteration 389. Batch 0. Epoch [388/599]\n",
      "2019-07-31 15:45:00,139 [MainThread] INFO UNet3DTrainer - Training iteration 390. Batch 0. Epoch [389/599]\n",
      "2019-07-31 15:45:00,139 [MainThread] INFO UNet3DTrainer - Training iteration 390. Batch 0. Epoch [389/599]\n",
      "2019-07-31 15:45:01,397 [MainThread] INFO UNet3DTrainer - Training iteration 391. Batch 0. Epoch [390/599]\n",
      "2019-07-31 15:45:01,397 [MainThread] INFO UNet3DTrainer - Training iteration 391. Batch 0. Epoch [390/599]\n",
      "2019-07-31 15:45:02,689 [MainThread] INFO UNet3DTrainer - Training iteration 392. Batch 0. Epoch [391/599]\n",
      "2019-07-31 15:45:02,689 [MainThread] INFO UNet3DTrainer - Training iteration 392. Batch 0. Epoch [391/599]\n",
      "2019-07-31 15:45:03,913 [MainThread] INFO UNet3DTrainer - Training iteration 393. Batch 0. Epoch [392/599]\n",
      "2019-07-31 15:45:03,913 [MainThread] INFO UNet3DTrainer - Training iteration 393. Batch 0. Epoch [392/599]\n",
      "2019-07-31 15:45:05,146 [MainThread] INFO UNet3DTrainer - Training iteration 394. Batch 0. Epoch [393/599]\n",
      "2019-07-31 15:45:05,146 [MainThread] INFO UNet3DTrainer - Training iteration 394. Batch 0. Epoch [393/599]\n",
      "2019-07-31 15:45:06,394 [MainThread] INFO UNet3DTrainer - Training iteration 395. Batch 0. Epoch [394/599]\n",
      "2019-07-31 15:45:06,394 [MainThread] INFO UNet3DTrainer - Training iteration 395. Batch 0. Epoch [394/599]\n",
      "2019-07-31 15:45:07,635 [MainThread] INFO UNet3DTrainer - Training iteration 396. Batch 0. Epoch [395/599]\n",
      "2019-07-31 15:45:07,635 [MainThread] INFO UNet3DTrainer - Training iteration 396. Batch 0. Epoch [395/599]\n",
      "2019-07-31 15:45:08,924 [MainThread] INFO UNet3DTrainer - Training iteration 397. Batch 0. Epoch [396/599]\n",
      "2019-07-31 15:45:08,924 [MainThread] INFO UNet3DTrainer - Training iteration 397. Batch 0. Epoch [396/599]\n",
      "2019-07-31 15:45:10,202 [MainThread] INFO UNet3DTrainer - Training iteration 398. Batch 0. Epoch [397/599]\n",
      "2019-07-31 15:45:10,202 [MainThread] INFO UNet3DTrainer - Training iteration 398. Batch 0. Epoch [397/599]\n",
      "2019-07-31 15:45:11,423 [MainThread] INFO UNet3DTrainer - Training iteration 399. Batch 0. Epoch [398/599]\n",
      "2019-07-31 15:45:11,423 [MainThread] INFO UNet3DTrainer - Training iteration 399. Batch 0. Epoch [398/599]\n",
      "2019-07-31 15:45:12,671 [MainThread] INFO UNet3DTrainer - Training iteration 400. Batch 0. Epoch [399/599]\n",
      "2019-07-31 15:45:12,671 [MainThread] INFO UNet3DTrainer - Training iteration 400. Batch 0. Epoch [399/599]\n",
      "2019-07-31 15:45:13,486 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:45:13,486 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:45:14,806 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.32055386900901794. Evaluation score: 0.5462709069252014\n",
      "2019-07-31 15:45:14,806 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.32055386900901794. Evaluation score: 0.5462709069252014\n",
      "2019-07-31 15:45:14,812 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5462709069252014\n",
      "2019-07-31 15:45:14,812 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5462709069252014\n",
      "2019-07-31 15:45:14,813 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:45:14,813 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:45:15,082 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:45:15,082 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:45:15,309 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.014073530212044716. Evaluation score: 0.5459096431732178\n",
      "2019-07-31 15:45:15,309 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.014073530212044716. Evaluation score: 0.5459096431732178\n",
      "2019-07-31 15:45:15,311 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:45:15,311 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:45:16,779 [MainThread] INFO UNet3DTrainer - Training iteration 401. Batch 0. Epoch [400/599]\n",
      "2019-07-31 15:45:16,779 [MainThread] INFO UNet3DTrainer - Training iteration 401. Batch 0. Epoch [400/599]\n",
      "2019-07-31 15:45:17,478 [MainThread] INFO UNet3DTrainer - Training iteration 402. Batch 0. Epoch [401/599]\n",
      "2019-07-31 15:45:17,478 [MainThread] INFO UNet3DTrainer - Training iteration 402. Batch 0. Epoch [401/599]\n",
      "2019-07-31 15:45:18,728 [MainThread] INFO UNet3DTrainer - Training iteration 403. Batch 0. Epoch [402/599]\n",
      "2019-07-31 15:45:18,728 [MainThread] INFO UNet3DTrainer - Training iteration 403. Batch 0. Epoch [402/599]\n",
      "2019-07-31 15:45:19,979 [MainThread] INFO UNet3DTrainer - Training iteration 404. Batch 0. Epoch [403/599]\n",
      "2019-07-31 15:45:19,979 [MainThread] INFO UNet3DTrainer - Training iteration 404. Batch 0. Epoch [403/599]\n",
      "2019-07-31 15:45:21,230 [MainThread] INFO UNet3DTrainer - Training iteration 405. Batch 0. Epoch [404/599]\n",
      "2019-07-31 15:45:21,230 [MainThread] INFO UNet3DTrainer - Training iteration 405. Batch 0. Epoch [404/599]\n",
      "2019-07-31 15:45:22,484 [MainThread] INFO UNet3DTrainer - Training iteration 406. Batch 0. Epoch [405/599]\n",
      "2019-07-31 15:45:22,484 [MainThread] INFO UNet3DTrainer - Training iteration 406. Batch 0. Epoch [405/599]\n",
      "2019-07-31 15:45:23,732 [MainThread] INFO UNet3DTrainer - Training iteration 407. Batch 0. Epoch [406/599]\n",
      "2019-07-31 15:45:23,732 [MainThread] INFO UNet3DTrainer - Training iteration 407. Batch 0. Epoch [406/599]\n",
      "2019-07-31 15:45:24,973 [MainThread] INFO UNet3DTrainer - Training iteration 408. Batch 0. Epoch [407/599]\n",
      "2019-07-31 15:45:24,973 [MainThread] INFO UNet3DTrainer - Training iteration 408. Batch 0. Epoch [407/599]\n",
      "2019-07-31 15:45:26,234 [MainThread] INFO UNet3DTrainer - Training iteration 409. Batch 0. Epoch [408/599]\n",
      "2019-07-31 15:45:26,234 [MainThread] INFO UNet3DTrainer - Training iteration 409. Batch 0. Epoch [408/599]\n",
      "2019-07-31 15:45:27,479 [MainThread] INFO UNet3DTrainer - Training iteration 410. Batch 0. Epoch [409/599]\n",
      "2019-07-31 15:45:27,479 [MainThread] INFO UNet3DTrainer - Training iteration 410. Batch 0. Epoch [409/599]\n",
      "2019-07-31 15:45:28,722 [MainThread] INFO UNet3DTrainer - Training iteration 411. Batch 0. Epoch [410/599]\n",
      "2019-07-31 15:45:28,722 [MainThread] INFO UNet3DTrainer - Training iteration 411. Batch 0. Epoch [410/599]\n",
      "2019-07-31 15:45:29,981 [MainThread] INFO UNet3DTrainer - Training iteration 412. Batch 0. Epoch [411/599]\n",
      "2019-07-31 15:45:29,981 [MainThread] INFO UNet3DTrainer - Training iteration 412. Batch 0. Epoch [411/599]\n",
      "2019-07-31 15:45:31,235 [MainThread] INFO UNet3DTrainer - Training iteration 413. Batch 0. Epoch [412/599]\n",
      "2019-07-31 15:45:31,235 [MainThread] INFO UNet3DTrainer - Training iteration 413. Batch 0. Epoch [412/599]\n",
      "2019-07-31 15:45:32,510 [MainThread] INFO UNet3DTrainer - Training iteration 414. Batch 0. Epoch [413/599]\n",
      "2019-07-31 15:45:32,510 [MainThread] INFO UNet3DTrainer - Training iteration 414. Batch 0. Epoch [413/599]\n",
      "2019-07-31 15:45:33,746 [MainThread] INFO UNet3DTrainer - Training iteration 415. Batch 0. Epoch [414/599]\n",
      "2019-07-31 15:45:33,746 [MainThread] INFO UNet3DTrainer - Training iteration 415. Batch 0. Epoch [414/599]\n",
      "2019-07-31 15:45:35,031 [MainThread] INFO UNet3DTrainer - Training iteration 416. Batch 0. Epoch [415/599]\n",
      "2019-07-31 15:45:35,031 [MainThread] INFO UNet3DTrainer - Training iteration 416. Batch 0. Epoch [415/599]\n",
      "2019-07-31 15:45:36,254 [MainThread] INFO UNet3DTrainer - Training iteration 417. Batch 0. Epoch [416/599]\n",
      "2019-07-31 15:45:36,254 [MainThread] INFO UNet3DTrainer - Training iteration 417. Batch 0. Epoch [416/599]\n",
      "2019-07-31 15:45:37,539 [MainThread] INFO UNet3DTrainer - Training iteration 418. Batch 0. Epoch [417/599]\n",
      "2019-07-31 15:45:37,539 [MainThread] INFO UNet3DTrainer - Training iteration 418. Batch 0. Epoch [417/599]\n",
      "2019-07-31 15:45:38,777 [MainThread] INFO UNet3DTrainer - Training iteration 419. Batch 0. Epoch [418/599]\n",
      "2019-07-31 15:45:38,777 [MainThread] INFO UNet3DTrainer - Training iteration 419. Batch 0. Epoch [418/599]\n",
      "2019-07-31 15:45:40,052 [MainThread] INFO UNet3DTrainer - Training iteration 420. Batch 0. Epoch [419/599]\n",
      "2019-07-31 15:45:40,052 [MainThread] INFO UNet3DTrainer - Training iteration 420. Batch 0. Epoch [419/599]\n",
      "2019-07-31 15:45:40,818 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:45:40,818 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:45:42,134 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.32040226459503174. Evaluation score: 0.5490533113479614\n",
      "2019-07-31 15:45:42,134 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.32040226459503174. Evaluation score: 0.5490533113479614\n",
      "2019-07-31 15:45:42,140 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5490533113479614\n",
      "2019-07-31 15:45:42,140 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5490533113479614\n",
      "2019-07-31 15:45:42,142 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:45:42,142 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:45:42,408 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:45:42,408 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:45:42,631 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.013791413977742195. Evaluation score: 0.5490533113479614\n",
      "2019-07-31 15:45:42,631 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.013791413977742195. Evaluation score: 0.5490533113479614\n",
      "2019-07-31 15:45:42,633 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:45:42,633 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:45:44,115 [MainThread] INFO UNet3DTrainer - Training iteration 421. Batch 0. Epoch [420/599]\n",
      "2019-07-31 15:45:44,115 [MainThread] INFO UNet3DTrainer - Training iteration 421. Batch 0. Epoch [420/599]\n",
      "2019-07-31 15:45:44,863 [MainThread] INFO UNet3DTrainer - Training iteration 422. Batch 0. Epoch [421/599]\n",
      "2019-07-31 15:45:44,863 [MainThread] INFO UNet3DTrainer - Training iteration 422. Batch 0. Epoch [421/599]\n",
      "2019-07-31 15:45:46,071 [MainThread] INFO UNet3DTrainer - Training iteration 423. Batch 0. Epoch [422/599]\n",
      "2019-07-31 15:45:46,071 [MainThread] INFO UNet3DTrainer - Training iteration 423. Batch 0. Epoch [422/599]\n",
      "2019-07-31 15:45:47,326 [MainThread] INFO UNet3DTrainer - Training iteration 424. Batch 0. Epoch [423/599]\n",
      "2019-07-31 15:45:47,326 [MainThread] INFO UNet3DTrainer - Training iteration 424. Batch 0. Epoch [423/599]\n",
      "2019-07-31 15:45:48,572 [MainThread] INFO UNet3DTrainer - Training iteration 425. Batch 0. Epoch [424/599]\n",
      "2019-07-31 15:45:48,572 [MainThread] INFO UNet3DTrainer - Training iteration 425. Batch 0. Epoch [424/599]\n",
      "2019-07-31 15:45:49,822 [MainThread] INFO UNet3DTrainer - Training iteration 426. Batch 0. Epoch [425/599]\n",
      "2019-07-31 15:45:49,822 [MainThread] INFO UNet3DTrainer - Training iteration 426. Batch 0. Epoch [425/599]\n",
      "2019-07-31 15:45:51,064 [MainThread] INFO UNet3DTrainer - Training iteration 427. Batch 0. Epoch [426/599]\n",
      "2019-07-31 15:45:51,064 [MainThread] INFO UNet3DTrainer - Training iteration 427. Batch 0. Epoch [426/599]\n",
      "2019-07-31 15:45:52,324 [MainThread] INFO UNet3DTrainer - Training iteration 428. Batch 0. Epoch [427/599]\n",
      "2019-07-31 15:45:52,324 [MainThread] INFO UNet3DTrainer - Training iteration 428. Batch 0. Epoch [427/599]\n",
      "2019-07-31 15:45:53,556 [MainThread] INFO UNet3DTrainer - Training iteration 429. Batch 0. Epoch [428/599]\n",
      "2019-07-31 15:45:53,556 [MainThread] INFO UNet3DTrainer - Training iteration 429. Batch 0. Epoch [428/599]\n",
      "2019-07-31 15:45:54,811 [MainThread] INFO UNet3DTrainer - Training iteration 430. Batch 0. Epoch [429/599]\n",
      "2019-07-31 15:45:54,811 [MainThread] INFO UNet3DTrainer - Training iteration 430. Batch 0. Epoch [429/599]\n",
      "2019-07-31 15:45:56,073 [MainThread] INFO UNet3DTrainer - Training iteration 431. Batch 0. Epoch [430/599]\n",
      "2019-07-31 15:45:56,073 [MainThread] INFO UNet3DTrainer - Training iteration 431. Batch 0. Epoch [430/599]\n",
      "2019-07-31 15:45:57,372 [MainThread] INFO UNet3DTrainer - Training iteration 432. Batch 0. Epoch [431/599]\n",
      "2019-07-31 15:45:57,372 [MainThread] INFO UNet3DTrainer - Training iteration 432. Batch 0. Epoch [431/599]\n",
      "2019-07-31 15:45:58,573 [MainThread] INFO UNet3DTrainer - Training iteration 433. Batch 0. Epoch [432/599]\n",
      "2019-07-31 15:45:58,573 [MainThread] INFO UNet3DTrainer - Training iteration 433. Batch 0. Epoch [432/599]\n",
      "2019-07-31 15:45:59,844 [MainThread] INFO UNet3DTrainer - Training iteration 434. Batch 0. Epoch [433/599]\n",
      "2019-07-31 15:45:59,844 [MainThread] INFO UNet3DTrainer - Training iteration 434. Batch 0. Epoch [433/599]\n",
      "2019-07-31 15:46:01,110 [MainThread] INFO UNet3DTrainer - Training iteration 435. Batch 0. Epoch [434/599]\n",
      "2019-07-31 15:46:01,110 [MainThread] INFO UNet3DTrainer - Training iteration 435. Batch 0. Epoch [434/599]\n",
      "2019-07-31 15:46:02,359 [MainThread] INFO UNet3DTrainer - Training iteration 436. Batch 0. Epoch [435/599]\n",
      "2019-07-31 15:46:02,359 [MainThread] INFO UNet3DTrainer - Training iteration 436. Batch 0. Epoch [435/599]\n",
      "2019-07-31 15:46:03,627 [MainThread] INFO UNet3DTrainer - Training iteration 437. Batch 0. Epoch [436/599]\n",
      "2019-07-31 15:46:03,627 [MainThread] INFO UNet3DTrainer - Training iteration 437. Batch 0. Epoch [436/599]\n",
      "2019-07-31 15:46:04,840 [MainThread] INFO UNet3DTrainer - Training iteration 438. Batch 0. Epoch [437/599]\n",
      "2019-07-31 15:46:04,840 [MainThread] INFO UNet3DTrainer - Training iteration 438. Batch 0. Epoch [437/599]\n",
      "2019-07-31 15:46:06,132 [MainThread] INFO UNet3DTrainer - Training iteration 439. Batch 0. Epoch [438/599]\n",
      "2019-07-31 15:46:06,132 [MainThread] INFO UNet3DTrainer - Training iteration 439. Batch 0. Epoch [438/599]\n",
      "2019-07-31 15:46:07,337 [MainThread] INFO UNet3DTrainer - Training iteration 440. Batch 0. Epoch [439/599]\n",
      "2019-07-31 15:46:07,337 [MainThread] INFO UNet3DTrainer - Training iteration 440. Batch 0. Epoch [439/599]\n",
      "2019-07-31 15:46:08,155 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:46:08,155 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:46:09,471 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.32025402784347534. Evaluation score: 0.551576554775238\n",
      "2019-07-31 15:46:09,471 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.32025402784347534. Evaluation score: 0.551576554775238\n",
      "2019-07-31 15:46:09,476 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.551576554775238\n",
      "2019-07-31 15:46:09,476 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.551576554775238\n",
      "2019-07-31 15:46:09,477 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:46:09,477 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:46:09,743 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:46:09,743 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:46:09,968 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.013514463789761066. Evaluation score: 0.5512161254882812\n",
      "2019-07-31 15:46:09,968 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.013514463789761066. Evaluation score: 0.5512161254882812\n",
      "2019-07-31 15:46:09,969 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:46:09,969 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:46:11,747 [MainThread] INFO UNet3DTrainer - Training iteration 441. Batch 0. Epoch [440/599]\n",
      "2019-07-31 15:46:11,747 [MainThread] INFO UNet3DTrainer - Training iteration 441. Batch 0. Epoch [440/599]\n",
      "2019-07-31 15:46:12,448 [MainThread] INFO UNet3DTrainer - Training iteration 442. Batch 0. Epoch [441/599]\n",
      "2019-07-31 15:46:12,448 [MainThread] INFO UNet3DTrainer - Training iteration 442. Batch 0. Epoch [441/599]\n",
      "2019-07-31 15:46:13,685 [MainThread] INFO UNet3DTrainer - Training iteration 443. Batch 0. Epoch [442/599]\n",
      "2019-07-31 15:46:13,685 [MainThread] INFO UNet3DTrainer - Training iteration 443. Batch 0. Epoch [442/599]\n",
      "2019-07-31 15:46:14,940 [MainThread] INFO UNet3DTrainer - Training iteration 444. Batch 0. Epoch [443/599]\n",
      "2019-07-31 15:46:14,940 [MainThread] INFO UNet3DTrainer - Training iteration 444. Batch 0. Epoch [443/599]\n",
      "2019-07-31 15:46:16,198 [MainThread] INFO UNet3DTrainer - Training iteration 445. Batch 0. Epoch [444/599]\n",
      "2019-07-31 15:46:16,198 [MainThread] INFO UNet3DTrainer - Training iteration 445. Batch 0. Epoch [444/599]\n",
      "2019-07-31 15:46:17,442 [MainThread] INFO UNet3DTrainer - Training iteration 446. Batch 0. Epoch [445/599]\n",
      "2019-07-31 15:46:17,442 [MainThread] INFO UNet3DTrainer - Training iteration 446. Batch 0. Epoch [445/599]\n",
      "2019-07-31 15:46:18,688 [MainThread] INFO UNet3DTrainer - Training iteration 447. Batch 0. Epoch [446/599]\n",
      "2019-07-31 15:46:18,688 [MainThread] INFO UNet3DTrainer - Training iteration 447. Batch 0. Epoch [446/599]\n",
      "2019-07-31 15:46:19,950 [MainThread] INFO UNet3DTrainer - Training iteration 448. Batch 0. Epoch [447/599]\n",
      "2019-07-31 15:46:19,950 [MainThread] INFO UNet3DTrainer - Training iteration 448. Batch 0. Epoch [447/599]\n",
      "2019-07-31 15:46:21,210 [MainThread] INFO UNet3DTrainer - Training iteration 449. Batch 0. Epoch [448/599]\n",
      "2019-07-31 15:46:21,210 [MainThread] INFO UNet3DTrainer - Training iteration 449. Batch 0. Epoch [448/599]\n",
      "2019-07-31 15:46:22,473 [MainThread] INFO UNet3DTrainer - Training iteration 450. Batch 0. Epoch [449/599]\n",
      "2019-07-31 15:46:22,473 [MainThread] INFO UNet3DTrainer - Training iteration 450. Batch 0. Epoch [449/599]\n",
      "2019-07-31 15:46:23,699 [MainThread] INFO UNet3DTrainer - Training iteration 451. Batch 0. Epoch [450/599]\n",
      "2019-07-31 15:46:23,699 [MainThread] INFO UNet3DTrainer - Training iteration 451. Batch 0. Epoch [450/599]\n",
      "2019-07-31 15:46:24,962 [MainThread] INFO UNet3DTrainer - Training iteration 452. Batch 0. Epoch [451/599]\n",
      "2019-07-31 15:46:24,962 [MainThread] INFO UNet3DTrainer - Training iteration 452. Batch 0. Epoch [451/599]\n",
      "2019-07-31 15:46:26,242 [MainThread] INFO UNet3DTrainer - Training iteration 453. Batch 0. Epoch [452/599]\n",
      "2019-07-31 15:46:26,242 [MainThread] INFO UNet3DTrainer - Training iteration 453. Batch 0. Epoch [452/599]\n",
      "2019-07-31 15:46:27,482 [MainThread] INFO UNet3DTrainer - Training iteration 454. Batch 0. Epoch [453/599]\n",
      "2019-07-31 15:46:27,482 [MainThread] INFO UNet3DTrainer - Training iteration 454. Batch 0. Epoch [453/599]\n",
      "2019-07-31 15:46:28,763 [MainThread] INFO UNet3DTrainer - Training iteration 455. Batch 0. Epoch [454/599]\n",
      "2019-07-31 15:46:28,763 [MainThread] INFO UNet3DTrainer - Training iteration 455. Batch 0. Epoch [454/599]\n",
      "2019-07-31 15:46:30,028 [MainThread] INFO UNet3DTrainer - Training iteration 456. Batch 0. Epoch [455/599]\n",
      "2019-07-31 15:46:30,028 [MainThread] INFO UNet3DTrainer - Training iteration 456. Batch 0. Epoch [455/599]\n",
      "2019-07-31 15:46:31,245 [MainThread] INFO UNet3DTrainer - Training iteration 457. Batch 0. Epoch [456/599]\n",
      "2019-07-31 15:46:31,245 [MainThread] INFO UNet3DTrainer - Training iteration 457. Batch 0. Epoch [456/599]\n",
      "2019-07-31 15:46:32,533 [MainThread] INFO UNet3DTrainer - Training iteration 458. Batch 0. Epoch [457/599]\n",
      "2019-07-31 15:46:32,533 [MainThread] INFO UNet3DTrainer - Training iteration 458. Batch 0. Epoch [457/599]\n",
      "2019-07-31 15:46:33,749 [MainThread] INFO UNet3DTrainer - Training iteration 459. Batch 0. Epoch [458/599]\n",
      "2019-07-31 15:46:33,749 [MainThread] INFO UNet3DTrainer - Training iteration 459. Batch 0. Epoch [458/599]\n",
      "2019-07-31 15:46:35,001 [MainThread] INFO UNet3DTrainer - Training iteration 460. Batch 0. Epoch [459/599]\n",
      "2019-07-31 15:46:35,001 [MainThread] INFO UNet3DTrainer - Training iteration 460. Batch 0. Epoch [459/599]\n",
      "2019-07-31 15:46:35,793 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:46:35,793 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:46:37,120 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3201088011264801. Evaluation score: 0.55326247215271\n",
      "2019-07-31 15:46:37,120 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3201088011264801. Evaluation score: 0.55326247215271\n",
      "2019-07-31 15:46:37,127 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.55326247215271\n",
      "2019-07-31 15:46:37,127 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.55326247215271\n",
      "2019-07-31 15:46:37,130 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:46:37,130 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:46:37,402 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:46:37,402 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:46:37,811 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.013242860324680805. Evaluation score: 0.55326247215271\n",
      "2019-07-31 15:46:37,811 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.013242860324680805. Evaluation score: 0.55326247215271\n",
      "2019-07-31 15:46:37,813 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:46:37,813 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:46:39,382 [MainThread] INFO UNet3DTrainer - Training iteration 461. Batch 0. Epoch [460/599]\n",
      "2019-07-31 15:46:39,382 [MainThread] INFO UNet3DTrainer - Training iteration 461. Batch 0. Epoch [460/599]\n",
      "2019-07-31 15:46:40,125 [MainThread] INFO UNet3DTrainer - Training iteration 462. Batch 0. Epoch [461/599]\n",
      "2019-07-31 15:46:40,125 [MainThread] INFO UNet3DTrainer - Training iteration 462. Batch 0. Epoch [461/599]\n",
      "2019-07-31 15:46:41,335 [MainThread] INFO UNet3DTrainer - Training iteration 463. Batch 0. Epoch [462/599]\n",
      "2019-07-31 15:46:41,335 [MainThread] INFO UNet3DTrainer - Training iteration 463. Batch 0. Epoch [462/599]\n",
      "2019-07-31 15:46:42,582 [MainThread] INFO UNet3DTrainer - Training iteration 464. Batch 0. Epoch [463/599]\n",
      "2019-07-31 15:46:42,582 [MainThread] INFO UNet3DTrainer - Training iteration 464. Batch 0. Epoch [463/599]\n",
      "2019-07-31 15:46:43,829 [MainThread] INFO UNet3DTrainer - Training iteration 465. Batch 0. Epoch [464/599]\n",
      "2019-07-31 15:46:43,829 [MainThread] INFO UNet3DTrainer - Training iteration 465. Batch 0. Epoch [464/599]\n",
      "2019-07-31 15:46:45,084 [MainThread] INFO UNet3DTrainer - Training iteration 466. Batch 0. Epoch [465/599]\n",
      "2019-07-31 15:46:45,084 [MainThread] INFO UNet3DTrainer - Training iteration 466. Batch 0. Epoch [465/599]\n",
      "2019-07-31 15:46:46,345 [MainThread] INFO UNet3DTrainer - Training iteration 467. Batch 0. Epoch [466/599]\n",
      "2019-07-31 15:46:46,345 [MainThread] INFO UNet3DTrainer - Training iteration 467. Batch 0. Epoch [466/599]\n",
      "2019-07-31 15:46:47,571 [MainThread] INFO UNet3DTrainer - Training iteration 468. Batch 0. Epoch [467/599]\n",
      "2019-07-31 15:46:47,571 [MainThread] INFO UNet3DTrainer - Training iteration 468. Batch 0. Epoch [467/599]\n",
      "2019-07-31 15:46:48,818 [MainThread] INFO UNet3DTrainer - Training iteration 469. Batch 0. Epoch [468/599]\n",
      "2019-07-31 15:46:48,818 [MainThread] INFO UNet3DTrainer - Training iteration 469. Batch 0. Epoch [468/599]\n",
      "2019-07-31 15:46:50,076 [MainThread] INFO UNet3DTrainer - Training iteration 470. Batch 0. Epoch [469/599]\n",
      "2019-07-31 15:46:50,076 [MainThread] INFO UNet3DTrainer - Training iteration 470. Batch 0. Epoch [469/599]\n",
      "2019-07-31 15:46:51,382 [MainThread] INFO UNet3DTrainer - Training iteration 471. Batch 0. Epoch [470/599]\n",
      "2019-07-31 15:46:51,382 [MainThread] INFO UNet3DTrainer - Training iteration 471. Batch 0. Epoch [470/599]\n",
      "2019-07-31 15:46:52,587 [MainThread] INFO UNet3DTrainer - Training iteration 472. Batch 0. Epoch [471/599]\n",
      "2019-07-31 15:46:52,587 [MainThread] INFO UNet3DTrainer - Training iteration 472. Batch 0. Epoch [471/599]\n",
      "2019-07-31 15:46:53,832 [MainThread] INFO UNet3DTrainer - Training iteration 473. Batch 0. Epoch [472/599]\n",
      "2019-07-31 15:46:53,832 [MainThread] INFO UNet3DTrainer - Training iteration 473. Batch 0. Epoch [472/599]\n",
      "2019-07-31 15:46:55,089 [MainThread] INFO UNet3DTrainer - Training iteration 474. Batch 0. Epoch [473/599]\n",
      "2019-07-31 15:46:55,089 [MainThread] INFO UNet3DTrainer - Training iteration 474. Batch 0. Epoch [473/599]\n",
      "2019-07-31 15:46:56,369 [MainThread] INFO UNet3DTrainer - Training iteration 475. Batch 0. Epoch [474/599]\n",
      "2019-07-31 15:46:56,369 [MainThread] INFO UNet3DTrainer - Training iteration 475. Batch 0. Epoch [474/599]\n",
      "2019-07-31 15:46:57,614 [MainThread] INFO UNet3DTrainer - Training iteration 476. Batch 0. Epoch [475/599]\n",
      "2019-07-31 15:46:57,614 [MainThread] INFO UNet3DTrainer - Training iteration 476. Batch 0. Epoch [475/599]\n",
      "2019-07-31 15:46:58,849 [MainThread] INFO UNet3DTrainer - Training iteration 477. Batch 0. Epoch [476/599]\n",
      "2019-07-31 15:46:58,849 [MainThread] INFO UNet3DTrainer - Training iteration 477. Batch 0. Epoch [476/599]\n",
      "2019-07-31 15:47:00,136 [MainThread] INFO UNet3DTrainer - Training iteration 478. Batch 0. Epoch [477/599]\n",
      "2019-07-31 15:47:00,136 [MainThread] INFO UNet3DTrainer - Training iteration 478. Batch 0. Epoch [477/599]\n",
      "2019-07-31 15:47:01,379 [MainThread] INFO UNet3DTrainer - Training iteration 479. Batch 0. Epoch [478/599]\n",
      "2019-07-31 15:47:01,379 [MainThread] INFO UNet3DTrainer - Training iteration 479. Batch 0. Epoch [478/599]\n",
      "2019-07-31 15:47:02,655 [MainThread] INFO UNet3DTrainer - Training iteration 480. Batch 0. Epoch [479/599]\n",
      "2019-07-31 15:47:02,655 [MainThread] INFO UNet3DTrainer - Training iteration 480. Batch 0. Epoch [479/599]\n",
      "2019-07-31 15:47:03,423 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:47:03,423 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:47:04,747 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.31996622681617737. Evaluation score: 0.5567768216133118\n",
      "2019-07-31 15:47:04,747 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.31996622681617737. Evaluation score: 0.5567768216133118\n",
      "2019-07-31 15:47:04,753 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5567768216133118\n",
      "2019-07-31 15:47:04,753 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5567768216133118\n",
      "2019-07-31 15:47:04,755 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:47:04,755 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:47:05,018 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:47:05,018 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:47:05,253 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.01297671813517809. Evaluation score: 0.5564176440238953\n",
      "2019-07-31 15:47:05,253 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.01297671813517809. Evaluation score: 0.5564176440238953\n",
      "2019-07-31 15:47:05,255 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:47:05,255 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:47:06,785 [MainThread] INFO UNet3DTrainer - Training iteration 481. Batch 0. Epoch [480/599]\n",
      "2019-07-31 15:47:06,785 [MainThread] INFO UNet3DTrainer - Training iteration 481. Batch 0. Epoch [480/599]\n",
      "2019-07-31 15:47:07,492 [MainThread] INFO UNet3DTrainer - Training iteration 482. Batch 0. Epoch [481/599]\n",
      "2019-07-31 15:47:07,492 [MainThread] INFO UNet3DTrainer - Training iteration 482. Batch 0. Epoch [481/599]\n",
      "2019-07-31 15:47:08,722 [MainThread] INFO UNet3DTrainer - Training iteration 483. Batch 0. Epoch [482/599]\n",
      "2019-07-31 15:47:08,722 [MainThread] INFO UNet3DTrainer - Training iteration 483. Batch 0. Epoch [482/599]\n",
      "2019-07-31 15:47:09,990 [MainThread] INFO UNet3DTrainer - Training iteration 484. Batch 0. Epoch [483/599]\n",
      "2019-07-31 15:47:09,990 [MainThread] INFO UNet3DTrainer - Training iteration 484. Batch 0. Epoch [483/599]\n",
      "2019-07-31 15:47:11,240 [MainThread] INFO UNet3DTrainer - Training iteration 485. Batch 0. Epoch [484/599]\n",
      "2019-07-31 15:47:11,240 [MainThread] INFO UNet3DTrainer - Training iteration 485. Batch 0. Epoch [484/599]\n",
      "2019-07-31 15:47:12,472 [MainThread] INFO UNet3DTrainer - Training iteration 486. Batch 0. Epoch [485/599]\n",
      "2019-07-31 15:47:12,472 [MainThread] INFO UNet3DTrainer - Training iteration 486. Batch 0. Epoch [485/599]\n",
      "2019-07-31 15:47:13,741 [MainThread] INFO UNet3DTrainer - Training iteration 487. Batch 0. Epoch [486/599]\n",
      "2019-07-31 15:47:13,741 [MainThread] INFO UNet3DTrainer - Training iteration 487. Batch 0. Epoch [486/599]\n",
      "2019-07-31 15:47:14,976 [MainThread] INFO UNet3DTrainer - Training iteration 488. Batch 0. Epoch [487/599]\n",
      "2019-07-31 15:47:14,976 [MainThread] INFO UNet3DTrainer - Training iteration 488. Batch 0. Epoch [487/599]\n",
      "2019-07-31 15:47:16,231 [MainThread] INFO UNet3DTrainer - Training iteration 489. Batch 0. Epoch [488/599]\n",
      "2019-07-31 15:47:16,231 [MainThread] INFO UNet3DTrainer - Training iteration 489. Batch 0. Epoch [488/599]\n",
      "2019-07-31 15:47:17,487 [MainThread] INFO UNet3DTrainer - Training iteration 490. Batch 0. Epoch [489/599]\n",
      "2019-07-31 15:47:17,487 [MainThread] INFO UNet3DTrainer - Training iteration 490. Batch 0. Epoch [489/599]\n",
      "2019-07-31 15:47:18,735 [MainThread] INFO UNet3DTrainer - Training iteration 491. Batch 0. Epoch [490/599]\n",
      "2019-07-31 15:47:18,735 [MainThread] INFO UNet3DTrainer - Training iteration 491. Batch 0. Epoch [490/599]\n",
      "2019-07-31 15:47:20,001 [MainThread] INFO UNet3DTrainer - Training iteration 492. Batch 0. Epoch [491/599]\n",
      "2019-07-31 15:47:20,001 [MainThread] INFO UNet3DTrainer - Training iteration 492. Batch 0. Epoch [491/599]\n",
      "2019-07-31 15:47:21,235 [MainThread] INFO UNet3DTrainer - Training iteration 493. Batch 0. Epoch [492/599]\n",
      "2019-07-31 15:47:21,235 [MainThread] INFO UNet3DTrainer - Training iteration 493. Batch 0. Epoch [492/599]\n",
      "2019-07-31 15:47:22,489 [MainThread] INFO UNet3DTrainer - Training iteration 494. Batch 0. Epoch [493/599]\n",
      "2019-07-31 15:47:22,489 [MainThread] INFO UNet3DTrainer - Training iteration 494. Batch 0. Epoch [493/599]\n",
      "2019-07-31 15:47:23,755 [MainThread] INFO UNet3DTrainer - Training iteration 495. Batch 0. Epoch [494/599]\n",
      "2019-07-31 15:47:23,755 [MainThread] INFO UNet3DTrainer - Training iteration 495. Batch 0. Epoch [494/599]\n",
      "2019-07-31 15:47:25,016 [MainThread] INFO UNet3DTrainer - Training iteration 496. Batch 0. Epoch [495/599]\n",
      "2019-07-31 15:47:25,016 [MainThread] INFO UNet3DTrainer - Training iteration 496. Batch 0. Epoch [495/599]\n",
      "2019-07-31 15:47:26,283 [MainThread] INFO UNet3DTrainer - Training iteration 497. Batch 0. Epoch [496/599]\n",
      "2019-07-31 15:47:26,283 [MainThread] INFO UNet3DTrainer - Training iteration 497. Batch 0. Epoch [496/599]\n",
      "2019-07-31 15:47:27,532 [MainThread] INFO UNet3DTrainer - Training iteration 498. Batch 0. Epoch [497/599]\n",
      "2019-07-31 15:47:27,532 [MainThread] INFO UNet3DTrainer - Training iteration 498. Batch 0. Epoch [497/599]\n",
      "2019-07-31 15:47:28,754 [MainThread] INFO UNet3DTrainer - Training iteration 499. Batch 0. Epoch [498/599]\n",
      "2019-07-31 15:47:28,754 [MainThread] INFO UNet3DTrainer - Training iteration 499. Batch 0. Epoch [498/599]\n",
      "2019-07-31 15:47:30,021 [MainThread] INFO UNet3DTrainer - Training iteration 500. Batch 0. Epoch [499/599]\n",
      "2019-07-31 15:47:30,021 [MainThread] INFO UNet3DTrainer - Training iteration 500. Batch 0. Epoch [499/599]\n",
      "2019-07-31 15:47:30,827 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:47:30,827 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:47:32,133 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3198256194591522. Evaluation score: 0.5577704906463623\n",
      "2019-07-31 15:47:32,133 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3198256194591522. Evaluation score: 0.5577704906463623\n",
      "2019-07-31 15:47:32,139 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5577704906463623\n",
      "2019-07-31 15:47:32,139 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5577704906463623\n",
      "2019-07-31 15:47:32,142 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:47:32,142 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:47:32,412 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:47:32,412 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:47:32,635 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.012715126387774944. Evaluation score: 0.5577704906463623\n",
      "2019-07-31 15:47:32,635 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.012715126387774944. Evaluation score: 0.5577704906463623\n",
      "2019-07-31 15:47:32,637 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:47:32,637 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:47:34,048 [MainThread] INFO UNet3DTrainer - Training iteration 501. Batch 0. Epoch [500/599]\n",
      "2019-07-31 15:47:34,048 [MainThread] INFO UNet3DTrainer - Training iteration 501. Batch 0. Epoch [500/599]\n",
      "2019-07-31 15:47:34,793 [MainThread] INFO UNet3DTrainer - Training iteration 502. Batch 0. Epoch [501/599]\n",
      "2019-07-31 15:47:34,793 [MainThread] INFO UNet3DTrainer - Training iteration 502. Batch 0. Epoch [501/599]\n",
      "2019-07-31 15:47:36,003 [MainThread] INFO UNet3DTrainer - Training iteration 503. Batch 0. Epoch [502/599]\n",
      "2019-07-31 15:47:36,003 [MainThread] INFO UNet3DTrainer - Training iteration 503. Batch 0. Epoch [502/599]\n",
      "2019-07-31 15:47:37,251 [MainThread] INFO UNet3DTrainer - Training iteration 504. Batch 0. Epoch [503/599]\n",
      "2019-07-31 15:47:37,251 [MainThread] INFO UNet3DTrainer - Training iteration 504. Batch 0. Epoch [503/599]\n",
      "2019-07-31 15:47:38,500 [MainThread] INFO UNet3DTrainer - Training iteration 505. Batch 0. Epoch [504/599]\n",
      "2019-07-31 15:47:38,500 [MainThread] INFO UNet3DTrainer - Training iteration 505. Batch 0. Epoch [504/599]\n",
      "2019-07-31 15:47:39,735 [MainThread] INFO UNet3DTrainer - Training iteration 506. Batch 0. Epoch [505/599]\n",
      "2019-07-31 15:47:39,735 [MainThread] INFO UNet3DTrainer - Training iteration 506. Batch 0. Epoch [505/599]\n",
      "2019-07-31 15:47:40,998 [MainThread] INFO UNet3DTrainer - Training iteration 507. Batch 0. Epoch [506/599]\n",
      "2019-07-31 15:47:40,998 [MainThread] INFO UNet3DTrainer - Training iteration 507. Batch 0. Epoch [506/599]\n",
      "2019-07-31 15:47:42,251 [MainThread] INFO UNet3DTrainer - Training iteration 508. Batch 0. Epoch [507/599]\n",
      "2019-07-31 15:47:42,251 [MainThread] INFO UNet3DTrainer - Training iteration 508. Batch 0. Epoch [507/599]\n",
      "2019-07-31 15:47:43,500 [MainThread] INFO UNet3DTrainer - Training iteration 509. Batch 0. Epoch [508/599]\n",
      "2019-07-31 15:47:43,500 [MainThread] INFO UNet3DTrainer - Training iteration 509. Batch 0. Epoch [508/599]\n",
      "2019-07-31 15:47:44,750 [MainThread] INFO UNet3DTrainer - Training iteration 510. Batch 0. Epoch [509/599]\n",
      "2019-07-31 15:47:44,750 [MainThread] INFO UNet3DTrainer - Training iteration 510. Batch 0. Epoch [509/599]\n",
      "2019-07-31 15:47:46,002 [MainThread] INFO UNet3DTrainer - Training iteration 511. Batch 0. Epoch [510/599]\n",
      "2019-07-31 15:47:46,002 [MainThread] INFO UNet3DTrainer - Training iteration 511. Batch 0. Epoch [510/599]\n",
      "2019-07-31 15:47:47,252 [MainThread] INFO UNet3DTrainer - Training iteration 512. Batch 0. Epoch [511/599]\n",
      "2019-07-31 15:47:47,252 [MainThread] INFO UNet3DTrainer - Training iteration 512. Batch 0. Epoch [511/599]\n",
      "2019-07-31 15:47:48,517 [MainThread] INFO UNet3DTrainer - Training iteration 513. Batch 0. Epoch [512/599]\n",
      "2019-07-31 15:47:48,517 [MainThread] INFO UNet3DTrainer - Training iteration 513. Batch 0. Epoch [512/599]\n",
      "2019-07-31 15:47:49,805 [MainThread] INFO UNet3DTrainer - Training iteration 514. Batch 0. Epoch [513/599]\n",
      "2019-07-31 15:47:49,805 [MainThread] INFO UNet3DTrainer - Training iteration 514. Batch 0. Epoch [513/599]\n",
      "2019-07-31 15:47:51,055 [MainThread] INFO UNet3DTrainer - Training iteration 515. Batch 0. Epoch [514/599]\n",
      "2019-07-31 15:47:51,055 [MainThread] INFO UNet3DTrainer - Training iteration 515. Batch 0. Epoch [514/599]\n",
      "2019-07-31 15:47:52,287 [MainThread] INFO UNet3DTrainer - Training iteration 516. Batch 0. Epoch [515/599]\n",
      "2019-07-31 15:47:52,287 [MainThread] INFO UNet3DTrainer - Training iteration 516. Batch 0. Epoch [515/599]\n",
      "2019-07-31 15:47:53,547 [MainThread] INFO UNet3DTrainer - Training iteration 517. Batch 0. Epoch [516/599]\n",
      "2019-07-31 15:47:53,547 [MainThread] INFO UNet3DTrainer - Training iteration 517. Batch 0. Epoch [516/599]\n",
      "2019-07-31 15:47:54,797 [MainThread] INFO UNet3DTrainer - Training iteration 518. Batch 0. Epoch [517/599]\n",
      "2019-07-31 15:47:54,797 [MainThread] INFO UNet3DTrainer - Training iteration 518. Batch 0. Epoch [517/599]\n",
      "2019-07-31 15:47:56,034 [MainThread] INFO UNet3DTrainer - Training iteration 519. Batch 0. Epoch [518/599]\n",
      "2019-07-31 15:47:56,034 [MainThread] INFO UNet3DTrainer - Training iteration 519. Batch 0. Epoch [518/599]\n",
      "2019-07-31 15:47:57,286 [MainThread] INFO UNet3DTrainer - Training iteration 520. Batch 0. Epoch [519/599]\n",
      "2019-07-31 15:47:57,286 [MainThread] INFO UNet3DTrainer - Training iteration 520. Batch 0. Epoch [519/599]\n",
      "2019-07-31 15:47:58,090 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:47:58,090 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:47:59,411 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3196897804737091. Evaluation score: 0.5597493648529053\n",
      "2019-07-31 15:47:59,411 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.3196897804737091. Evaluation score: 0.5597493648529053\n",
      "2019-07-31 15:47:59,417 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5597493648529053\n",
      "2019-07-31 15:47:59,417 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5597493648529053\n",
      "2019-07-31 15:47:59,419 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:47:59,419 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:47:59,683 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:47:59,683 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:47:59,906 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.012459201738238335. Evaluation score: 0.5597493648529053\n",
      "2019-07-31 15:47:59,906 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.012459201738238335. Evaluation score: 0.5597493648529053\n",
      "2019-07-31 15:47:59,908 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:47:59,908 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:48:01,350 [MainThread] INFO UNet3DTrainer - Training iteration 521. Batch 0. Epoch [520/599]\n",
      "2019-07-31 15:48:01,350 [MainThread] INFO UNet3DTrainer - Training iteration 521. Batch 0. Epoch [520/599]\n",
      "2019-07-31 15:48:02,053 [MainThread] INFO UNet3DTrainer - Training iteration 522. Batch 0. Epoch [521/599]\n",
      "2019-07-31 15:48:02,053 [MainThread] INFO UNet3DTrainer - Training iteration 522. Batch 0. Epoch [521/599]\n",
      "2019-07-31 15:48:03,294 [MainThread] INFO UNet3DTrainer - Training iteration 523. Batch 0. Epoch [522/599]\n",
      "2019-07-31 15:48:03,294 [MainThread] INFO UNet3DTrainer - Training iteration 523. Batch 0. Epoch [522/599]\n",
      "2019-07-31 15:48:04,545 [MainThread] INFO UNet3DTrainer - Training iteration 524. Batch 0. Epoch [523/599]\n",
      "2019-07-31 15:48:04,545 [MainThread] INFO UNet3DTrainer - Training iteration 524. Batch 0. Epoch [523/599]\n",
      "2019-07-31 15:48:05,792 [MainThread] INFO UNet3DTrainer - Training iteration 525. Batch 0. Epoch [524/599]\n",
      "2019-07-31 15:48:05,792 [MainThread] INFO UNet3DTrainer - Training iteration 525. Batch 0. Epoch [524/599]\n",
      "2019-07-31 15:48:07,048 [MainThread] INFO UNet3DTrainer - Training iteration 526. Batch 0. Epoch [525/599]\n",
      "2019-07-31 15:48:07,048 [MainThread] INFO UNet3DTrainer - Training iteration 526. Batch 0. Epoch [525/599]\n",
      "2019-07-31 15:48:08,316 [MainThread] INFO UNet3DTrainer - Training iteration 527. Batch 0. Epoch [526/599]\n",
      "2019-07-31 15:48:08,316 [MainThread] INFO UNet3DTrainer - Training iteration 527. Batch 0. Epoch [526/599]\n",
      "2019-07-31 15:48:09,560 [MainThread] INFO UNet3DTrainer - Training iteration 528. Batch 0. Epoch [527/599]\n",
      "2019-07-31 15:48:09,560 [MainThread] INFO UNet3DTrainer - Training iteration 528. Batch 0. Epoch [527/599]\n",
      "2019-07-31 15:48:10,790 [MainThread] INFO UNet3DTrainer - Training iteration 529. Batch 0. Epoch [528/599]\n",
      "2019-07-31 15:48:10,790 [MainThread] INFO UNet3DTrainer - Training iteration 529. Batch 0. Epoch [528/599]\n",
      "2019-07-31 15:48:12,034 [MainThread] INFO UNet3DTrainer - Training iteration 530. Batch 0. Epoch [529/599]\n",
      "2019-07-31 15:48:12,034 [MainThread] INFO UNet3DTrainer - Training iteration 530. Batch 0. Epoch [529/599]\n",
      "2019-07-31 15:48:13,292 [MainThread] INFO UNet3DTrainer - Training iteration 531. Batch 0. Epoch [530/599]\n",
      "2019-07-31 15:48:13,292 [MainThread] INFO UNet3DTrainer - Training iteration 531. Batch 0. Epoch [530/599]\n",
      "2019-07-31 15:48:14,588 [MainThread] INFO UNet3DTrainer - Training iteration 532. Batch 0. Epoch [531/599]\n",
      "2019-07-31 15:48:14,588 [MainThread] INFO UNet3DTrainer - Training iteration 532. Batch 0. Epoch [531/599]\n",
      "2019-07-31 15:48:15,794 [MainThread] INFO UNet3DTrainer - Training iteration 533. Batch 0. Epoch [532/599]\n",
      "2019-07-31 15:48:15,794 [MainThread] INFO UNet3DTrainer - Training iteration 533. Batch 0. Epoch [532/599]\n",
      "2019-07-31 15:48:17,071 [MainThread] INFO UNet3DTrainer - Training iteration 534. Batch 0. Epoch [533/599]\n",
      "2019-07-31 15:48:17,071 [MainThread] INFO UNet3DTrainer - Training iteration 534. Batch 0. Epoch [533/599]\n",
      "2019-07-31 15:48:18,340 [MainThread] INFO UNet3DTrainer - Training iteration 535. Batch 0. Epoch [534/599]\n",
      "2019-07-31 15:48:18,340 [MainThread] INFO UNet3DTrainer - Training iteration 535. Batch 0. Epoch [534/599]\n",
      "2019-07-31 15:48:19,581 [MainThread] INFO UNet3DTrainer - Training iteration 536. Batch 0. Epoch [535/599]\n",
      "2019-07-31 15:48:19,581 [MainThread] INFO UNet3DTrainer - Training iteration 536. Batch 0. Epoch [535/599]\n",
      "2019-07-31 15:48:20,853 [MainThread] INFO UNet3DTrainer - Training iteration 537. Batch 0. Epoch [536/599]\n",
      "2019-07-31 15:48:20,853 [MainThread] INFO UNet3DTrainer - Training iteration 537. Batch 0. Epoch [536/599]\n",
      "2019-07-31 15:48:22,088 [MainThread] INFO UNet3DTrainer - Training iteration 538. Batch 0. Epoch [537/599]\n",
      "2019-07-31 15:48:22,088 [MainThread] INFO UNet3DTrainer - Training iteration 538. Batch 0. Epoch [537/599]\n",
      "2019-07-31 15:48:23,351 [MainThread] INFO UNet3DTrainer - Training iteration 539. Batch 0. Epoch [538/599]\n",
      "2019-07-31 15:48:23,351 [MainThread] INFO UNet3DTrainer - Training iteration 539. Batch 0. Epoch [538/599]\n",
      "2019-07-31 15:48:24,582 [MainThread] INFO UNet3DTrainer - Training iteration 540. Batch 0. Epoch [539/599]\n",
      "2019-07-31 15:48:24,582 [MainThread] INFO UNet3DTrainer - Training iteration 540. Batch 0. Epoch [539/599]\n",
      "2019-07-31 15:48:25,385 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:48:25,385 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:48:26,703 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.31955620646476746. Evaluation score: 0.5622528195381165\n",
      "2019-07-31 15:48:26,703 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.31955620646476746. Evaluation score: 0.5622528195381165\n",
      "2019-07-31 15:48:26,709 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5622528195381165\n",
      "2019-07-31 15:48:26,709 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5622528195381165\n",
      "2019-07-31 15:48:26,712 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:48:26,712 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:48:26,954 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:48:26,954 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:48:27,273 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.01220936793833971. Evaluation score: 0.5622528195381165\n",
      "2019-07-31 15:48:27,273 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.01220936793833971. Evaluation score: 0.5622528195381165\n",
      "2019-07-31 15:48:27,275 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:48:27,275 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:48:28,674 [MainThread] INFO UNet3DTrainer - Training iteration 541. Batch 0. Epoch [540/599]\n",
      "2019-07-31 15:48:28,674 [MainThread] INFO UNet3DTrainer - Training iteration 541. Batch 0. Epoch [540/599]\n",
      "2019-07-31 15:48:29,376 [MainThread] INFO UNet3DTrainer - Training iteration 542. Batch 0. Epoch [541/599]\n",
      "2019-07-31 15:48:29,376 [MainThread] INFO UNet3DTrainer - Training iteration 542. Batch 0. Epoch [541/599]\n",
      "2019-07-31 15:48:30,628 [MainThread] INFO UNet3DTrainer - Training iteration 543. Batch 0. Epoch [542/599]\n",
      "2019-07-31 15:48:30,628 [MainThread] INFO UNet3DTrainer - Training iteration 543. Batch 0. Epoch [542/599]\n",
      "2019-07-31 15:48:31,859 [MainThread] INFO UNet3DTrainer - Training iteration 544. Batch 0. Epoch [543/599]\n",
      "2019-07-31 15:48:31,859 [MainThread] INFO UNet3DTrainer - Training iteration 544. Batch 0. Epoch [543/599]\n",
      "2019-07-31 15:48:33,128 [MainThread] INFO UNet3DTrainer - Training iteration 545. Batch 0. Epoch [544/599]\n",
      "2019-07-31 15:48:33,128 [MainThread] INFO UNet3DTrainer - Training iteration 545. Batch 0. Epoch [544/599]\n",
      "2019-07-31 15:48:34,375 [MainThread] INFO UNet3DTrainer - Training iteration 546. Batch 0. Epoch [545/599]\n",
      "2019-07-31 15:48:34,375 [MainThread] INFO UNet3DTrainer - Training iteration 546. Batch 0. Epoch [545/599]\n",
      "2019-07-31 15:48:35,627 [MainThread] INFO UNet3DTrainer - Training iteration 547. Batch 0. Epoch [546/599]\n",
      "2019-07-31 15:48:35,627 [MainThread] INFO UNet3DTrainer - Training iteration 547. Batch 0. Epoch [546/599]\n",
      "2019-07-31 15:48:36,867 [MainThread] INFO UNet3DTrainer - Training iteration 548. Batch 0. Epoch [547/599]\n",
      "2019-07-31 15:48:36,867 [MainThread] INFO UNet3DTrainer - Training iteration 548. Batch 0. Epoch [547/599]\n",
      "2019-07-31 15:48:38,142 [MainThread] INFO UNet3DTrainer - Training iteration 549. Batch 0. Epoch [548/599]\n",
      "2019-07-31 15:48:38,142 [MainThread] INFO UNet3DTrainer - Training iteration 549. Batch 0. Epoch [548/599]\n",
      "2019-07-31 15:48:39,393 [MainThread] INFO UNet3DTrainer - Training iteration 550. Batch 0. Epoch [549/599]\n",
      "2019-07-31 15:48:39,393 [MainThread] INFO UNet3DTrainer - Training iteration 550. Batch 0. Epoch [549/599]\n",
      "2019-07-31 15:48:40,643 [MainThread] INFO UNet3DTrainer - Training iteration 551. Batch 0. Epoch [550/599]\n",
      "2019-07-31 15:48:40,643 [MainThread] INFO UNet3DTrainer - Training iteration 551. Batch 0. Epoch [550/599]\n",
      "2019-07-31 15:48:41,889 [MainThread] INFO UNet3DTrainer - Training iteration 552. Batch 0. Epoch [551/599]\n",
      "2019-07-31 15:48:41,889 [MainThread] INFO UNet3DTrainer - Training iteration 552. Batch 0. Epoch [551/599]\n",
      "2019-07-31 15:48:43,132 [MainThread] INFO UNet3DTrainer - Training iteration 553. Batch 0. Epoch [552/599]\n",
      "2019-07-31 15:48:43,132 [MainThread] INFO UNet3DTrainer - Training iteration 553. Batch 0. Epoch [552/599]\n",
      "2019-07-31 15:48:44,418 [MainThread] INFO UNet3DTrainer - Training iteration 554. Batch 0. Epoch [553/599]\n",
      "2019-07-31 15:48:44,418 [MainThread] INFO UNet3DTrainer - Training iteration 554. Batch 0. Epoch [553/599]\n",
      "2019-07-31 15:48:45,642 [MainThread] INFO UNet3DTrainer - Training iteration 555. Batch 0. Epoch [554/599]\n",
      "2019-07-31 15:48:45,642 [MainThread] INFO UNet3DTrainer - Training iteration 555. Batch 0. Epoch [554/599]\n",
      "2019-07-31 15:48:46,884 [MainThread] INFO UNet3DTrainer - Training iteration 556. Batch 0. Epoch [555/599]\n",
      "2019-07-31 15:48:46,884 [MainThread] INFO UNet3DTrainer - Training iteration 556. Batch 0. Epoch [555/599]\n",
      "2019-07-31 15:48:48,166 [MainThread] INFO UNet3DTrainer - Training iteration 557. Batch 0. Epoch [556/599]\n",
      "2019-07-31 15:48:48,166 [MainThread] INFO UNet3DTrainer - Training iteration 557. Batch 0. Epoch [556/599]\n",
      "2019-07-31 15:48:49,403 [MainThread] INFO UNet3DTrainer - Training iteration 558. Batch 0. Epoch [557/599]\n",
      "2019-07-31 15:48:49,403 [MainThread] INFO UNet3DTrainer - Training iteration 558. Batch 0. Epoch [557/599]\n",
      "2019-07-31 15:48:50,661 [MainThread] INFO UNet3DTrainer - Training iteration 559. Batch 0. Epoch [558/599]\n",
      "2019-07-31 15:48:50,661 [MainThread] INFO UNet3DTrainer - Training iteration 559. Batch 0. Epoch [558/599]\n",
      "2019-07-31 15:48:51,904 [MainThread] INFO UNet3DTrainer - Training iteration 560. Batch 0. Epoch [559/599]\n",
      "2019-07-31 15:48:51,904 [MainThread] INFO UNet3DTrainer - Training iteration 560. Batch 0. Epoch [559/599]\n",
      "2019-07-31 15:48:52,710 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:48:52,710 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:48:54,031 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.31942591071128845. Evaluation score: 0.5656870007514954\n",
      "2019-07-31 15:48:54,031 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.31942591071128845. Evaluation score: 0.5656870007514954\n",
      "2019-07-31 15:48:54,037 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5656870007514954\n",
      "2019-07-31 15:48:54,037 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5656870007514954\n",
      "2019-07-31 15:48:54,039 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:48:54,039 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:48:54,303 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:48:54,303 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:48:54,536 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.011965153738856316. Evaluation score: 0.5653301477432251\n",
      "2019-07-31 15:48:54,536 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.011965153738856316. Evaluation score: 0.5653301477432251\n",
      "2019-07-31 15:48:54,538 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:48:54,538 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:48:56,448 [MainThread] INFO UNet3DTrainer - Training iteration 561. Batch 0. Epoch [560/599]\n",
      "2019-07-31 15:48:56,448 [MainThread] INFO UNet3DTrainer - Training iteration 561. Batch 0. Epoch [560/599]\n",
      "2019-07-31 15:48:57,194 [MainThread] INFO UNet3DTrainer - Training iteration 562. Batch 0. Epoch [561/599]\n",
      "2019-07-31 15:48:57,194 [MainThread] INFO UNet3DTrainer - Training iteration 562. Batch 0. Epoch [561/599]\n",
      "2019-07-31 15:48:58,401 [MainThread] INFO UNet3DTrainer - Training iteration 563. Batch 0. Epoch [562/599]\n",
      "2019-07-31 15:48:58,401 [MainThread] INFO UNet3DTrainer - Training iteration 563. Batch 0. Epoch [562/599]\n",
      "2019-07-31 15:48:59,647 [MainThread] INFO UNet3DTrainer - Training iteration 564. Batch 0. Epoch [563/599]\n",
      "2019-07-31 15:48:59,647 [MainThread] INFO UNet3DTrainer - Training iteration 564. Batch 0. Epoch [563/599]\n",
      "2019-07-31 15:49:00,897 [MainThread] INFO UNet3DTrainer - Training iteration 565. Batch 0. Epoch [564/599]\n",
      "2019-07-31 15:49:00,897 [MainThread] INFO UNet3DTrainer - Training iteration 565. Batch 0. Epoch [564/599]\n",
      "2019-07-31 15:49:02,155 [MainThread] INFO UNet3DTrainer - Training iteration 566. Batch 0. Epoch [565/599]\n",
      "2019-07-31 15:49:02,155 [MainThread] INFO UNet3DTrainer - Training iteration 566. Batch 0. Epoch [565/599]\n",
      "2019-07-31 15:49:03,406 [MainThread] INFO UNet3DTrainer - Training iteration 567. Batch 0. Epoch [566/599]\n",
      "2019-07-31 15:49:03,406 [MainThread] INFO UNet3DTrainer - Training iteration 567. Batch 0. Epoch [566/599]\n",
      "2019-07-31 15:49:04,634 [MainThread] INFO UNet3DTrainer - Training iteration 568. Batch 0. Epoch [567/599]\n",
      "2019-07-31 15:49:04,634 [MainThread] INFO UNet3DTrainer - Training iteration 568. Batch 0. Epoch [567/599]\n",
      "2019-07-31 15:49:05,891 [MainThread] INFO UNet3DTrainer - Training iteration 569. Batch 0. Epoch [568/599]\n",
      "2019-07-31 15:49:05,891 [MainThread] INFO UNet3DTrainer - Training iteration 569. Batch 0. Epoch [568/599]\n",
      "2019-07-31 15:49:07,144 [MainThread] INFO UNet3DTrainer - Training iteration 570. Batch 0. Epoch [569/599]\n",
      "2019-07-31 15:49:07,144 [MainThread] INFO UNet3DTrainer - Training iteration 570. Batch 0. Epoch [569/599]\n",
      "2019-07-31 15:49:08,435 [MainThread] INFO UNet3DTrainer - Training iteration 571. Batch 0. Epoch [570/599]\n",
      "2019-07-31 15:49:08,435 [MainThread] INFO UNet3DTrainer - Training iteration 571. Batch 0. Epoch [570/599]\n",
      "2019-07-31 15:49:09,683 [MainThread] INFO UNet3DTrainer - Training iteration 572. Batch 0. Epoch [571/599]\n",
      "2019-07-31 15:49:09,683 [MainThread] INFO UNet3DTrainer - Training iteration 572. Batch 0. Epoch [571/599]\n",
      "2019-07-31 15:49:10,921 [MainThread] INFO UNet3DTrainer - Training iteration 573. Batch 0. Epoch [572/599]\n",
      "2019-07-31 15:49:10,921 [MainThread] INFO UNet3DTrainer - Training iteration 573. Batch 0. Epoch [572/599]\n",
      "2019-07-31 15:49:12,160 [MainThread] INFO UNet3DTrainer - Training iteration 574. Batch 0. Epoch [573/599]\n",
      "2019-07-31 15:49:12,160 [MainThread] INFO UNet3DTrainer - Training iteration 574. Batch 0. Epoch [573/599]\n",
      "2019-07-31 15:49:13,451 [MainThread] INFO UNet3DTrainer - Training iteration 575. Batch 0. Epoch [574/599]\n",
      "2019-07-31 15:49:13,451 [MainThread] INFO UNet3DTrainer - Training iteration 575. Batch 0. Epoch [574/599]\n",
      "2019-07-31 15:49:14,661 [MainThread] INFO UNet3DTrainer - Training iteration 576. Batch 0. Epoch [575/599]\n",
      "2019-07-31 15:49:14,661 [MainThread] INFO UNet3DTrainer - Training iteration 576. Batch 0. Epoch [575/599]\n",
      "2019-07-31 15:49:15,918 [MainThread] INFO UNet3DTrainer - Training iteration 577. Batch 0. Epoch [576/599]\n",
      "2019-07-31 15:49:15,918 [MainThread] INFO UNet3DTrainer - Training iteration 577. Batch 0. Epoch [576/599]\n",
      "2019-07-31 15:49:17,164 [MainThread] INFO UNet3DTrainer - Training iteration 578. Batch 0. Epoch [577/599]\n",
      "2019-07-31 15:49:17,164 [MainThread] INFO UNet3DTrainer - Training iteration 578. Batch 0. Epoch [577/599]\n",
      "2019-07-31 15:49:18,439 [MainThread] INFO UNet3DTrainer - Training iteration 579. Batch 0. Epoch [578/599]\n",
      "2019-07-31 15:49:18,439 [MainThread] INFO UNet3DTrainer - Training iteration 579. Batch 0. Epoch [578/599]\n",
      "2019-07-31 15:49:19,681 [MainThread] INFO UNet3DTrainer - Training iteration 580. Batch 0. Epoch [579/599]\n",
      "2019-07-31 15:49:19,681 [MainThread] INFO UNet3DTrainer - Training iteration 580. Batch 0. Epoch [579/599]\n",
      "2019-07-31 15:49:20,483 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:49:20,483 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:49:21,809 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.31929856538772583. Evaluation score: 0.5684434175491333\n",
      "2019-07-31 15:49:21,809 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.31929856538772583. Evaluation score: 0.5684434175491333\n",
      "2019-07-31 15:49:21,815 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5684434175491333\n",
      "2019-07-31 15:49:21,815 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5684434175491333\n",
      "2019-07-31 15:49:21,817 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:49:21,817 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:49:22,087 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:49:22,087 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:49:22,487 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.011726150289177895. Evaluation score: 0.5684926509857178\n",
      "2019-07-31 15:49:22,487 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.011726150289177895. Evaluation score: 0.5684926509857178\n",
      "2019-07-31 15:49:22,489 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:49:22,489 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:49:24,277 [MainThread] INFO UNet3DTrainer - Training iteration 581. Batch 0. Epoch [580/599]\n",
      "2019-07-31 15:49:24,277 [MainThread] INFO UNet3DTrainer - Training iteration 581. Batch 0. Epoch [580/599]\n",
      "2019-07-31 15:49:25,004 [MainThread] INFO UNet3DTrainer - Training iteration 582. Batch 0. Epoch [581/599]\n",
      "2019-07-31 15:49:25,004 [MainThread] INFO UNet3DTrainer - Training iteration 582. Batch 0. Epoch [581/599]\n",
      "2019-07-31 15:49:26,217 [MainThread] INFO UNet3DTrainer - Training iteration 583. Batch 0. Epoch [582/599]\n",
      "2019-07-31 15:49:26,217 [MainThread] INFO UNet3DTrainer - Training iteration 583. Batch 0. Epoch [582/599]\n",
      "2019-07-31 15:49:27,476 [MainThread] INFO UNet3DTrainer - Training iteration 584. Batch 0. Epoch [583/599]\n",
      "2019-07-31 15:49:27,476 [MainThread] INFO UNet3DTrainer - Training iteration 584. Batch 0. Epoch [583/599]\n",
      "2019-07-31 15:49:28,708 [MainThread] INFO UNet3DTrainer - Training iteration 585. Batch 0. Epoch [584/599]\n",
      "2019-07-31 15:49:28,708 [MainThread] INFO UNet3DTrainer - Training iteration 585. Batch 0. Epoch [584/599]\n",
      "2019-07-31 15:49:29,973 [MainThread] INFO UNet3DTrainer - Training iteration 586. Batch 0. Epoch [585/599]\n",
      "2019-07-31 15:49:29,973 [MainThread] INFO UNet3DTrainer - Training iteration 586. Batch 0. Epoch [585/599]\n",
      "2019-07-31 15:49:31,222 [MainThread] INFO UNet3DTrainer - Training iteration 587. Batch 0. Epoch [586/599]\n",
      "2019-07-31 15:49:31,222 [MainThread] INFO UNet3DTrainer - Training iteration 587. Batch 0. Epoch [586/599]\n",
      "2019-07-31 15:49:32,490 [MainThread] INFO UNet3DTrainer - Training iteration 588. Batch 0. Epoch [587/599]\n",
      "2019-07-31 15:49:32,490 [MainThread] INFO UNet3DTrainer - Training iteration 588. Batch 0. Epoch [587/599]\n",
      "2019-07-31 15:49:33,712 [MainThread] INFO UNet3DTrainer - Training iteration 589. Batch 0. Epoch [588/599]\n",
      "2019-07-31 15:49:33,712 [MainThread] INFO UNet3DTrainer - Training iteration 589. Batch 0. Epoch [588/599]\n",
      "2019-07-31 15:49:34,977 [MainThread] INFO UNet3DTrainer - Training iteration 590. Batch 0. Epoch [589/599]\n",
      "2019-07-31 15:49:34,977 [MainThread] INFO UNet3DTrainer - Training iteration 590. Batch 0. Epoch [589/599]\n",
      "2019-07-31 15:49:36,232 [MainThread] INFO UNet3DTrainer - Training iteration 591. Batch 0. Epoch [590/599]\n",
      "2019-07-31 15:49:36,232 [MainThread] INFO UNet3DTrainer - Training iteration 591. Batch 0. Epoch [590/599]\n",
      "2019-07-31 15:49:37,486 [MainThread] INFO UNet3DTrainer - Training iteration 592. Batch 0. Epoch [591/599]\n",
      "2019-07-31 15:49:37,486 [MainThread] INFO UNet3DTrainer - Training iteration 592. Batch 0. Epoch [591/599]\n",
      "2019-07-31 15:49:38,755 [MainThread] INFO UNet3DTrainer - Training iteration 593. Batch 0. Epoch [592/599]\n",
      "2019-07-31 15:49:38,755 [MainThread] INFO UNet3DTrainer - Training iteration 593. Batch 0. Epoch [592/599]\n",
      "2019-07-31 15:49:40,010 [MainThread] INFO UNet3DTrainer - Training iteration 594. Batch 0. Epoch [593/599]\n",
      "2019-07-31 15:49:40,010 [MainThread] INFO UNet3DTrainer - Training iteration 594. Batch 0. Epoch [593/599]\n",
      "2019-07-31 15:49:41,284 [MainThread] INFO UNet3DTrainer - Training iteration 595. Batch 0. Epoch [594/599]\n",
      "2019-07-31 15:49:41,284 [MainThread] INFO UNet3DTrainer - Training iteration 595. Batch 0. Epoch [594/599]\n",
      "2019-07-31 15:49:42,512 [MainThread] INFO UNet3DTrainer - Training iteration 596. Batch 0. Epoch [595/599]\n",
      "2019-07-31 15:49:42,512 [MainThread] INFO UNet3DTrainer - Training iteration 596. Batch 0. Epoch [595/599]\n",
      "2019-07-31 15:49:43,758 [MainThread] INFO UNet3DTrainer - Training iteration 597. Batch 0. Epoch [596/599]\n",
      "2019-07-31 15:49:43,758 [MainThread] INFO UNet3DTrainer - Training iteration 597. Batch 0. Epoch [596/599]\n",
      "2019-07-31 15:49:45,020 [MainThread] INFO UNet3DTrainer - Training iteration 598. Batch 0. Epoch [597/599]\n",
      "2019-07-31 15:49:45,020 [MainThread] INFO UNet3DTrainer - Training iteration 598. Batch 0. Epoch [597/599]\n",
      "2019-07-31 15:49:46,260 [MainThread] INFO UNet3DTrainer - Training iteration 599. Batch 0. Epoch [598/599]\n",
      "2019-07-31 15:49:46,260 [MainThread] INFO UNet3DTrainer - Training iteration 599. Batch 0. Epoch [598/599]\n",
      "2019-07-31 15:49:47,513 [MainThread] INFO UNet3DTrainer - Training iteration 600. Batch 0. Epoch [599/599]\n",
      "2019-07-31 15:49:47,513 [MainThread] INFO UNet3DTrainer - Training iteration 600. Batch 0. Epoch [599/599]\n",
      "2019-07-31 15:49:48,316 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:49:48,316 [MainThread] INFO UNet3DTrainer - Validating...\n",
      "2019-07-31 15:49:49,639 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.31917399168014526. Evaluation score: 0.5726155042648315\n",
      "2019-07-31 15:49:49,639 [MainThread] INFO UNet3DTrainer - Validation finished. Loss: 0.31917399168014526. Evaluation score: 0.5726155042648315\n",
      "2019-07-31 15:49:49,645 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5726155042648315\n",
      "2019-07-31 15:49:49,645 [MainThread] INFO UNet3DTrainer - Saving new best evaluation metric: 0.5726155042648315\n",
      "2019-07-31 15:49:49,647 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:49:49,647 [MainThread] INFO UNet3DTrainer - Saving last checkpoint to '3dunet/last_checkpoint.pytorch'\n",
      "2019-07-31 15:49:49,912 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:49:49,912 [MainThread] INFO UNet3DTrainer - Saving best checkpoint to '3dunet/best_checkpoint.pytorch'\n",
      "2019-07-31 15:49:50,134 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.011492030695080757. Evaluation score: 0.5726155042648315\n",
      "2019-07-31 15:49:50,134 [MainThread] INFO UNet3DTrainer - Training stats. Loss: 0.011492030695080757. Evaluation score: 0.5726155042648315\n",
      "2019-07-31 15:49:50,136 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n",
      "2019-07-31 15:49:50,136 [MainThread] INFO UNet3DTrainer - Logging model parameters and gradients\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Load and log experiment configuration\n",
    "manual_seed = config.get('manual_seed', None)\n",
    "if manual_seed is not None:\n",
    "    logger.info(f'Seed the RNG for all devices with {manual_seed}')\n",
    "    torch.manual_seed(manual_seed)\n",
    "    # see https://pytorch.org/docs/stable/notes/randomness.html\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "\n",
    "# Create the model\n",
    "model = get_model(config)\n",
    "\n",
    "# put the model on GPUs (if available)\n",
    "logger.info(f\"Sending the model to '{config['device']}'\")\n",
    "model = model.to(config['device'])\n",
    "                \n",
    "# Log the number of learnable parameters\n",
    "logger.info(f'Number of learnable params {get_number_of_learnable_parameters(model)}')\n",
    "\n",
    "# Create loss criterion\n",
    "loss_criterion = get_loss_criterion(config)\n",
    "# Create evaluation metric\n",
    "eval_criterion = get_evaluation_metric(config)\n",
    "\n",
    "# Create data loaders\n",
    "loaders = get_train_loaders(config)\n",
    "\n",
    "# Create the optimizer\n",
    "optimizer = _create_optimizer(config, model)\n",
    "\n",
    "# Create learning rate adjustment strategy\n",
    "lr_scheduler = _create_lr_scheduler(config, optimizer)\n",
    "\n",
    "# Create model trainer\n",
    "trainer = _create_trainer(config, model=model, optimizer=optimizer, lr_scheduler=lr_scheduler,\n",
    "                          loss_criterion=loss_criterion, eval_criterion=eval_criterion, loaders=loaders,\n",
    "                          logger=logger)\n",
    "# Start training\n",
    "trainer.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}