train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import simplejson\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import threading\n",
    "import tensorflow.contrib.slim as slim\n",
    "from utils import data_utils, train_utils\n",
    "import datetime\n",
    "import os\n",
    "import time\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def argument_scope(H, phase):\n",
    "    '''\n",
    "    This returns the arg_scope for slim.arg_scope(), which defines the options for slim.functions\n",
    "    '''\n",
    "    padding = H['padding']\n",
    "    is_training = {'train': True, 'validate': False, 'test': False}[phase]\n",
    "    pool_kernel = [2, 2]\n",
    "    pool_stride = 2\n",
    "\n",
    "    \n",
    "    params = {\n",
    "        \"decay\": 0.997,\n",
    "        \"epsilon\": 0.001,\n",
    "    }\n",
    "\n",
    "    with slim.arg_scope([slim.conv2d], \n",
    "                        # slim.relu would raise an error here\n",
    "                        activation_fn=tf.nn.relu, \n",
    "                        padding=padding, \n",
    "                        normalizer_fn=slim.batch_norm, \n",
    "                        # normalizer_fn=None,\n",
    "                        weights_initializer=tf.contrib.layers.variance_scaling_initializer()):\n",
    "        with slim.arg_scope([slim.batch_norm, slim.dropout], is_training=is_training):\n",
    "            with slim.arg_scope([slim.max_pool2d], stride=pool_stride, kernel_size=pool_kernel):\n",
    "                with slim.arg_scope([slim.conv2d_transpose], \n",
    "                                    activation_fn=None, \n",
    "                                    normalizer_fn=None,\n",
    "                                    padding=padding, \n",
    "                                    weights_initializer=tf.contrib.layers.variance_scaling_initializer()) as sc:\n",
    "                    return sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def build_pred(x_in, H, phase):\n",
    "    '''\n",
    "    This function builds the prediction model\n",
    "    '''\n",
    "    num_class = H['num_class']\n",
    "    \n",
    "    conv_kernel_1 = [1, 1]\n",
    "    conv_kernel_3 = [3, 3]\n",
    "    pool_kernel = [2, 2]\n",
    "    pool_stride = 2\n",
    "\n",
    "    early_feature = {}\n",
    "    reuse = {'train': False, 'validate': True, 'test': False}[phase]\n",
    "    \n",
    "    with slim.arg_scope(argument_scope(H, phase)):\n",
    "        \n",
    "        scope_name = 'block_1'\n",
    "        x_input = x_in\n",
    "        num_outputs = 64\n",
    "        with tf.variable_scope(scope_name, reuse = reuse):\n",
    "            layer_1 = slim.conv2d(x_input, num_outputs, conv_kernel_3, scope='conv1')\n",
    "            layer_2 = slim.conv2d(layer_1, num_outputs, conv_kernel_3, scope='conv2')\n",
    "            early_feature[scope_name] = layer_2\n",
    "        \n",
    "        scope_name = 'block_2'\n",
    "        x_input = slim.max_pool2d(layer_2)\n",
    "        num_outputs = 128\n",
    "        with tf.variable_scope(scope_name, reuse = reuse):\n",
    "            layer_1 = slim.conv2d(x_input, num_outputs, conv_kernel_3, scope='conv1')\n",
    "            layer_2 = slim.conv2d(layer_1, num_outputs, conv_kernel_3, scope='conv2')\n",
    "            early_feature[scope_name] = layer_2\n",
    "\n",
    "        scope_name = 'block_3'\n",
    "        x_input = slim.max_pool2d(layer_2)\n",
    "        num_outputs = 256\n",
    "        with tf.variable_scope(scope_name, reuse = reuse):\n",
    "            layer_1 = slim.conv2d(x_input, num_outputs, conv_kernel_3, scope='conv1')\n",
    "            layer_2 = slim.conv2d(layer_1, num_outputs, conv_kernel_3, scope='conv2')\n",
    "            early_feature[scope_name] = layer_2\n",
    "            \n",
    "        scope_name = 'block_4'\n",
    "        x_input = slim.max_pool2d(layer_2)\n",
    "        num_outputs = 512\n",
    "        with tf.variable_scope(scope_name, reuse = reuse):\n",
    "            layer_1 = slim.conv2d(x_input, num_outputs, conv_kernel_3, scope='conv1')\n",
    "            layer_2 = slim.conv2d(layer_1, num_outputs, conv_kernel_3, scope='conv2')\n",
    "            early_feature[scope_name] = layer_2\n",
    "\n",
    "        scope_name = 'block_5'\n",
    "        x_input = slim.max_pool2d(layer_2)\n",
    "        num_outputs = 1024\n",
    "        with tf.variable_scope(scope_name, reuse = reuse):\n",
    "            layer_1 = slim.conv2d(x_input, num_outputs, conv_kernel_3, scope='conv1')\n",
    "            layer_2 = slim.conv2d(layer_1, num_outputs, conv_kernel_3, scope='conv2')\n",
    "            early_feature[scope_name] = layer_2\n",
    "            \n",
    "        scope_name = 'block_6'\n",
    "        num_outputs = 512\n",
    "        with tf.variable_scope(scope_name, reuse = reuse):\n",
    "            trans_layer = slim.conv2d_transpose(\n",
    "                layer_2, num_outputs, pool_kernel, pool_stride, scope='conv_trans')\n",
    "            x_input = tf.concat([early_feature['block_4'], trans_layer], axis=3)\n",
    "            layer_1 = slim.conv2d(x_input, num_outputs, conv_kernel_3, scope='conv1')\n",
    "            layer_2 = slim.conv2d(layer_1, num_outputs, conv_kernel_3, scope='conv2')\n",
    "            early_feature[scope_name] = layer_2\n",
    "            \n",
    "        scope_name = 'block_7'\n",
    "        num_outputs = 256\n",
    "        with tf.variable_scope(scope_name, reuse = reuse):\n",
    "            trans_layer = slim.conv2d_transpose(\n",
    "                layer_2, num_outputs, pool_kernel, pool_stride, scope='conv_trans')\n",
    "            x_input = tf.concat([early_feature['block_3'], trans_layer], axis=3)\n",
    "            layer_1 = slim.conv2d(x_input, num_outputs, conv_kernel_3, scope='conv1')\n",
    "            layer_2 = slim.conv2d(layer_1, num_outputs, conv_kernel_3, scope='conv2')\n",
    "            early_feature[scope_name] = layer_2\n",
    "            \n",
    "        scope_name = 'block_8'\n",
    "        num_outputs = 128\n",
    "        with tf.variable_scope(scope_name, reuse = reuse):\n",
    "            trans_layer = slim.conv2d_transpose(\n",
    "                layer_2, num_outputs, pool_kernel, pool_stride, scope='conv_trans')\n",
    "            x_input = tf.concat([early_feature['block_2'], trans_layer], axis=3)\n",
    "            layer_1 = slim.conv2d(x_input, num_outputs, conv_kernel_3, scope='conv1')\n",
    "            layer_2 = slim.conv2d(layer_1, num_outputs, conv_kernel_3, scope='conv2')\n",
    "            early_feature[scope_name] = layer_2\n",
    "        \n",
    "        scope_name = 'block_9'\n",
    "        num_outputs = 64\n",
    "        with tf.variable_scope(scope_name, reuse = reuse):\n",
    "            trans_layer = slim.conv2d_transpose(\n",
    "                layer_2, num_outputs, pool_kernel, pool_stride, scope='conv_trans')\n",
    "            x_input = tf.concat([early_feature['block_1'], trans_layer], axis=3)\n",
    "            layer_1 = slim.conv2d(x_input, num_outputs, conv_kernel_3, scope='conv1')\n",
    "            layer_2 = slim.conv2d(layer_1, num_outputs, conv_kernel_3, scope='conv2')\n",
    "            early_feature[scope_name] = layer_2\n",
    "        \n",
    "        scope_name = 'pred'\n",
    "        with tf.variable_scope(scope_name, reuse = reuse):\n",
    "            # layer_1 = slim.conv2d(layer_2, num_class, conv_kernel_1, scope='conv1', activation_fn=None, normalizer_fn=None)\n",
    "            layer_1 = slim.conv2d(layer_2, 1, conv_kernel_1, scope='conv1', activation_fn=None, normalizer_fn=None)\n",
    "\n",
    "            early_feature[scope_name] = layer_1\n",
    "            \n",
    "            # pred = tf.argmax(tf.nn.softmax(logits=layer_1), axis=3)\n",
    "            pred = tf.sigmoid(layer_1)\n",
    "                \n",
    "        return tf.squeeze(layer_1), tf.squeeze(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def build_loss(x_in, y_in, H, phase):\n",
    "    '''\n",
    "    This function builds the loss and accuracy\n",
    "    '''\n",
    "    im_width = H['im_width']\n",
    "    im_height = H['im_height']\n",
    "    batch_size = H['batch_size']\n",
    "    start_ind = H['start_ind']\n",
    "    valid_size = H['valid_size']\n",
    "    num_class = H['num_class']\n",
    "    epsilon = H['epsilon']\n",
    "    apply_class_balancing = H['apply_class_balancing']\n",
    "    \n",
    "    logits, pred = build_pred(x_in, H, phase)\n",
    "    y_crop = tf.cast(tf.slice(y_in, begin=[0, start_ind, start_ind], size=[-1, valid_size, valid_size]), tf.float32)\n",
    "    logits_crop = tf.slice(logits, \n",
    "                         begin=[0, start_ind, start_ind], \n",
    "                         size=[-1, valid_size, valid_size])\n",
    "    pred_crop = tf.cast(tf.slice(pred, \n",
    "                             begin=[0, start_ind, start_ind], \n",
    "                             size=[-1, valid_size, valid_size]), tf.float32)\n",
    "    \n",
    "    # formulation of weighted cross entropy loss, dice index: https://arxiv.org/pdf/1707.03237.pdf\n",
    "    if H['loss_function'] == 'cross_entropy':\n",
    "        if apply_class_balancing:\n",
    "            loss = tf.reduce_mean(\n",
    "                tf.nn.weighted_cross_entropy_with_logits(\n",
    "                    targets=y_crop, logits=logits_crop, pos_weight=1. / class_weight))\n",
    "        else:\n",
    "            loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=y_crop, logits=logits_crop))\n",
    "        \n",
    "    elif H['loss_function'] == 'dice':\n",
    "        \n",
    "        intersection = tf.reduce_sum(tf.multiply(y_crop, pred_crop))\n",
    "        union = tf.reduce_sum(tf.square(y_crop)) + tf.reduce_sum(tf.square(pred_crop))\n",
    "        loss = 1. - 2 * intersection / (union + tf.constant(epsilon))\n",
    "                \n",
    "    elif H['loss_function'] == 'jaccard':\n",
    "        intersection = tf.reduce_sum(tf.multiply(y_crop, pred_crop))\n",
    "        union = tf.reduce_sum(y_crop) + tf.reduce_sum(pred_crop) - intersection\n",
    "        loss = 1. - intersection / (union + tf.constant(epsilon))\n",
    "    \n",
    "    elif H['loss_function'] == 'combo-jaccard':\n",
    "        if apply_class_balancing:\n",
    "            cen_loss = tf.reduce_mean(\n",
    "                tf.nn.weighted_cross_entropy_with_logits(\n",
    "                    targets=y_crop, logits=logits_crop, pos_weight=1. / class_weight))\n",
    "        else:\n",
    "            cen_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=y_crop, logits=logits_crop))\n",
    "            \n",
    "        intersection = tf.reduce_sum(tf.multiply(y_crop, pred_crop))\n",
    "        union = tf.reduce_sum(y_crop) + tf.reduce_sum(pred_crop) - intersection\n",
    "        jaccard_loss = - tf.log((intersection + tf.constant(epsilon)) / (union + tf.constant(epsilon)))\n",
    "        loss = cen_loss + jaccard_loss\n",
    "        \n",
    "    elif H['loss_function'] == 'combo-dice':\n",
    "        if apply_class_balancing:\n",
    "            cen_loss = tf.reduce_mean(\n",
    "                tf.nn.weighted_cross_entropy_with_logits(\n",
    "                    targets=y_crop, logits=logits_crop, pos_weight=1. / class_weight))\n",
    "        else:\n",
    "            cen_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=y_crop, logits=logits_crop))\n",
    "            \n",
    "        intersection = tf.reduce_sum(tf.multiply(y_crop, pred_crop))\n",
    "        union = tf.reduce_sum(y_crop) + tf.reduce_sum(pred_crop) - intersection\n",
    "        dice_loss = - tf.log((intersection + tf.constant(epsilon)) / (union + tf.constant(epsilon)))\n",
    "        loss = cen_loss + dice_loss\n",
    "        \n",
    "    pred_thres = tf.cast(tf.greater(pred_crop, 0.5), tf.float32)\n",
    "    inter = tf.reduce_sum(tf.multiply(tf.cast(y_crop, tf.float32), pred_thres))\n",
    "    uni = tf.reduce_sum(tf.cast(y_crop, tf.float32)) + tf.reduce_sum(pred_thres) - inter\n",
    "    jaccard = inter / (uni + tf.constant(epsilon))\n",
    "    \n",
    "    return loss, jaccard, logits_crop, pred_crop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def build(queues, H):\n",
    "    '''\n",
    "    This function returns the train operation, summary, global step\n",
    "    '''\n",
    "    im_width = H['im_width']\n",
    "    im_height = H['im_height']\n",
    "    num_class = H['num_class']\n",
    "    num_channel = H['num_channel']\n",
    "    batch_size = H['batch_size']\n",
    "    log_dir = H['log_dir']\n",
    "    norm_threshold = H['norm_threshold']\n",
    "    \n",
    "    loss, accuracy, x_in, y_in, logits, pred = {}, {}, {}, {}, {}, {}\n",
    "    for phase in ['train', 'validate']:\n",
    "        x_in[phase], y_in[phase] = queues[phase].dequeue_many(batch_size)\n",
    "        loss[phase], accuracy[phase], logits[phase], pred[phase] = build_loss(x_in[phase], y_in[phase], H, phase)\n",
    "    \n",
    "    learning_rate = tf.placeholder(dtype=tf.float32)\n",
    "    opt = tf.train.AdamOptimizer(learning_rate=learning_rate, beta1=0.8, beta2=0.99)\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "    tvars = tf.trainable_variables()\n",
    "    grads= tf.gradients(loss['train'], tvars)\n",
    "    grads, norm = tf.clip_by_global_norm(grads, norm_threshold)\n",
    "    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "    with tf.control_dependencies(update_ops):\n",
    "        train_op = opt.apply_gradients(zip(grads, tvars), global_step = global_step)\n",
    "    \n",
    "    for phase in ['train', 'validate']:\n",
    "        \n",
    "        tf.summary.scalar(name=phase + '/loss', tensor=loss[phase])\n",
    "        tf.summary.scalar(name=phase + '/accuracy', tensor=accuracy[phase])\n",
    "        \n",
    "        mean, var = tf.nn.moments(logits[phase], axes=[0, 1, 2])\n",
    "        tf.summary.scalar(name=phase + '/logits/mean', tensor=mean)\n",
    "        tf.summary.scalar(name=phase + '/logits/variance', tensor=var)\n",
    "        \n",
    "        mean, var = tf.nn.moments(pred[phase], axes=[0, 1, 2])\n",
    "        tf.summary.scalar(name=phase + '/pred/mean', tensor=mean)\n",
    "        tf.summary.scalar(name=phase + '/pred/variance', tensor=var)\n",
    "        \n",
    "    summary_op = tf.summary.merge_all()\n",
    "    return loss, accuracy, train_op, summary_op, learning_rate, global_step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "hypes = './hypes/hypes.json'\n",
    "with open(hypes, 'r') as f:\n",
    "    H = simplejson.load(f)\n",
    "    # H['loss_function'] = 'dice'\n",
    "    im_width = H['im_width']\n",
    "    im_height = H['im_height']\n",
    "    num_class = H['num_class']\n",
    "    num_channel = H['num_channel']\n",
    "    queue_size = H['queue_size']\n",
    "    save_iter = H['save_iter']\n",
    "    print_iter = H['print_iter']\n",
    "    class_type = H['class_type']\n",
    "    train_iter = H['train_iter']\n",
    "    lr = H['lr']\n",
    "    lr_decay_iter = H['lr_decay_iter']\n",
    "    log_dir = H['log_dir']\n",
    "    batch_size = H['batch_size']\n",
    "now = datetime.datetime.now()\n",
    "\n",
    "now_path = str(now.month) + '-' + str(now.day) + '_' + str(now.hour) + '-' + str(now.minute) + '_' + H['loss_function']\n",
    "ckpt_path = os.path.join(log_dir, now_path, 'ckpt', 'ckpt')\n",
    "hypes_path = os.path.join(log_dir, now_path, 'hypes')\n",
    "summary_path = os.path.join(log_dir, now_path, 'summary')\n",
    "\n",
    "for path in [ckpt_path, hypes_path, summary_path]:\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "\n",
    "with open(os.path.join(hypes_path, 'hypes.json'), 'w') as f:\n",
    "    simplejson.dump(H, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'apply_class_balancing': False,\n",
       " 'batch_size': 60,\n",
       " 'class_type': 0,\n",
       " 'epsilon': 1.0,\n",
       " 'im_height': 144,\n",
       " 'im_width': 144,\n",
       " 'log_dir': 'log_dir',\n",
       " 'loss_function': 'combo-jaccard',\n",
       " 'lr': 0.0001,\n",
       " 'lr_decay_iter': 2000,\n",
       " 'norm_threshold': 1.0,\n",
       " 'num_channel': 16,\n",
       " 'num_class': 2,\n",
       " 'padding': 'SAME',\n",
       " 'print_iter': 100,\n",
       " 'queue_size': 200,\n",
       " 'save_iter': 1000,\n",
       " 'start_ind': 32,\n",
       " 'train_iter': 20000,\n",
       " 'valid_size': 80}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "H"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting class stats [========================] 100%\n"
     ]
    }
   ],
   "source": [
    "class_weight = data_utils.calculate_class_weights()[data_utils.CLASSES[class_type + 1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def enqueue_thread(sess, data_gen, coord, phase, enqueue_op):\n",
    "    while not coord.should_stop():\n",
    "        img, label = data_gen.next()\n",
    "        sess.run(enqueue_op, feed_dict={x_in[phase]: img, y_in[phase]: label})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "x_in, y_in, queues, enqueue_op = {}, {}, {}, {}\n",
    "shape = ((im_width, im_height, num_channel), \n",
    "         (im_width, im_height))\n",
    "for phase in ['train', 'validate']:\n",
    "    x_in[phase] = tf.placeholder(dtype=tf.float32)\n",
    "    y_in[phase] = tf.placeholder(dtype=tf.float32)\n",
    "    queues[phase] = tf.FIFOQueue(capacity=queue_size, shapes=shape, dtypes=(tf.float32,tf.float32))\n",
    "    enqueue_op[phase] = queues[phase].enqueue_many([x_in[phase], y_in[phase]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading training data: [====================] 100%\n",
      "Loading validation data: [===] 100%\n"
     ]
    }
   ],
   "source": [
    "loss, accuracy, train_op, summary_op, learning_rate, global_step = build(queues, H)\n",
    "data_gen = {}\n",
    "for phase in ['train', 'validate']:\n",
    "    is_train = {'train': True, 'validate': False}[phase]\n",
    "    data_gen[phase] = train_utils.input_data(crop_per_img=1, class_id=class_type, reflection=True,\n",
    "                                             rotation=360, train=is_train, crop_size=im_width)\n",
    "    # Run the generator once to make sure the data is loaded into the memory\n",
    "    # This will take a few minutes\n",
    "    data_gen[phase].next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Global step (1): LR: 0.00010; Train loss 4.66; Train accuracy 2%; Validate loss 290.63; Validate accuracy 14%; Time / image: 0.0ms\n",
      "\n",
      "WARNING:tensorflow:Error encountered when serializing LAYER_NAME_UIDS.\n",
      "Type is unsupported, or the types of the items don't match field type in CollectionDef.\n",
      "'dict' object has no attribute 'name'\n",
      "Global step (101): LR: 0.00010; Train loss 0.49; Train accuracy 69%; Validate loss 9.74; Validate accuracy 0%; Time / image: 78.2ms\n",
      "\n",
      "Global step (201): LR: 0.00010; Train loss 0.70; Train accuracy 58%; Validate loss 10.66; Validate accuracy 14%; Time / image: 74.4ms\n",
      "\n",
      "Global step (301): LR: 0.00010; Train loss 0.56; Train accuracy 62%; Validate loss 8.84; Validate accuracy 16%; Time / image: 74.4ms\n",
      "\n",
      "Global step (401): LR: 0.00010; Train loss 0.49; Train accuracy 66%; Validate loss 2.59; Validate accuracy 21%; Time / image: 74.4ms\n",
      "\n",
      "Global step (501): LR: 0.00010; Train loss 0.49; Train accuracy 65%; Validate loss 4.29; Validate accuracy 23%; Time / image: 74.4ms\n",
      "\n",
      "Global step (601): LR: 0.00010; Train loss 0.89; Train accuracy 43%; Validate loss 7.59; Validate accuracy 11%; Time / image: 74.4ms\n",
      "\n",
      "Global step (701): LR: 0.00010; Train loss 0.34; Train accuracy 75%; Validate loss 4.24; Validate accuracy 22%; Time / image: 74.3ms\n",
      "\n",
      "Global step (801): LR: 0.00010; Train loss 0.50; Train accuracy 64%; Validate loss 2.39; Validate accuracy 20%; Time / image: 74.5ms\n",
      "\n",
      "Global step (901): LR: 0.00010; Train loss 0.35; Train accuracy 73%; Validate loss 4.47; Validate accuracy 19%; Time / image: 74.8ms\n",
      "\n",
      "Global step (1001): LR: 0.00010; Train loss 0.61; Train accuracy 57%; Validate loss 4.89; Validate accuracy 9%; Time / image: 74.9ms\n",
      "\n",
      "WARNING:tensorflow:Error encountered when serializing LAYER_NAME_UIDS.\n",
      "Type is unsupported, or the types of the items don't match field type in CollectionDef.\n",
      "'dict' object has no attribute 'name'\n",
      "Global step (1101): LR: 0.00010; Train loss 0.52; Train accuracy 63%; Validate loss 4.83; Validate accuracy 21%; Time / image: 74.5ms\n",
      "\n",
      "Global step (1201): LR: 0.00010; Train loss 0.24; Train accuracy 81%; Validate loss 4.44; Validate accuracy 18%; Time / image: 74.4ms\n",
      "\n",
      "Global step (1301): LR: 0.00010; Train loss 0.68; Train accuracy 56%; Validate loss 6.05; Validate accuracy 15%; Time / image: 74.3ms\n",
      "\n",
      "Global step (1401): LR: 0.00010; Train loss 1.42; Train accuracy 25%; Validate loss 1.87; Validate accuracy 36%; Time / image: 74.4ms\n",
      "\n",
      "Global step (1501): LR: 0.00010; Train loss 0.74; Train accuracy 50%; Validate loss 2.26; Validate accuracy 35%; Time / image: 74.4ms\n",
      "\n",
      "Global step (1601): LR: 0.00010; Train loss 0.64; Train accuracy 55%; Validate loss 4.12; Validate accuracy 21%; Time / image: 74.4ms\n",
      "\n",
      "Global step (1701): LR: 0.00010; Train loss 0.45; Train accuracy 67%; Validate loss 3.43; Validate accuracy 22%; Time / image: 74.4ms\n",
      "\n"
     ]
    }
   ],
   "source": [
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "coord = tf.train.Coordinator()\n",
    "threads = {}\n",
    "saver = tf.train.Saver(max_to_keep = train_iter / save_iter + 1)\n",
    "\n",
    "with tf.Session(config=config).as_default() as sess:\n",
    "    summary_writer = tf.summary.FileWriter(logdir = summary_path, flush_secs = 10)\n",
    "    summary_writer.add_graph(sess.graph)\n",
    "    for phase in ['train', 'validate']:\n",
    "        \n",
    "        threads[phase] = threading.Thread(\n",
    "            target=enqueue_thread, \n",
    "            args=(sess, data_gen[phase], coord, phase, enqueue_op[phase]))\n",
    "        threads[phase].start()\n",
    "        \n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    start = time.time()\n",
    "    for step in xrange(train_iter):\n",
    "        if step and step % lr_decay_iter == 0:\n",
    "            lr *= 0.1\n",
    "        \n",
    "        if step % print_iter == 0 or step == (train_iter - 1):\n",
    "            dt = (time.time() - start) / batch_size / print_iter\n",
    "            start = time.time()\n",
    "            _, train_loss, train_accuracy, validate_loss, validate_accuracy, summaries = \\\n",
    "            sess.run([train_op, loss['train'], accuracy['train'], loss['validate'], accuracy['validate'], summary_op],\n",
    "                    feed_dict = {learning_rate: lr})\n",
    "            summary_writer.add_summary(summaries, global_step = global_step.eval())\n",
    "            str0 = 'Global step ({0}): LR: {1:0.5f}; '.format(global_step.eval(), lr)\n",
    "            str1 = 'Train loss {0:.2f}; '.format(train_loss)\n",
    "            str2 = 'Train accuracy {}%; '.format(int(100 * train_accuracy))\n",
    "            str3 = 'Validate loss {0:.2f}; '.format(validate_loss)\n",
    "            str4 = 'Validate accuracy {}%; '.format(int(100 * validate_accuracy))\n",
    "            str5 = 'Time / image: {0:0.1f}ms'.format(1000 * dt if step else 0)\n",
    "            print str0 + str1 + str2 + str3 + str4 + str5 + '\\n'\n",
    "        else:\n",
    "            sess.run([train_op, loss['train']], feed_dict = {learning_rate: lr})\n",
    "        \n",
    "        if step % save_iter == 0 or step == (train_iter - 1):\n",
    "            saver.save(sess, ckpt_path, global_step = global_step.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# coord.requst_stop()\n",
    "# coord.join()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}