cifar10bn-dropout.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import cifar\n",
    "tf.logging.set_verbosity(tf.logging.WARN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download and extract the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cifar.prepare_cifar_10()\n",
    "cifar10_labels = cifar.cifar10_labels()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(features, labels, mode, params):\n",
    "    net = features['images']\n",
    "\n",
    "    for filt, kern, stride in zip(params['filters'], params['kern'], params['strides']):\n",
    "        net = tf.layers.batch_normalization(net,\n",
    "                                            momentum=params['momentum'],\n",
    "                                            training=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "        net = tf.layers.conv2d(net, filt,\n",
    "                               kern, stride, activation=tf.nn.relu)\n",
    "\n",
    "    net = tf.layers.flatten(net)\n",
    "    net = tf.layers.batch_normalization(net,\n",
    "                                        momentum=params['momentum'],\n",
    "                                        training=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "\n",
    "    for units, drop in zip(params['dense'], params['drop']):\n",
    "        net = tf.layers.dense(net, units, activation=tf.nn.relu)\n",
    "\n",
    "        if params['with_bn']:\n",
    "            net = tf.layers.batch_normalization(net,\n",
    "                                                momentum=params['momentum'],\n",
    "                                                training=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "        net = tf.layers.dropout(net, drop, training=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "\n",
    "    logits = tf.layers.dense(net, params['n_classes'])\n",
    "    cls = tf.argmax(logits, -1)\n",
    "\n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(mode, predictions={\n",
    "            \"class\": cls,\n",
    "            \"score\": tf.nn.softmax(logits)\n",
    "        })\n",
    "\n",
    "    loss = tf.losses.sparse_softmax_cross_entropy(labels, logits)\n",
    "\n",
    "    if mode == tf.estimator.ModeKeys.EVAL:\n",
    "        return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops={\n",
    "            \"accuracy\": tf.metrics.accuracy(labels, cls)\n",
    "        })\n",
    "    adam = tf.train.AdamOptimizer()\n",
    "\n",
    "    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "    with tf.control_dependencies(update_ops):\n",
    "        opt = adam.minimize(loss, global_step=tf.train.get_global_step())\n",
    "\n",
    "    return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=opt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inp_fn():\n",
    "    return tf.data.Dataset.from_generator(cifar.cifar10_train, ({\"images\": tf.float32}, tf.int64),\n",
    "                                          ({\"images\": tf.TensorShape([None, 32, 32, 3])}, tf.TensorShape(None)))\n",
    "\n",
    "\n",
    "def test_inp_fn():\n",
    "    return tf.data.Dataset.from_generator(cifar.cifar10_test, ({\"images\": tf.float32}, tf.int64),\n",
    "                                          ({\"images\": tf.TensorShape([None, 32, 32, 3])}, tf.TensorShape(None)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "hparams = {\n",
    "    \"filters\": [30, 50, 60],\n",
    "    \"kern\": [[3, 3]]*3,\n",
    "    \"strides\": [[2, 2], [1, 1], [1, 1]],\n",
    "    \"dense\": [3500, 700],\n",
    "    \"drop\": [0.4, 0.5],\n",
    "    \"n_classes\": 10,\n",
    "    \"with_bn\": True,\n",
    "    \"momentum\": 0.75,\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dropout with BN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "drwbn = tf.estimator.Estimator(model, 'drwbn-ckpts', config=tf.estimator.RunConfig(save_summary_steps=2),\n",
    "                               params=hparams)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Start Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_ipython().system_raw(\"start tensorboard --logdir drwbn-ckpts\")  # Windows\n",
    "# get_ipython().system_raw(\"tensorboard --logdir drop-ckpts &\") #Linux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'accuracy': 0.5226, 'loss': 1.3642848, 'global_step': 20}\n",
      "{'accuracy': 0.5917, 'loss': 1.1845695, 'global_step': 40}\n",
      "{'accuracy': 0.6246, 'loss': 1.1305641, 'global_step': 60}\n",
      "{'accuracy': 0.6149, 'loss': 1.2645891, 'global_step': 80}\n",
      "{'accuracy': 0.6093, 'loss': 1.4570963, 'global_step': 100}\n",
      "{'accuracy': 0.6105, 'loss': 1.6007249, 'global_step': 120}\n",
      "{'accuracy': 0.6072, 'loss': 1.660339, 'global_step': 140}\n",
      "{'accuracy': 0.6148, 'loss': 1.7931831, 'global_step': 160}\n",
      "{'accuracy': 0.6144, 'loss': 1.7849233, 'global_step': 180}\n",
      "{'accuracy': 0.6165, 'loss': 1.825989, 'global_step': 200}\n",
      "{'accuracy': 0.6178, 'loss': 1.8372722, 'global_step': 220}\n",
      "{'accuracy': 0.627, 'loss': 1.8474933, 'global_step': 240}\n",
      "{'accuracy': 0.6287, 'loss': 1.8738902, 'global_step': 260}\n",
      "{'accuracy': 0.6262, 'loss': 1.8911006, 'global_step': 280}\n",
      "{'accuracy': 0.6263, 'loss': 1.9236542, 'global_step': 300}\n",
      "{'accuracy': 0.6279, 'loss': 1.9401443, 'global_step': 320}\n",
      "{'accuracy': 0.6273, 'loss': 1.950819, 'global_step': 340}\n",
      "{'accuracy': 0.6282, 'loss': 1.9540635, 'global_step': 360}\n",
      "{'accuracy': 0.629, 'loss': 1.9550527, 'global_step': 380}\n",
      "{'accuracy': 0.6295, 'loss': 1.983186, 'global_step': 400}\n",
      "{'accuracy': 0.6312, 'loss': 2.0035515, 'global_step': 420}\n",
      "{'accuracy': 0.6273, 'loss': 2.017928, 'global_step': 440}\n",
      "{'accuracy': 0.6295, 'loss': 2.0265088, 'global_step': 460}\n",
      "{'accuracy': 0.6294, 'loss': 2.024135, 'global_step': 480}\n",
      "{'accuracy': 0.6296, 'loss': 2.0443437, 'global_step': 500}\n",
      "{'accuracy': 0.629, 'loss': 2.048645, 'global_step': 520}\n",
      "{'accuracy': 0.6304, 'loss': 2.064659, 'global_step': 540}\n",
      "{'accuracy': 0.6292, 'loss': 2.0688968, 'global_step': 560}\n",
      "{'accuracy': 0.6291, 'loss': 2.0785952, 'global_step': 580}\n",
      "{'accuracy': 0.6294, 'loss': 2.0939922, 'global_step': 600}\n"
     ]
    }
   ],
   "source": [
    "for i in range(30):\n",
    "    drwbn.train(inp_fn)\n",
    "    print(drwbn.evaluate(test_inp_fn))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dropout without BN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "hparams['with_bn'] = False\n",
    "drwobn = tf.estimator.Estimator(model, 'drwobn-ckpts', config=tf.estimator.RunConfig(save_summary_steps=2),\n",
    "                                params=hparams)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Start Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_ipython().system_raw(\"start tensorboard --logdir drwobn-ckpts --port 6007\")  # Windows\n",
    "# get_ipython().system_raw(\"tensorboard --logdir drwobn-ckpts &\") #Linux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'accuracy': 0.4223, 'loss': 1.6613123, 'global_step': 20}\n",
      "{'accuracy': 0.5224, 'loss': 1.3384712, 'global_step': 40}\n",
      "{'accuracy': 0.5726, 'loss': 1.208832, 'global_step': 60}\n",
      "{'accuracy': 0.6025, 'loss': 1.1299886, 'global_step': 80}\n",
      "{'accuracy': 0.6201, 'loss': 1.1092131, 'global_step': 100}\n",
      "{'accuracy': 0.6266, 'loss': 1.1479524, 'global_step': 120}\n",
      "{'accuracy': 0.6286, 'loss': 1.1950808, 'global_step': 140}\n",
      "{'accuracy': 0.6219, 'loss': 1.2852988, 'global_step': 160}\n",
      "{'accuracy': 0.6252, 'loss': 1.3611405, 'global_step': 180}\n",
      "{'accuracy': 0.6312, 'loss': 1.4825554, 'global_step': 200}\n",
      "{'accuracy': 0.6028, 'loss': 1.7469711, 'global_step': 220}\n",
      "{'accuracy': 0.6209, 'loss': 1.6120105, 'global_step': 240}\n",
      "{'accuracy': 0.6206, 'loss': 1.7405858, 'global_step': 260}\n",
      "{'accuracy': 0.621, 'loss': 1.8695172, 'global_step': 280}\n",
      "{'accuracy': 0.6173, 'loss': 1.9884601, 'global_step': 300}\n",
      "{'accuracy': 0.6183, 'loss': 2.04527, 'global_step': 320}\n",
      "{'accuracy': 0.6216, 'loss': 1.9960701, 'global_step': 340}\n",
      "{'accuracy': 0.6134, 'loss': 2.0442882, 'global_step': 360}\n",
      "{'accuracy': 0.6224, 'loss': 2.1645799, 'global_step': 380}\n",
      "{'accuracy': 0.6285, 'loss': 2.1302357, 'global_step': 400}\n",
      "{'accuracy': 0.6271, 'loss': 2.229645, 'global_step': 420}\n",
      "{'accuracy': 0.6297, 'loss': 2.2214031, 'global_step': 440}\n",
      "{'accuracy': 0.6308, 'loss': 2.302416, 'global_step': 460}\n",
      "{'accuracy': 0.6309, 'loss': 2.3133702, 'global_step': 480}\n",
      "{'accuracy': 0.6325, 'loss': 2.3474002, 'global_step': 500}\n",
      "{'accuracy': 0.6341, 'loss': 2.3391428, 'global_step': 520}\n",
      "{'accuracy': 0.6333, 'loss': 2.3756914, 'global_step': 540}\n",
      "{'accuracy': 0.6302, 'loss': 2.435812, 'global_step': 560}\n",
      "{'accuracy': 0.6272, 'loss': 2.4602158, 'global_step': 580}\n",
      "{'accuracy': 0.6321, 'loss': 2.4804482, 'global_step': 600}\n"
     ]
    }
   ],
   "source": [
    "for i in range(30):\n",
    "    drwobn.train(inp_fn)\n",
    "    print(drwobn.evaluate(test_inp_fn))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results\n",
    "#### BN after Dropout\n",
    "![Graph (BN after dropout)](images/drwbn.png)\n",
    "#### BN not after Dropout\n",
    "![Graph (BN not after dropout)](images/drwobn.png)\n",
    "\n",
    "As we can see, accuracy is more stable when BN is not applied after dropout. Also, accuracy for the version without BN after Dropout is greater than that having BN after dropout."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read more about Batch Normalization and dropout in the [Disharmony between Dropout and Batch Normalization](https://arxiv.org/abs/1801.05134). You can try running this notebook in your local system or in [Google Colab](https://colab.research.google.com/drive/1D9_ltbQaT7yCpnEyg8il1jpRrxEQEWfx)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}