BATCH_NORMALIZATION_AND_WITHOUT_BATCH_NORMALIZATION.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BATCH_NORMALIZATION_AND_WITHOUT_BATCH_NORMALIZATION.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "_vP5HQo6BXId",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2d917ee6-b5d7-4e47-d26b-ed7fff90c072"
      },
      "source": [
        "from keras.models import Sequential\n",
        "from keras.layers import Conv2D, BatchNormalization\n",
        "from IPython.display import SVG\n",
        "from keras.utils.vis_utils import model_to_dot"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dhTN1dVPCFHk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#WITHOUT BATCHNORMALIZATION LAYER MODEL HAS LESS NUMBER OF PARAMETERS"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lh8lrmOeCF00",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nDV1DcXNBXMi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "9e38a2cf-5cad-458e-a019-7c5f032b97f2"
      },
      "source": [
        "model = Sequential()\n",
        "model.add(Conv2D(512, (3,3), padding='same', activation='relu', input_shape=(256, 256, 3)))\n",
        "model.add(Conv2D(64, (5,5), activation='relu'))\n",
        "model.summary()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d_3 (Conv2D)            (None, 256, 256, 512)     14336     \n",
            "_________________________________________________________________\n",
            "conv2d_4 (Conv2D)            (None, 252, 252, 64)      819264    \n",
            "=================================================================\n",
            "Total params: 833,600\n",
            "Trainable params: 833,600\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QEmVsTv5CXEU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 302
        },
        "outputId": "fe8ba54d-1a20-499a-f71b-9d12448c3bf5"
      },
      "source": [
        "SVG(model_to_dot(model, show_shapes=True).create(prog='dot', format='svg'))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<IPython.core.display.SVG object>"
            ],
            "image/svg+xml": "<svg height=\"211pt\" viewBox=\"0.00 0.00 346.00 211.00\" width=\"346pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 207)\">\n<title>G</title>\n<polygon fill=\"#ffffff\" points=\"-4,4 -4,-207 342,-207 342,4 -4,4\" stroke=\"transparent\"/>\n<!-- 140397378543912 -->\n<g class=\"node\" id=\"node1\">\n<title>140397378543912</title>\n<polygon fill=\"none\" points=\"0,-83.5 0,-129.5 338,-129.5 338,-83.5 0,-83.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"66.5\" y=\"-102.8\">conv2d_3: Conv2D</text>\n<polyline fill=\"none\" points=\"133,-83.5 133,-129.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"162\" y=\"-114.3\">input:</text>\n<polyline fill=\"none\" points=\"133,-106.5 191,-106.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"162\" y=\"-91.3\">output:</text>\n<polyline fill=\"none\" points=\"191,-83.5 191,-129.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"264.5\" y=\"-114.3\">(None, 256, 256, 3)</text>\n<polyline fill=\"none\" points=\"191,-106.5 338,-106.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"264.5\" y=\"-91.3\">(None, 256, 256, 512)</text>\n</g>\n<!-- 140397378545704 -->\n<g class=\"node\" id=\"node2\">\n<title>140397378545704</title>\n<polygon fill=\"none\" points=\"0,-.5 0,-46.5 338,-46.5 338,-.5 0,-.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"66.5\" y=\"-19.8\">conv2d_4: Conv2D</text>\n<polyline fill=\"none\" points=\"133,-.5 133,-46.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"162\" y=\"-31.3\">input:</text>\n<polyline fill=\"none\" points=\"133,-23.5 191,-23.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"162\" y=\"-8.3\">output:</text>\n<polyline fill=\"none\" points=\"191,-.5 191,-46.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"264.5\" y=\"-31.3\">(None, 256, 256, 512)</text>\n<polyline fill=\"none\" points=\"191,-23.5 338,-23.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"264.5\" y=\"-8.3\">(None, 252, 252, 64)</text>\n</g>\n<!-- 140397378543912&#45;&gt;140397378545704 -->\n<g class=\"edge\" id=\"edge2\">\n<title>140397378543912-&gt;140397378545704</title>\n<path d=\"M169,-83.3799C169,-75.1745 169,-65.7679 169,-56.8786\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"172.5001,-56.784 169,-46.784 165.5001,-56.784 172.5001,-56.784\" stroke=\"#000000\"/>\n</g>\n<!-- 140397369906904 -->\n<g class=\"node\" id=\"node3\">\n<title>140397369906904</title>\n<polygon fill=\"none\" points=\"104.5,-166.5 104.5,-202.5 233.5,-202.5 233.5,-166.5 104.5,-166.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"169\" y=\"-180.8\">140397369906904</text>\n</g>\n<!-- 140397369906904&#45;&gt;140397378543912 -->\n<g class=\"edge\" id=\"edge1\">\n<title>140397369906904-&gt;140397378543912</title>\n<path d=\"M169,-166.4092C169,-158.4308 169,-148.795 169,-139.606\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"172.5001,-139.5333 169,-129.5333 165.5001,-139.5334 172.5001,-139.5333\" stroke=\"#000000\"/>\n</g>\n</g>\n</svg>"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bOYmgddUCcMF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#MODEL WITH BATCHNORMALIZATION NUMBER OF PARAMETERS INCREASES"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3MYhk-A-Bci0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "outputId": "d8cecc24-b0a0-4ba7-ddf5-36739ac55d86"
      },
      "source": [
        "model = Sequential()\n",
        "model.add(Conv2D(512, (3,3), padding='same', activation='relu', input_shape=(256, 256, 3)))\n",
        "model.add(BatchNormalization(axis = 3, name = 'Batch_Norm_1'))\n",
        "model.add(Conv2D(64, (5,5), activation='relu'))\n",
        "model.summary()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d_7 (Conv2D)            (None, 256, 256, 512)     14336     \n",
            "_________________________________________________________________\n",
            "Batch_Norm_1 (BatchNormaliza (None, 256, 256, 512)     2048      \n",
            "_________________________________________________________________\n",
            "conv2d_8 (Conv2D)            (None, 252, 252, 64)      819264    \n",
            "=================================================================\n",
            "Total params: 835,648\n",
            "Trainable params: 834,624\n",
            "Non-trainable params: 1,024\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h7Tqv_cABw0p",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 413
        },
        "outputId": "bb684c65-7108-4257-8749-eeb1b3be2649"
      },
      "source": [
        "SVG(model_to_dot(model, show_shapes=True).create(prog='dot', format='svg'))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<IPython.core.display.SVG object>"
            ],
            "image/svg+xml": "<svg height=\"294pt\" viewBox=\"0.00 0.00 441.00 294.00\" width=\"441pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 290)\">\n<title>G</title>\n<polygon fill=\"#ffffff\" points=\"-4,4 -4,-290 437,-290 437,4 -4,4\" stroke=\"transparent\"/>\n<!-- 140397118570168 -->\n<g class=\"node\" id=\"node1\">\n<title>140397118570168</title>\n<polygon fill=\"none\" points=\"47.5,-166.5 47.5,-212.5 385.5,-212.5 385.5,-166.5 47.5,-166.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"114\" y=\"-185.8\">conv2d_7: Conv2D</text>\n<polyline fill=\"none\" points=\"180.5,-166.5 180.5,-212.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"209.5\" y=\"-197.3\">input:</text>\n<polyline fill=\"none\" points=\"180.5,-189.5 238.5,-189.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"209.5\" y=\"-174.3\">output:</text>\n<polyline fill=\"none\" points=\"238.5,-166.5 238.5,-212.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"312\" y=\"-197.3\">(None, 256, 256, 3)</text>\n<polyline fill=\"none\" points=\"238.5,-189.5 385.5,-189.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"312\" y=\"-174.3\">(None, 256, 256, 512)</text>\n</g>\n<!-- 140397118570336 -->\n<g class=\"node\" id=\"node2\">\n<title>140397118570336</title>\n<polygon fill=\"none\" points=\"0,-83.5 0,-129.5 433,-129.5 433,-83.5 0,-83.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"114\" y=\"-102.8\">Batch_Norm_1: BatchNormalization</text>\n<polyline fill=\"none\" points=\"228,-83.5 228,-129.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"257\" y=\"-114.3\">input:</text>\n<polyline fill=\"none\" points=\"228,-106.5 286,-106.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"257\" y=\"-91.3\">output:</text>\n<polyline fill=\"none\" points=\"286,-83.5 286,-129.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"359.5\" y=\"-114.3\">(None, 256, 256, 512)</text>\n<polyline fill=\"none\" points=\"286,-106.5 433,-106.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"359.5\" y=\"-91.3\">(None, 256, 256, 512)</text>\n</g>\n<!-- 140397118570168&#45;&gt;140397118570336 -->\n<g class=\"edge\" id=\"edge2\">\n<title>140397118570168-&gt;140397118570336</title>\n<path d=\"M216.5,-166.3799C216.5,-158.1745 216.5,-148.7679 216.5,-139.8786\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"220.0001,-139.784 216.5,-129.784 213.0001,-139.784 220.0001,-139.784\" stroke=\"#000000\"/>\n</g>\n<!-- 140397118657592 -->\n<g class=\"node\" id=\"node3\">\n<title>140397118657592</title>\n<polygon fill=\"none\" points=\"47.5,-.5 47.5,-46.5 385.5,-46.5 385.5,-.5 47.5,-.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"114\" y=\"-19.8\">conv2d_8: Conv2D</text>\n<polyline fill=\"none\" points=\"180.5,-.5 180.5,-46.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"209.5\" y=\"-31.3\">input:</text>\n<polyline fill=\"none\" points=\"180.5,-23.5 238.5,-23.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"209.5\" y=\"-8.3\">output:</text>\n<polyline fill=\"none\" points=\"238.5,-.5 238.5,-46.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"312\" y=\"-31.3\">(None, 256, 256, 512)</text>\n<polyline fill=\"none\" points=\"238.5,-23.5 385.5,-23.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"312\" y=\"-8.3\">(None, 252, 252, 64)</text>\n</g>\n<!-- 140397118570336&#45;&gt;140397118657592 -->\n<g class=\"edge\" id=\"edge3\">\n<title>140397118570336-&gt;140397118657592</title>\n<path d=\"M216.5,-83.3799C216.5,-75.1745 216.5,-65.7679 216.5,-56.8786\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"220.0001,-56.784 216.5,-46.784 213.0001,-56.784 220.0001,-56.784\" stroke=\"#000000\"/>\n</g>\n<!-- 140397118657872 -->\n<g class=\"node\" id=\"node4\">\n<title>140397118657872</title>\n<polygon fill=\"none\" points=\"152.5,-249.5 152.5,-285.5 280.5,-285.5 280.5,-249.5 152.5,-249.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"216.5\" y=\"-263.8\">140397118657872</text>\n</g>\n<!-- 140397118657872&#45;&gt;140397118570168 -->\n<g class=\"edge\" id=\"edge1\">\n<title>140397118657872-&gt;140397118570168</title>\n<path d=\"M216.5,-249.4092C216.5,-241.4308 216.5,-231.795 216.5,-222.606\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"220.0001,-222.5333 216.5,-212.5333 213.0001,-222.5334 220.0001,-222.5333\" stroke=\"#000000\"/>\n</g>\n</g>\n</svg>"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T-95jnQpCoAv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}