gpu_forward.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from resnet import resnet18, resnet34, resnet50, resnet101, resnet152\n",
        "from resnet import resnet18_bottleneck, resnet34_bottleneck, resnet50_bottleneck, resnet101_bottleneck, resnet152_bottleneck\n",
        "import numpy as np\n",
        "import torch\n",
        "from torch import nn"
      ],
      "metadata": {
        "id": "YbfoDIkIrRnh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Profiling resnets on GPU - Forward\n",
        "\n",
        "Here we profile the forward passes only of various resnets executed on GPUs. It's expected that the execution times are significantly lower than with CPUs."
      ],
      "metadata": {
        "id": "7NY3HZk5rahk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda\")"
      ],
      "metadata": {
        "id": "h-sxF1U25rkj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.profiler import profile, record_function, ProfilerActivity\n",
        "\n",
        "def profile_model(model, inputs, sort_by_self=True):\n",
        "    sort_by_string = \"self_cuda_time_total\" if sort_by_self else \"cuda_time_total\"\n",
        "    model(inputs)   # Warmup\n",
        "\n",
        "    with profile(activities=[\n",
        "        ProfilerActivity.CPU, ProfilerActivity.CUDA], record_shapes=True) as prof:\n",
        "        with record_function(\"model_inference\"):\n",
        "            model(inputs)\n",
        "\n",
        "    print(prof.key_averages().table(sort_by=sort_by_string, row_limit=15))"
      ],
      "metadata": {
        "id": "X9reGy3ArSAg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### ResNet-18"
      ],
      "metadata": {
        "id": "i7EkKV_f6y9D"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet18().to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8CEg6XA6xBGN",
        "outputId": "10735626-7d71-42d7-c053-e904263ac47c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution        12.91%       6.082ms        13.84%       6.518ms     325.900us      26.386ms        68.36%      29.420ms       1.471ms            20  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us      11.150ms        28.89%      11.150ms     857.692us            13  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       7.581ms        19.64%       7.581ms       1.895ms             4  \n",
            "                                 aten::cudnn_batch_norm         5.99%       2.820ms         7.64%       3.596ms     179.800us       4.895ms        12.68%       4.895ms     244.750us            20  \n",
            "                                             aten::add_         1.31%     616.000us         2.15%       1.012ms      21.083us       4.349ms        11.27%       5.713ms     119.021us            48  \n",
            "                                       cudaLaunchKernel         2.47%       1.162ms         2.47%       1.162ms       7.497us       3.507ms         9.09%       3.507ms      22.626us           155  \n",
            "void cudnn::winograd_nonfused::winogradForwardData4x...         0.00%       0.000us         0.00%       0.000us       0.000us       3.196ms         8.28%       3.196ms     245.846us            13  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       3.096ms         8.02%       3.096ms     154.800us            20  \n",
            "void cudnn::winograd_nonfused::winogradForwardOutput...         0.00%       0.000us         0.00%       0.000us       0.000us       2.765ms         7.16%       2.765ms     212.692us            13  \n",
            "                                        aten::clamp_min         0.46%     216.000us         0.70%     329.000us      20.562us       1.590ms         4.12%       1.604ms     100.250us            16  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       1.590ms         4.12%       1.590ms      99.375us            16  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.554ms         4.03%       1.554ms       1.554ms             1  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.551ms         4.02%       1.551ms     387.750us             4  \n",
            "                          aten::max_pool2d_with_indices         0.07%      33.000us         0.10%      45.000us      45.000us       1.257ms         3.26%       1.651ms       1.651ms             1  \n",
            "void at::native::(anonymous namespace)::max_pool_for...         0.00%       0.000us         0.00%       0.000us       0.000us       1.257ms         3.26%       1.257ms       1.257ms             1  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 47.095ms\n",
            "Self CUDA time total: 38.596ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### ResNet-34"
      ],
      "metadata": {
        "id": "GkLNceov61AQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet34().to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dofW0OrV6lOY",
        "outputId": "43003c00-304e-49bd-9431-70a227c88e1f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution         2.52%       2.983ms         3.28%       3.886ms     107.944us      47.194ms        71.70%      47.194ms       1.311ms            36  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us      25.083ms        38.11%      25.083ms     864.931us            29  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       7.580ms        11.52%       7.580ms       1.895ms             4  \n",
            "                                 aten::cudnn_batch_norm         1.68%       1.995ms         3.05%       3.613ms     100.361us       7.322ms        11.12%       7.322ms     203.389us            36  \n",
            "                                             aten::add_         1.07%       1.264ms         1.63%       1.932ms      21.955us       6.989ms        10.62%       6.989ms      79.420us            88  \n",
            "void cudnn::winograd_nonfused::winogradForwardData4x...         0.00%       0.000us         0.00%       0.000us       0.000us       6.237ms         9.48%       6.237ms     215.069us            29  \n",
            "void cudnn::winograd_nonfused::winogradForwardOutput...         0.00%       0.000us         0.00%       0.000us       0.000us       5.505ms         8.36%       5.505ms     189.828us            29  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       4.690ms         7.13%       4.690ms     130.278us            36  \n",
            "                                        aten::clamp_min         0.45%     532.000us         0.69%     823.000us      25.719us       2.941ms         4.47%       2.941ms      91.906us            32  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       2.941ms         4.47%       2.941ms      91.906us            32  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       2.314ms         3.52%       2.314ms     385.667us             6  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       2.123ms         3.23%       2.123ms     132.688us            16  \n",
            "void cudnn::winograd_nonfused::winogradForwardFilter...         0.00%       0.000us         0.00%       0.000us       0.000us       2.097ms         3.19%       2.097ms      72.310us            29  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.746ms         2.65%       1.746ms     194.000us             9  \n",
            "void cudnn::bn_fw_tr_1C11_singleread<float, 512, tru...         0.00%       0.000us         0.00%       0.000us       0.000us       1.715ms         2.61%       1.715ms      85.750us            20  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 118.444ms\n",
            "Self CUDA time total: 65.822ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### ResNet-50"
      ],
      "metadata": {
        "id": "U7WzEVeu7XYg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet50().to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mJvFqtpZ7cb8",
        "outputId": "3731af55-5434-4614-f3be-e21f74096946"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution         5.25%       5.781ms         6.31%       6.947ms     192.972us      46.319ms        71.44%      46.319ms       1.287ms            36  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us      24.462ms        37.73%      24.462ms     843.517us            29  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       7.388ms        11.39%       7.388ms       1.847ms             4  \n",
            "                                 aten::cudnn_batch_norm         2.06%       2.272ms         3.88%       4.270ms     118.611us       7.306ms        11.27%       7.306ms     202.944us            36  \n",
            "                                             aten::add_         1.39%       1.529ms         2.19%       2.413ms      27.420us       6.942ms        10.71%       6.942ms      78.886us            88  \n",
            "void cudnn::winograd_nonfused::winogradForwardData4x...         0.00%       0.000us         0.00%       0.000us       0.000us       6.262ms         9.66%       6.262ms     215.931us            29  \n",
            "void cudnn::winograd_nonfused::winogradForwardOutput...         0.00%       0.000us         0.00%       0.000us       0.000us       5.462ms         8.42%       5.462ms     188.345us            29  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       4.643ms         7.16%       4.643ms     128.972us            36  \n",
            "                                        aten::clamp_min         0.59%     649.000us         0.91%       1.004ms      31.375us       2.925ms         4.51%       2.925ms      91.406us            32  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       2.925ms         4.51%       2.925ms      91.406us            32  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       2.315ms         3.57%       2.315ms     385.833us             6  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       2.127ms         3.28%       2.127ms     132.938us            16  \n",
            "void cudnn::winograd_nonfused::winogradForwardFilter...         0.00%       0.000us         0.00%       0.000us       0.000us       2.065ms         3.18%       2.065ms      71.207us            29  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.743ms         2.69%       1.743ms     193.667us             9  \n",
            "void cudnn::bn_fw_tr_1C11_singleread<float, 512, tru...         0.00%       0.000us         0.00%       0.000us       0.000us       1.698ms         2.62%       1.698ms      84.900us            20  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 110.076ms\n",
            "Self CUDA time total: 64.840ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### ResNet-101"
      ],
      "metadata": {
        "id": "QeR9wZSL7xDO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet101().to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ia9JJ_Cr70fz",
        "outputId": "9a2f9018-de57-4e18-8edc-96263ef3b34a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution         2.69%       4.549ms         4.13%       6.998ms      99.971us      62.125ms        71.37%      62.125ms     887.500us            70  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us      32.372ms        37.19%      32.372ms     513.841us            63  \n",
            "void cudnn::winograd_nonfused::winogradForwardData4x...         0.00%       0.000us         0.00%       0.000us       0.000us      10.577ms        12.15%      10.577ms     167.889us            63  \n",
            "                                 aten::cudnn_batch_norm         2.02%       3.418ms         3.49%       5.908ms      84.400us       9.546ms        10.97%       9.546ms     136.371us            70  \n",
            "                                             aten::add_         1.28%       2.159ms         1.93%       3.262ms      18.855us       9.481ms        10.89%       9.481ms      54.803us           173  \n",
            "void cudnn::winograd_nonfused::winogradForwardOutput...         0.00%       0.000us         0.00%       0.000us       0.000us       9.326ms        10.71%       9.326ms     148.032us            63  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       5.832ms         6.70%       5.832ms      83.314us            70  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       5.251ms         6.03%       5.251ms       1.313ms             4  \n",
            "                                        aten::clamp_min         0.55%     924.000us         0.85%       1.439ms      21.803us       4.723ms         5.43%       4.723ms      71.561us            66  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       4.723ms         5.43%       4.723ms      71.561us            66  \n",
            "void cudnn::winograd_nonfused::winogradForwardFilter...         0.00%       0.000us         0.00%       0.000us       0.000us       4.107ms         4.72%       4.107ms      65.190us            63  \n",
            "void cudnn::bn_fw_tr_1C11_singleread<float, 512, tru...         0.00%       0.000us         0.00%       0.000us       0.000us       4.069ms         4.67%       4.069ms      75.352us            54  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       3.412ms         3.92%       3.412ms     103.394us            33  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       2.255ms         2.59%       2.255ms     375.833us             6  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.686ms         1.94%       1.686ms     187.333us             9  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 169.307ms\n",
            "Self CUDA time total: 87.051ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### ResNet-152"
      ],
      "metadata": {
        "id": "TjPgMGvh8YFf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet152().to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZIEQGvxI8YPU",
        "outputId": "2929256a-c1cb-497c-b263-b853336eea3c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution         3.37%       7.075ms        14.74%      30.990ms     297.981us      88.257ms        72.09%      88.257ms     848.625us           104  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us      47.984ms        39.19%      47.984ms     494.680us            97  \n",
            "void cudnn::winograd_nonfused::winogradForwardData4x...         0.00%       0.000us         0.00%       0.000us       0.000us      15.678ms        12.81%      15.678ms     161.629us            97  \n",
            "void cudnn::winograd_nonfused::winogradForwardOutput...         0.00%       0.000us         0.00%       0.000us       0.000us      14.071ms        11.49%      14.071ms     145.062us            97  \n",
            "                                             aten::add_         1.93%       4.061ms         3.61%       7.582ms      29.388us      13.339ms        10.90%      13.339ms      51.702us           258  \n",
            "                                 aten::cudnn_batch_norm         2.48%       5.215ms         4.38%       9.207ms      88.529us      12.959ms        10.59%      12.959ms     124.606us           104  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       7.986ms         6.52%       7.986ms      76.788us           104  \n",
            "                                        aten::clamp_min         0.73%       1.539ms         1.81%       3.803ms      38.030us       6.986ms         5.71%       6.986ms      69.860us           100  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       6.986ms         5.71%       6.986ms      69.860us           100  \n",
            "void cudnn::bn_fw_tr_1C11_singleread<float, 512, tru...         0.00%       0.000us         0.00%       0.000us       0.000us       6.080ms         4.97%       6.080ms      76.000us            80  \n",
            "void cudnn::winograd_nonfused::winogradForwardFilter...         0.00%       0.000us         0.00%       0.000us       0.000us       5.775ms         4.72%       5.775ms      59.536us            97  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       5.036ms         4.11%       5.036ms     100.720us            50  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       4.268ms         3.49%       4.268ms       1.067ms             4  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       3.114ms         2.54%       3.114ms     183.176us            17  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       2.255ms         1.84%       2.255ms     375.833us             6  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 210.221ms\n",
            "Self CUDA time total: 122.424ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Initially, we observe that the execution times of all plain resnets have been significantly reduced compared to any of the CPU resnets, even when the CPU networks are implemented with bottleneck layers. As a comparison, the largest GPU network ResNet-152 with plain layers execute in a total of 210ms CPU time and 122ms GPU time, which is much lower than even the ResNet-18 with bottleneck layers on CPU which executed in ~1.132 seconds (see CPU forward notebook)."
      ],
      "metadata": {
        "id": "Czbg9YHvEvZf"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Bottleneck ResNets\n",
        "\n",
        "### Bottleneck RN18"
      ],
      "metadata": {
        "id": "epH_zb6D9Yw5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet18_bottleneck().to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2RH-wXH2-BE1",
        "outputId": "1d326b53-ec40-4900-e067-f86250d25295"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution        13.87%       2.118ms        23.32%       3.560ms     127.143us       4.005ms        30.48%       4.005ms     143.036us            28  \n",
            "                                 aten::cudnn_batch_norm         9.20%       1.404ms        16.03%       2.447ms      87.393us       3.730ms        28.39%       3.730ms     133.214us            28  \n",
            "                                             aten::add_         5.22%     797.000us         8.19%       1.250ms      19.531us       3.428ms        26.09%       3.428ms      53.562us            64  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       2.182ms        16.61%       2.182ms      77.929us            28  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.525ms        11.61%       1.525ms       1.525ms             1  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       1.189ms         9.05%       1.189ms       1.189ms             1  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       1.185ms         9.02%       1.185ms     148.125us             8  \n",
            "                                        aten::clamp_min         2.23%     340.000us         3.46%     528.000us      22.000us       1.158ms         8.81%       1.158ms      48.250us            24  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       1.158ms         8.81%       1.158ms      48.250us            24  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.136ms         8.65%       1.136ms     189.333us             6  \n",
            "                          aten::max_pool2d_with_indices         0.25%      38.000us         0.33%      50.000us      50.000us     766.000us         5.83%     766.000us     766.000us             1  \n",
            "void at::native::(anonymous namespace)::max_pool_for...         0.00%       0.000us         0.00%       0.000us       0.000us     766.000us         5.83%     766.000us     766.000us             1  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us     662.000us         5.04%     662.000us      94.571us             7  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us     584.000us         4.44%     584.000us      97.333us             6  \n",
            "           cudnn_volta_scudnn_128x128_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us     572.000us         4.35%     572.000us      81.714us             7  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 15.266ms\n",
            "Self CUDA time total: 13.140ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Bottleneck RN34"
      ],
      "metadata": {
        "id": "samoRqnw-GEF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet34_bottleneck().to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o-C51Qne-LMM",
        "outputId": "cc45941f-0a29-4324-80f6-fddf43c21085"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution        13.66%       3.445ms        21.82%       5.502ms     105.808us       6.270ms        31.66%       6.270ms     120.577us            52  \n",
            "                                             aten::add_         7.04%       1.776ms        11.16%       2.815ms      23.458us       5.381ms        27.17%       5.381ms      44.842us           120  \n",
            "                                 aten::cudnn_batch_norm        11.52%       2.906ms        22.10%       5.574ms     107.192us       5.239ms        26.45%       5.239ms     100.750us            52  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       3.089ms        15.60%       3.089ms      59.404us            52  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       2.166ms        10.94%       2.166ms     135.375us            16  \n",
            "                                        aten::clamp_min         3.41%     859.000us         5.28%       1.332ms      27.750us       2.089ms        10.55%       2.089ms      43.521us            48  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       2.089ms        10.55%       2.089ms      43.521us            48  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.707ms         8.62%       1.707ms     189.667us             9  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.523ms         7.69%       1.523ms       1.523ms             1  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       1.189ms         6.00%       1.189ms       1.189ms             1  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.160ms         5.86%       1.160ms      89.231us            13  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us       1.091ms         5.51%       1.091ms      83.923us            13  \n",
            "           cudnn_volta_scudnn_128x128_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us     942.000us         4.76%     942.000us      78.500us            12  \n",
            "             cudnn_volta_scudnn_128x32_relu_small_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us     912.000us         4.61%     912.000us     130.286us             7  \n",
            "void cudnn::bn_fw_tr_1C11_singleread<float, 512, tru...         0.00%       0.000us         0.00%       0.000us       0.000us     849.000us         4.29%     849.000us      29.276us            29  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 25.220ms\n",
            "Self CUDA time total: 19.804ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Bottleneck RN50"
      ],
      "metadata": {
        "id": "aF0qa01-_l_H"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet50_bottleneck().to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)   # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P117e4Ro_pBL",
        "outputId": "c4501451-a871-4ba6-8dcd-a446f2ea0e9e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution        13.78%       3.710ms        20.58%       5.540ms     106.538us       6.309ms        31.78%       6.309ms     121.327us            52  \n",
            "                                             aten::add_         6.49%       1.748ms         9.76%       2.626ms      21.883us       5.383ms        27.11%       5.383ms      44.858us           120  \n",
            "                                 aten::cudnn_batch_norm         9.92%       2.670ms        17.62%       4.742ms      91.192us       5.262ms        26.50%       5.262ms     101.192us            52  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       3.096ms        15.59%       3.096ms      59.538us            52  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       2.156ms        10.86%       2.156ms     134.750us            16  \n",
            "                                        aten::clamp_min         2.72%     731.000us         4.09%       1.102ms      22.958us       2.077ms        10.46%       2.077ms      43.271us            48  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       2.077ms        10.46%       2.077ms      43.271us            48  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.702ms         8.57%       1.702ms     189.111us             9  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.533ms         7.72%       1.533ms       1.533ms             1  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       1.192ms         6.00%       1.192ms       1.192ms             1  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.159ms         5.84%       1.159ms      89.154us            13  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us       1.100ms         5.54%       1.100ms      84.615us            13  \n",
            "           cudnn_volta_scudnn_128x128_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us     953.000us         4.80%     953.000us      79.417us            12  \n",
            "             cudnn_volta_scudnn_128x32_relu_small_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us     919.000us         4.63%     919.000us     131.286us             7  \n",
            "void cudnn::bn_fw_tr_1C11_singleread<float, 512, tru...         0.00%       0.000us         0.00%       0.000us       0.000us     868.000us         4.37%     868.000us      29.931us            29  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 26.918ms\n",
            "Self CUDA time total: 19.855ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Bottleneck RN 101"
      ],
      "metadata": {
        "id": "78Ao0HbC_1Kc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet101_bottleneck().to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2ztL2yYRApgJ",
        "outputId": "23fd7b89-17c8-4300-8c76-3a302e4b71a5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution        12.35%       7.389ms        18.14%      10.856ms     105.398us      12.212ms        38.14%      12.212ms     118.563us           103  \n",
            "                                             aten::add_         7.35%       4.401ms        11.51%       6.890ms      28.828us       8.334ms        26.03%       8.334ms      34.870us           239  \n",
            "                                 aten::cudnn_batch_norm        11.54%       6.905ms        22.36%      13.381ms     129.913us       7.342ms        22.93%       7.342ms      71.282us           103  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       4.554ms        14.22%       4.554ms      44.214us           103  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       3.439ms        10.74%       3.439ms     104.212us            33  \n",
            "                                        aten::clamp_min         3.68%       2.201ms         5.58%       3.337ms      33.707us       3.281ms        10.25%       3.281ms      33.141us            99  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       3.281ms        10.25%       3.281ms      33.141us            99  \n",
            "void cudnn::bn_fw_tr_1C11_singleread<float, 512, tru...         0.00%       0.000us         0.00%       0.000us       0.000us       2.941ms         9.19%       2.941ms      36.763us            80  \n",
            "           cudnn_volta_scudnn_128x128_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       2.714ms         8.48%       2.714ms      93.586us            29  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us       2.099ms         6.56%       2.099ms      69.967us            30  \n",
            "                                   volta_sgemm_64x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us       2.083ms         6.51%       2.083ms      83.320us            25  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.702ms         5.32%       1.702ms     189.111us             9  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.525ms         4.76%       1.525ms       1.525ms             1  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       1.196ms         3.74%       1.196ms       1.196ms             1  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.174ms         3.67%       1.174ms      90.308us            13  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 59.853ms\n",
            "Self CUDA time total: 32.016ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Bottleneck RN152"
      ],
      "metadata": {
        "id": "P87uNj0gBRxV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet152_bottleneck().to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q0uY_snSGCUP",
        "outputId": "de32cfd5-b673-46ed-f5e2-b5cb2252cc1a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution        12.74%      10.334ms        18.87%      15.308ms      99.403us      24.380ms        45.01%      24.380ms     158.312us           154  \n",
            "                                             aten::add_         7.36%       5.966ms        11.87%       9.626ms      26.888us      12.776ms        23.59%      12.776ms      35.687us           358  \n",
            "                                 aten::cudnn_batch_norm        11.78%       9.552ms        23.23%      18.839ms     122.331us      10.855ms        20.04%      10.855ms      70.487us           154  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       7.108ms        13.12%       7.108ms      46.156us           154  \n",
            "void cudnn::bn_fw_tr_1C11_singleread<float, 512, tru...         0.00%       0.000us         0.00%       0.000us       0.000us       5.056ms         9.33%       5.056ms      42.487us           119  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       5.049ms         9.32%       5.049ms     100.980us            50  \n",
            "                                        aten::clamp_min         3.68%       2.981ms         5.61%       4.550ms      30.333us       4.870ms         8.99%       4.870ms      32.467us           150  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       4.870ms         8.99%       4.870ms      32.467us           150  \n",
            "           cudnn_volta_scudnn_128x128_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       4.790ms         8.84%       4.790ms     114.048us            42  \n",
            "                                   volta_sgemm_64x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us       4.245ms         7.84%       4.245ms     111.711us            38  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us       4.103ms         7.58%       4.103ms      87.298us            47  \n",
            "             cudnn_volta_scudnn_128x32_relu_small_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       3.025ms         5.59%       3.025ms     275.000us            11  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       2.838ms         5.24%       2.838ms       2.838ms             1  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       2.383ms         4.40%       2.383ms      95.320us            25  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.876ms         3.46%       1.876ms     208.444us             9  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 81.113ms\n",
            "Self CUDA time total: 54.162ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "With bottleneck layers and GPU, we achieve an even greater efficiency increase, where even the largest ResNet-152 runs in 81ms of CPU time and 54ms of GPU time. Combining GPU acceleration with bottleneck layers is an excellent configuration to achieve high performing networks. By using a smaller network, such as ResNet-50, we can trade some accuracy for tremendous speedups, where the bottleneck implementation of ResNet-50 achieved 26.9ms of CPU time and 19.9ms of CUDA time. The bottleneck layer increases efficiency resultng in fast GPU runtimes."
      ],
      "metadata": {
        "id": "Rgcwm38YFRnp"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Large ResNets\n",
        "\n",
        "Huge gains are observed, even with large-mode resnets where the number of internal channels have been quadrupled:\n"
      ],
      "metadata": {
        "id": "UBZ6cAD9GKL6"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Large ResNet-50"
      ],
      "metadata": {
        "id": "vQQNINALGx_j"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet50_bottleneck(large_mode=True).to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QFXb4EhbGK-Y",
        "outputId": "cf8b8eb6-6137-4fac-fe72-d3d0d36f3255"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution         7.82%       3.251ms        12.49%       5.195ms      98.019us      23.002ms        58.79%      23.002ms     434.000us            53  \n",
            "           cudnn_volta_scudnn_128x128_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       8.930ms        22.82%       8.930ms     343.462us            26  \n",
            "                                 aten::cudnn_batch_norm         5.99%       2.493ms        10.64%       4.426ms      83.509us       6.706ms        17.14%       6.706ms     126.528us            53  \n",
            "                                             aten::add_         3.67%       1.528ms         5.62%       2.336ms      19.148us       6.088ms        15.56%       6.088ms      49.902us           122  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       3.737ms         9.55%       3.737ms      70.509us            53  \n",
            "            cudnn_volta_scudnn_128x64_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       3.538ms         9.04%       3.538ms     353.800us            10  \n",
            "void cudnn::bn_fw_tr_1C11_singleread<float, 512, tru...         0.00%       0.000us         0.00%       0.000us       0.000us       3.141ms         8.03%       3.141ms      74.786us            42  \n",
            "cudnn_volta_scudnn_128x32_sliced1x4_ldg4_relu_exp_me...         0.00%       0.000us         0.00%       0.000us       0.000us       2.852ms         7.29%       2.852ms     950.667us             3  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us       2.600ms         6.65%       2.600ms     200.000us            13  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       2.387ms         6.10%       2.387ms       2.387ms             1  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       2.139ms         5.47%       2.139ms     133.688us            16  \n",
            "                                        aten::clamp_min         1.63%     676.000us         2.46%       1.025ms      21.354us       2.123ms         5.43%       2.123ms      44.229us            48  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       2.123ms         5.43%       2.123ms      44.229us            48  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       2.016ms         5.15%       2.016ms     201.600us            10  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.549ms         3.96%       1.549ms       1.549ms             1  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 41.594ms\n",
            "Self CUDA time total: 39.124ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Large ResNet-101 "
      ],
      "metadata": {
        "id": "UqNnl06bn5SC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet101_bottleneck(large_mode=True).to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VOVnQEJHn4dA",
        "outputId": "201ffa15-e4c9-4f22-da08-831d58877db1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution         4.47%       5.505ms         6.68%       8.230ms      79.135us      50.307ms        66.40%      50.307ms     483.721us           104  \n",
            "           cudnn_volta_scudnn_128x128_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us      26.045ms        34.37%      26.045ms     434.083us            60  \n",
            "                                 aten::cudnn_batch_norm         4.01%       4.940ms         7.20%       8.864ms      85.231us      10.719ms        14.15%      10.719ms     103.067us           104  \n",
            "                                             aten::add_         2.29%       2.820ms         3.58%       4.412ms      18.307us       9.822ms        12.96%       9.822ms      40.755us           241  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us       7.677ms        10.13%       7.677ms     255.900us            30  \n",
            "void cudnn::bn_fw_tr_1C11_singleread<float, 512, tru...         0.00%       0.000us         0.00%       0.000us       0.000us       7.090ms         9.36%       7.090ms      76.237us            93  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       5.913ms         7.80%       5.913ms      56.856us           104  \n",
            "            cudnn_volta_scudnn_128x64_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       4.444ms         5.87%       4.444ms     444.400us            10  \n",
            "cudnn_volta_scudnn_128x32_sliced1x4_ldg4_relu_exp_me...         0.00%       0.000us         0.00%       0.000us       0.000us       3.652ms         4.82%       3.652ms       1.217ms             3  \n",
            "                                        aten::clamp_min         1.18%       1.449ms         1.74%       2.148ms      21.697us       3.454ms         4.56%       3.454ms      34.889us            99  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       3.454ms         4.56%       3.454ms      34.889us            99  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       3.448ms         4.55%       3.448ms     104.485us            33  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       3.053ms         4.03%       3.053ms       3.053ms             1  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       2.060ms         2.72%       2.060ms     206.000us            10  \n",
            "void cudnn::bn_fw_tr_1C11_kernel_NCHW<float, float, ...         0.00%       0.000us         0.00%       0.000us       0.000us       1.569ms         2.07%       1.569ms       1.569ms             1  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 123.154ms\n",
            "Self CUDA time total: 75.769ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Large ResNet-152"
      ],
      "metadata": {
        "id": "B8NHlBO3n888"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet152_bottleneck(large_mode=True).to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "profile_model(model, inputs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s6fheY09oAEu",
        "outputId": "8491401b-613a-4e6a-b27f-9725525137c3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                aten::cudnn_convolution         5.74%       9.684ms         9.27%      15.649ms     100.961us      71.551ms        67.34%      71.551ms     461.619us           155  \n",
            "           cudnn_volta_scudnn_128x128_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us      38.723ms        36.44%      38.723ms     430.256us            90  \n",
            "                                 aten::cudnn_batch_norm         5.14%       8.673ms        10.50%      17.712ms     114.271us      14.439ms        13.59%      14.439ms      93.155us           155  \n",
            "                                             aten::add_         3.46%       5.844ms         6.53%      11.026ms      30.628us      13.759ms        12.95%      13.759ms      38.219us           360  \n",
            "                                  volta_sgemm_128x64_nn         0.00%       0.000us         0.00%       0.000us       0.000us      12.088ms        11.38%      12.088ms     257.191us            47  \n",
            "void cudnn::bn_fw_tr_1C11_singleread<float, 512, tru...         0.00%       0.000us         0.00%       0.000us       0.000us      10.826ms        10.19%      10.826ms      75.181us           144  \n",
            "void at::native::elementwise_kernel<128, 2, at::nati...         0.00%       0.000us         0.00%       0.000us       0.000us       8.000ms         7.53%       8.000ms      51.613us           155  \n",
            "            cudnn_volta_scudnn_128x64_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       6.055ms         5.70%       6.055ms     432.500us            14  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       5.066ms         4.77%       5.066ms     101.320us            50  \n",
            "                                        aten::clamp_min         1.45%       2.442ms         2.44%       4.117ms      27.447us       5.043ms         4.75%       5.043ms      33.620us           150  \n",
            "void at::native::vectorized_elementwise_kernel<4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       5.043ms         4.75%       5.043ms      33.620us           150  \n",
            "cudnn_volta_scudnn_128x32_sliced1x4_ldg4_relu_exp_me...         0.00%       0.000us         0.00%       0.000us       0.000us       3.656ms         3.44%       3.656ms       1.219ms             3  \n",
            "       cudnn_volta_scudnn_128x64_relu_xregs_large_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us       3.054ms         2.87%       3.054ms       3.054ms             1  \n",
            "void cudnn::winograd_nonfused::winogradForwardFilter...         0.00%       0.000us         0.00%       0.000us       0.000us       2.368ms         2.23%       2.368ms      50.383us            47  \n",
            "void cudnn::winograd_nonfused::winogradForwardData4x...         0.00%       0.000us         0.00%       0.000us       0.000us       2.331ms         2.19%       2.331ms      49.596us            47  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 168.725ms\n",
            "Self CUDA time total: 106.260ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Analysis\n",
        "\n",
        "As expected we see significant gains in performance when running various resnets. With the Standard ResNets we see upto 50x speedup compared to running on CPU with the execution times reducing from order of magnitude of a few seconds to a few hundered milliseconds.\n",
        "\n",
        "After adding bottleneck layers we get even faster execution times with order of magnitude less than one hundered milliseconds (the largest one, Bottleneck RN152 runs a forward pass in less than 150ms). Furthermore when running on GPU we get very fast runtimes for Large Resnets with Large ResNet-152 runnning in a fraction of the time it takes to run any sort of resnet on the CPU."
      ],
      "metadata": {
        "id": "PF0u2-Nzo-jq"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Forward pass and Backward pass"
      ],
      "metadata": {
        "id": "UuQpMCGixKdb"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Finally, we investigate the execution time of a full forward and backward pass."
      ],
      "metadata": {
        "id": "gW3rsWNuxNT8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = resnet152_bottleneck(large_mode=True).to(device)\n",
        "inputs = torch.rand(32, 3, 224, 224).to(device)    # Batch size 32\n",
        "targets = torch.randint(1000, size=(32,)).to(device)\n",
        "\n",
        "model(inputs)  # Warmup\n",
        "\n",
        "inputs.requires_grad_(True)\n",
        "loss = nn.CrossEntropyLoss()\n",
        "\n",
        "with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], record_shapes=True) as prof:\n",
        "    with record_function(\"model_inference\"):\n",
        "        y = model(inputs)\n",
        "    with record_function(\"backpropagation\"):\n",
        "        out = loss(y, targets)\n",
        "        out.backward()\n",
        "\n",
        "print(prof.key_averages().table(sort_by=\"cuda_time_total\", row_limit=15))\n",
        "    "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QnCQLAuyxPfi",
        "outputId": "68e02b4b-dcb5-4a3d-9543-064af7c2e940"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "autograd::engine::evaluate_function: ConvolutionBack...         0.63%       2.769ms        18.94%      83.292ms     537.368us       0.000us         0.00%     135.664ms     875.252us           155  \n",
            "                                   ConvolutionBackward0         0.25%       1.096ms        17.93%      78.828ms     508.568us       0.000us         0.00%     130.647ms     842.884us           155  \n",
            "                             aten::convolution_backward         3.53%      15.505ms        17.68%      77.732ms     501.497us     124.616ms        47.99%     130.647ms     842.884us           155  \n",
            "                                        model_inference         4.00%      17.568ms        16.26%      71.498ms      71.498ms       0.000us         0.00%      95.684ms      95.684ms             1  \n",
            "                                      aten::convolution         0.48%       2.101ms         5.43%      23.859ms     153.929us       0.000us         0.00%      70.296ms     453.523us           155  \n",
            "                                     aten::_convolution         0.52%       2.267ms         4.95%      21.758ms     140.374us       0.000us         0.00%      70.296ms     453.523us           155  \n",
            "                                           aten::conv2d         0.22%     960.000us         5.60%      24.628ms     158.890us       0.000us         0.00%      69.287ms     447.013us           155  \n",
            "                                aten::cudnn_convolution         2.15%       9.473ms         3.29%      14.476ms      93.394us      62.629ms        24.12%      62.629ms     404.058us           155  \n",
            "           cudnn_volta_scudnn_128x128_relu_medium_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us      33.427ms        12.87%      33.427ms     371.411us            90  \n",
            "                                  volta_sgemm_128x64_tn         0.00%       0.000us         0.00%       0.000us       0.000us      27.199ms        10.47%      27.199ms     298.890us            91  \n",
            "         cudnn_volta_scudnn_128x64_stridedB_small_nn_v1         0.00%       0.000us         0.00%       0.000us       0.000us      26.140ms        10.07%      26.140ms     297.045us            88  \n",
            "                                             aten::add_         1.29%       5.688ms         2.50%      11.005ms      26.841us      18.395ms         7.08%      18.395ms      44.866us           410  \n",
            "autograd::engine::evaluate_function: CudnnBatchNormB...         0.61%       2.704ms         3.89%      17.100ms     110.323us       0.000us         0.00%      17.103ms     110.342us           155  \n",
            "                                CudnnBatchNormBackward0         0.27%       1.207ms         3.27%      14.396ms      92.877us       0.000us         0.00%      17.103ms     110.342us           155  \n",
            "                        aten::cudnn_batch_norm_backward         1.39%       6.124ms         3.00%      13.189ms      85.090us      17.103ms         6.59%      17.103ms     110.342us           155  \n",
            "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
            "Self CPU time total: 439.688ms\n",
            "Self CUDA time total: 259.691ms\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We see here once again, as expected, backpropogation runs much faster when we run on GPUs. Training the model becomes much more feasible as now we can run backpropogation in just a few hundered milliseconds with a single training step taking less than a second. This makes it significantly more reasonable to train a complex model such as Large ResNet-152."
      ],
      "metadata": {
        "id": "Xyz0qkoOxow9"
      }
    }
  ]
}