Pytorch_Transfer_Learning_and_Fine_Tuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#Imports"
      ],
      "metadata": {
        "id": "nYCniL1j7LDF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "NvQWUmOI5hXu"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import DataLoader\n",
        "import torchvision.datasets as datasets\n",
        "import torchvision.transforms as transforms\n",
        "import torchvision\n",
        "import sys"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Set Device"
      ],
      "metadata": {
        "id": "iL0fBwEK_16T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ],
      "metadata": {
        "id": "k0MCq5ck_4Bx"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Hyperparameters"
      ],
      "metadata": {
        "id": "KGuggL72LJxj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "in_channel = 3\n",
        "num_classes = 10\n",
        "learning_rate = 0.001\n",
        "batch_size = 1024\n",
        "num_epochs = 5"
      ],
      "metadata": {
        "id": "abHSoGbuAONC"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load Pretrain Model and Modification"
      ],
      "metadata": {
        "id": "ervOOJ3e9TAB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Identity(nn.Module):\n",
        "  def __init__(self):\n",
        "    super(Identity, self).__init__()\n",
        "\n",
        "  def forward(self, x):\n",
        "    return x"
      ],
      "metadata": {
        "id": "N9R_eAp1SMB9"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = torchvision.models.vgg16(pretrained=True)\n",
        "\n",
        "for param in model.parameters():\n",
        "  param.requires_grad = False\n",
        "\n",
        "model.avgpool = Identity()\n",
        "model.classifier = nn.Sequential(nn.Linear(512,100),\n",
        "                                 nn.ReLU(),\n",
        "                                 nn.Linear(100,10))\n",
        "print(model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7jetto3cQM4P",
        "outputId": "c3dd25e5-b181-4aa0-c789-33f78e9bbc2a"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.9/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "VGG(\n",
            "  (features): Sequential(\n",
            "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (1): ReLU(inplace=True)\n",
            "    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (3): ReLU(inplace=True)\n",
            "    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
            "    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (6): ReLU(inplace=True)\n",
            "    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (8): ReLU(inplace=True)\n",
            "    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
            "    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (11): ReLU(inplace=True)\n",
            "    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (13): ReLU(inplace=True)\n",
            "    (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (15): ReLU(inplace=True)\n",
            "    (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
            "    (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (18): ReLU(inplace=True)\n",
            "    (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (20): ReLU(inplace=True)\n",
            "    (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (22): ReLU(inplace=True)\n",
            "    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
            "    (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (25): ReLU(inplace=True)\n",
            "    (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (27): ReLU(inplace=True)\n",
            "    (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
            "    (29): ReLU(inplace=True)\n",
            "    (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
            "  )\n",
            "  (avgpool): Identity()\n",
            "  (classifier): Sequential(\n",
            "    (0): Linear(in_features=512, out_features=100, bias=True)\n",
            "    (1): ReLU()\n",
            "    (2): Linear(in_features=100, out_features=10, bias=True)\n",
            "  )\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load Data"
      ],
      "metadata": {
        "id": "CHio82wVLQQz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset = datasets.CIFAR10(root='dataset/', train=True, transform=transforms.ToTensor(), download=True)\n",
        "train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
        "test_dataset = datasets.CIFAR10(root='dataset/', train=False, transform=transforms.ToTensor(), download=True)\n",
        "test_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)"
      ],
      "metadata": {
        "id": "mJCof2QRA-2L",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1c292333-75a6-4a48-8fe7-d8f54cd0a830"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Initialize network"
      ],
      "metadata": {
        "id": "BqUExuVrDzZn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.to(device)"
      ],
      "metadata": {
        "id": "yBkgd3uoCgRG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "47f8da9c-6322-4e35-cd67-56b8eb8b1997"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "VGG(\n",
              "  (features): Sequential(\n",
              "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (3): ReLU(inplace=True)\n",
              "    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (6): ReLU(inplace=True)\n",
              "    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (8): ReLU(inplace=True)\n",
              "    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (11): ReLU(inplace=True)\n",
              "    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (13): ReLU(inplace=True)\n",
              "    (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (15): ReLU(inplace=True)\n",
              "    (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (18): ReLU(inplace=True)\n",
              "    (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (20): ReLU(inplace=True)\n",
              "    (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (22): ReLU(inplace=True)\n",
              "    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (25): ReLU(inplace=True)\n",
              "    (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (27): ReLU(inplace=True)\n",
              "    (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (29): ReLU(inplace=True)\n",
              "    (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "  )\n",
              "  (avgpool): Identity()\n",
              "  (classifier): Sequential(\n",
              "    (0): Linear(in_features=512, out_features=100, bias=True)\n",
              "    (1): ReLU()\n",
              "    (2): Linear(in_features=100, out_features=10, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loss and optimizer"
      ],
      "metadata": {
        "id": "BOof4chADwcF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(model.parameters(), lr=learning_rate)"
      ],
      "metadata": {
        "id": "zH9LkEhnC9iB"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train Network"
      ],
      "metadata": {
        "id": "zIMDWQCZD-cF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in range(num_epochs):\n",
        "  \n",
        "  losses = []\n",
        "  for batch_idx, (data, targets) in enumerate(train_loader):\n",
        "    # Get data to cuda if possible\n",
        "    data = data.to(device=device)\n",
        "    targets = targets.to(device=device)\n",
        "\n",
        "    # Forward\n",
        "    scores = model(data)\n",
        "    loss = criterion(scores, targets)\n",
        "    losses.append(loss.item())\n",
        "\n",
        "    # backward\n",
        "    optimizer.zero_grad()\n",
        "    loss.backward()\n",
        "\n",
        "    # Gradient descent or adam step\n",
        "    optimizer.step()\n",
        "  mean_loss = sum(losses)/len(losses)\n",
        "  print(f'Loss at epoch {epoch} was {mean_loss:.5f}')\n"
      ],
      "metadata": {
        "id": "YJE32OKcD89T",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0e7bbb05-5f20-4879-9379-9a58075882eb"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss at epoch 0 was 1.60796\n",
            "Loss at epoch 1 was 1.22002\n",
            "Loss at epoch 2 was 1.14767\n",
            "Loss at epoch 3 was 1.11271\n",
            "Loss at epoch 4 was 1.08977\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Check accuracy on training and test to see how good our model"
      ],
      "metadata": {
        "id": "e7_3anF0Hr3S"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def check_accuracy(loader, model):\n",
        "  if loader.dataset.train:\n",
        "    print(\"Checking accuracy on training data\")\n",
        "  else:\n",
        "    print(\"Checking accuracy on test data\")\n",
        "  \n",
        "  num_correct = 0\n",
        "  num_samples = 0\n",
        "  model.eval()\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for x, y in loader:\n",
        "      x = x.to(device=device)\n",
        "      y = y.to(device=device)\n",
        "\n",
        "      scores = model(x)\n",
        "      _, predictions = scores.max(1)\n",
        "      num_correct+=(predictions==y).sum()\n",
        "      num_samples+=predictions.size(0)\n",
        "\n",
        "    print(f'Got {num_correct/num_samples} with accuracy {float(num_correct)/float(num_samples)*100:.2f}')\n",
        "  # model.train()\n",
        "\n"
      ],
      "metadata": {
        "id": "f-q9csl4Hzlj"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Check Accuracy"
      ],
      "metadata": {
        "id": "qsmFLA0XKrRd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "check_accuracy(train_loader, model)\n",
        "check_accuracy(test_loader, model)"
      ],
      "metadata": {
        "id": "yvOmLKI1Joho",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6bd4de7c-5a72-41d4-f3ec-f5205ff0cbf8"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Checking accuracy on training data\n",
            "Got 0.6256399750709534 with accuracy 62.56\n",
            "Checking accuracy on training data\n",
            "Got 0.6256399750709534 with accuracy 62.56\n"
          ]
        }
      ]
    }
  ]
}