Unlearn_test_DMM.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<a href=\"https://colab.research.google.com/github/nem-42098/SL_Proj_Unlearning/blob/main/Unlearn_test.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SCIXLmbpKuwo",
        "outputId": "f0452157-f739-4f42-c2c7-5d248ac5facc"
      },
      "outputs": [],
      "source": [
        "# !git clone https://github.com/nem-42098/SL_Proj_Unlearning.git\n",
        "# import os\n",
        "# os.chdir('/content/SL_Proj_Unlearning')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "6CbLaIs8Kuwq"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torchvision\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torchvision.transforms as transforms\n",
        "from torchvision import models\n",
        "import numpy as np\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iyJgdsOfKuwr"
      },
      "source": [
        "### Load Pre-Trained VGG network\n",
        "> #### https://github.com/chenyaofo\n",
        "> ### Note: There is some issue with using Batch Norm before ReLu as it creates a bias in the network. So people exchange the order between the two for tackling the bias"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5wvUMz6DKuwr",
        "outputId": "8b291d56-9a33-417f-a4ef-0d43d3bb4dbc"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Using cache found in C:\\Users\\nmura/.cache\\torch\\hub\\chenyaofo_pytorch-cifar-models_master\n"
          ]
        }
      ],
      "source": [
        "### First time when you wan to download the model\n",
        "device=torch.device('cuda')\n",
        "model = torch.hub.load(\"chenyaofo/pytorch-cifar-models\", \"cifar10_vgg19_bn\", pretrained=True)\n",
        "model=model.to(device)\n",
        "### For\n",
        "#  future uses:Loading from the local\n",
        "\n",
        "# model_1=torch.hub.load(\"C:/Users/nmura/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master\",'hubconf.py',source='local')\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[1, 2, 3, 2, 3, 4, 5]"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "a=[[1,2,3],[2,3,4,5]]\n",
        "import numpy as np\n",
        "from itertools import chain\n",
        "list(chain.from_iterable(a))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "VGG(\n",
              "  (features): Sequential(\n",
              "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (2): ReLU(inplace=True)\n",
              "    (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (5): ReLU(inplace=True)\n",
              "    (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (7): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (8): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (9): ReLU(inplace=True)\n",
              "    (10): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (11): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (12): ReLU(inplace=True)\n",
              "    (13): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (14): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (15): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (16): ReLU(inplace=True)\n",
              "    (17): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (18): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (19): ReLU(inplace=True)\n",
              "    (20): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (21): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (22): ReLU(inplace=True)\n",
              "    (23): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (24): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (25): ReLU(inplace=True)\n",
              "    (26): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (27): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (29): ReLU(inplace=True)\n",
              "    (30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (32): ReLU(inplace=True)\n",
              "    (33): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (34): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (35): ReLU(inplace=True)\n",
              "    (36): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (37): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (38): ReLU(inplace=True)\n",
              "    (39): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (40): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (41): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (42): ReLU(inplace=True)\n",
              "    (43): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (44): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (45): ReLU(inplace=True)\n",
              "    (46): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (47): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (48): ReLU(inplace=True)\n",
              "    (49): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (50): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (51): ReLU(inplace=True)\n",
              "    (52): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "  )\n",
              "  (classifier): Sequential(\n",
              "    (0): Linear(in_features=512, out_features=512, bias=True)\n",
              "    (1): ReLU(inplace=True)\n",
              "    (2): Dropout(p=0.5, inplace=False)\n",
              "    (3): Linear(in_features=512, out_features=512, bias=True)\n",
              "    (4): ReLU(inplace=True)\n",
              "    (5): Dropout(p=0.5, inplace=False)\n",
              "    (6): Linear(in_features=512, out_features=10, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "def get_named_layers(net, is_state_dict=True):\n",
        "    conv2d_idx = 0\n",
        "    convT2d_idx = 0\n",
        "    linear_idx = 0\n",
        "    batchnorm2d_idx = 0\n",
        "    named_layers = []\n",
        "    for mod in net.modules():\n",
        "        if isinstance(mod, torch.nn.Conv2d):\n",
        "            layer_name = 'Conv2d{}_{}-{}'.format(\n",
        "                conv2d_idx, mod.in_channels, mod.out_channels\n",
        "            )\n",
        "            named_layers.append(layer_name)\n",
        "            if mod.bias is not None:\n",
        "                named_layers.append(layer_name + '_bias')\n",
        "            conv2d_idx += 1\n",
        "        elif isinstance(mod, torch.nn.ConvTranspose2d):\n",
        "            layer_name = 'ConvT2d{}_{}-{}'.format(\n",
        "                conv2d_idx, mod.in_channels, mod.out_channels\n",
        "            )\n",
        "            named_layers.append(layer_name)\n",
        "            if hasattr(mod, \"bias\"):\n",
        "                named_layers.append(layer_name + '_bias')\n",
        "            convT2d_idx += 1\n",
        "        elif isinstance(mod, torch.nn.BatchNorm2d):\n",
        "            layer_name = 'BatchNorm2D{}_{}'.format(\n",
        "                batchnorm2d_idx, mod.num_features)\n",
        "            named_layers.append(layer_name)\n",
        "            named_layers.append(layer_name + '_bais')\n",
        "            if is_state_dict:\n",
        "                named_layers.append(layer_name + '_running_mean')\n",
        "                named_layers.append(layer_name + '_running_var')\n",
        "                named_layers.append(layer_name + '_num_bathes_tracked')\n",
        "            batchnorm2d_idx += 1\n",
        "        elif isinstance(mod, torch.nn.Linear):\n",
        "            layer_name = 'Linear{}_{}-{}'.format(\n",
        "                linear_idx, mod.in_features, mod.out_features\n",
        "            )\n",
        "            named_layers.append(layer_name)\n",
        "            if hasattr(mod, \"bias\"):\n",
        "                named_layers.append(layer_name + '_bias')\n",
        "            linear_idx += 1\n",
        "    return named_layers\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([1., 0., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.])"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "a=np.eye(N=5)\n",
        "a=a[:,0:3]\n",
        "a=a.reshape(-1)\n",
        "a"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['Conv2d0_3-64',\n",
              " 'Conv2d0_3-64_bias',\n",
              " 'BatchNorm2D0_64',\n",
              " 'BatchNorm2D0_64_bais',\n",
              " 'BatchNorm2D0_64_running_mean',\n",
              " 'BatchNorm2D0_64_running_var',\n",
              " 'BatchNorm2D0_64_num_bathes_tracked',\n",
              " 'Conv2d1_64-64',\n",
              " 'Conv2d1_64-64_bias',\n",
              " 'BatchNorm2D1_64',\n",
              " 'BatchNorm2D1_64_bais',\n",
              " 'BatchNorm2D1_64_running_mean',\n",
              " 'BatchNorm2D1_64_running_var',\n",
              " 'BatchNorm2D1_64_num_bathes_tracked',\n",
              " 'Conv2d2_64-128',\n",
              " 'Conv2d2_64-128_bias',\n",
              " 'BatchNorm2D2_128',\n",
              " 'BatchNorm2D2_128_bais',\n",
              " 'BatchNorm2D2_128_running_mean',\n",
              " 'BatchNorm2D2_128_running_var',\n",
              " 'BatchNorm2D2_128_num_bathes_tracked',\n",
              " 'Conv2d3_128-128',\n",
              " 'Conv2d3_128-128_bias',\n",
              " 'BatchNorm2D3_128',\n",
              " 'BatchNorm2D3_128_bais',\n",
              " 'BatchNorm2D3_128_running_mean',\n",
              " 'BatchNorm2D3_128_running_var',\n",
              " 'BatchNorm2D3_128_num_bathes_tracked',\n",
              " 'Conv2d4_128-256',\n",
              " 'Conv2d4_128-256_bias',\n",
              " 'BatchNorm2D4_256',\n",
              " 'BatchNorm2D4_256_bais',\n",
              " 'BatchNorm2D4_256_running_mean',\n",
              " 'BatchNorm2D4_256_running_var',\n",
              " 'BatchNorm2D4_256_num_bathes_tracked',\n",
              " 'Conv2d5_256-256',\n",
              " 'Conv2d5_256-256_bias',\n",
              " 'BatchNorm2D5_256',\n",
              " 'BatchNorm2D5_256_bais',\n",
              " 'BatchNorm2D5_256_running_mean',\n",
              " 'BatchNorm2D5_256_running_var',\n",
              " 'BatchNorm2D5_256_num_bathes_tracked',\n",
              " 'Conv2d6_256-256',\n",
              " 'Conv2d6_256-256_bias',\n",
              " 'BatchNorm2D6_256',\n",
              " 'BatchNorm2D6_256_bais',\n",
              " 'BatchNorm2D6_256_running_mean',\n",
              " 'BatchNorm2D6_256_running_var',\n",
              " 'BatchNorm2D6_256_num_bathes_tracked',\n",
              " 'Conv2d7_256-256',\n",
              " 'Conv2d7_256-256_bias',\n",
              " 'BatchNorm2D7_256',\n",
              " 'BatchNorm2D7_256_bais',\n",
              " 'BatchNorm2D7_256_running_mean',\n",
              " 'BatchNorm2D7_256_running_var',\n",
              " 'BatchNorm2D7_256_num_bathes_tracked',\n",
              " 'Conv2d8_256-512',\n",
              " 'Conv2d8_256-512_bias',\n",
              " 'BatchNorm2D8_512',\n",
              " 'BatchNorm2D8_512_bais',\n",
              " 'BatchNorm2D8_512_running_mean',\n",
              " 'BatchNorm2D8_512_running_var',\n",
              " 'BatchNorm2D8_512_num_bathes_tracked',\n",
              " 'Conv2d9_512-512',\n",
              " 'Conv2d9_512-512_bias',\n",
              " 'BatchNorm2D9_512',\n",
              " 'BatchNorm2D9_512_bais',\n",
              " 'BatchNorm2D9_512_running_mean',\n",
              " 'BatchNorm2D9_512_running_var',\n",
              " 'BatchNorm2D9_512_num_bathes_tracked',\n",
              " 'Conv2d10_512-512',\n",
              " 'Conv2d10_512-512_bias',\n",
              " 'BatchNorm2D10_512',\n",
              " 'BatchNorm2D10_512_bais',\n",
              " 'BatchNorm2D10_512_running_mean',\n",
              " 'BatchNorm2D10_512_running_var',\n",
              " 'BatchNorm2D10_512_num_bathes_tracked',\n",
              " 'Conv2d11_512-512',\n",
              " 'Conv2d11_512-512_bias',\n",
              " 'BatchNorm2D11_512',\n",
              " 'BatchNorm2D11_512_bais',\n",
              " 'BatchNorm2D11_512_running_mean',\n",
              " 'BatchNorm2D11_512_running_var',\n",
              " 'BatchNorm2D11_512_num_bathes_tracked',\n",
              " 'Conv2d12_512-512',\n",
              " 'Conv2d12_512-512_bias',\n",
              " 'BatchNorm2D12_512',\n",
              " 'BatchNorm2D12_512_bais',\n",
              " 'BatchNorm2D12_512_running_mean',\n",
              " 'BatchNorm2D12_512_running_var',\n",
              " 'BatchNorm2D12_512_num_bathes_tracked',\n",
              " 'Conv2d13_512-512',\n",
              " 'Conv2d13_512-512_bias',\n",
              " 'BatchNorm2D13_512',\n",
              " 'BatchNorm2D13_512_bais',\n",
              " 'BatchNorm2D13_512_running_mean',\n",
              " 'BatchNorm2D13_512_running_var',\n",
              " 'BatchNorm2D13_512_num_bathes_tracked',\n",
              " 'Conv2d14_512-512',\n",
              " 'Conv2d14_512-512_bias',\n",
              " 'BatchNorm2D14_512',\n",
              " 'BatchNorm2D14_512_bais',\n",
              " 'BatchNorm2D14_512_running_mean',\n",
              " 'BatchNorm2D14_512_running_var',\n",
              " 'BatchNorm2D14_512_num_bathes_tracked',\n",
              " 'Conv2d15_512-512',\n",
              " 'Conv2d15_512-512_bias',\n",
              " 'BatchNorm2D15_512',\n",
              " 'BatchNorm2D15_512_bais',\n",
              " 'BatchNorm2D15_512_running_mean',\n",
              " 'BatchNorm2D15_512_running_var',\n",
              " 'BatchNorm2D15_512_num_bathes_tracked',\n",
              " 'Linear0_512-512',\n",
              " 'Linear0_512-512_bias',\n",
              " 'Linear1_512-512',\n",
              " 'Linear1_512-512_bias',\n",
              " 'Linear2_512-10',\n",
              " 'Linear2_512-10_bias']"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "get_named_layers(model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "tensor([[0.6047, 0.1680, 0.2273],\n",
              "        [0.2125, 0.4605, 0.3270]])"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "m = nn.Softmax(dim=1)\n",
        "input = torch.randn(2, 3)\n",
        "output = m(input)\n",
        "output.data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XqWzDJ7YKuwr"
      },
      "source": [
        "### Check which pre-trained model are available:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RSxUWBRyKuwr",
        "outputId": "22605340-a9f4-46cf-fdc0-71e03ffda65b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Downloading: \"https://github.com/chenyaofo/pytorch-cifar-models/zipball/master\" to C:\\Users\\nmura/.cache\\torch\\hub\\master.zip\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "['cifar100_mobilenetv2_x0_5',\n",
              " 'cifar100_mobilenetv2_x0_75',\n",
              " 'cifar100_mobilenetv2_x1_0',\n",
              " 'cifar100_mobilenetv2_x1_4',\n",
              " 'cifar100_repvgg_a0',\n",
              " 'cifar100_repvgg_a1',\n",
              " 'cifar100_repvgg_a2',\n",
              " 'cifar100_resnet20',\n",
              " 'cifar100_resnet32',\n",
              " 'cifar100_resnet44',\n",
              " 'cifar100_resnet56',\n",
              " 'cifar100_shufflenetv2_x0_5',\n",
              " 'cifar100_shufflenetv2_x1_0',\n",
              " 'cifar100_shufflenetv2_x1_5',\n",
              " 'cifar100_shufflenetv2_x2_0',\n",
              " 'cifar100_vgg11_bn',\n",
              " 'cifar100_vgg13_bn',\n",
              " 'cifar100_vgg16_bn',\n",
              " 'cifar100_vgg19_bn',\n",
              " 'cifar100_vit_b16',\n",
              " 'cifar100_vit_b32',\n",
              " 'cifar100_vit_h14',\n",
              " 'cifar100_vit_l16',\n",
              " 'cifar100_vit_l32',\n",
              " 'cifar10_mobilenetv2_x0_5',\n",
              " 'cifar10_mobilenetv2_x0_75',\n",
              " 'cifar10_mobilenetv2_x1_0',\n",
              " 'cifar10_mobilenetv2_x1_4',\n",
              " 'cifar10_repvgg_a0',\n",
              " 'cifar10_repvgg_a1',\n",
              " 'cifar10_repvgg_a2',\n",
              " 'cifar10_resnet20',\n",
              " 'cifar10_resnet32',\n",
              " 'cifar10_resnet44',\n",
              " 'cifar10_resnet56',\n",
              " 'cifar10_shufflenetv2_x0_5',\n",
              " 'cifar10_shufflenetv2_x1_0',\n",
              " 'cifar10_shufflenetv2_x1_5',\n",
              " 'cifar10_shufflenetv2_x2_0',\n",
              " 'cifar10_vgg11_bn',\n",
              " 'cifar10_vgg13_bn',\n",
              " 'cifar10_vgg16_bn',\n",
              " 'cifar10_vgg19_bn',\n",
              " 'cifar10_vit_b16',\n",
              " 'cifar10_vit_b32',\n",
              " 'cifar10_vit_h14',\n",
              " 'cifar10_vit_l16',\n",
              " 'cifar10_vit_l32']"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "torch.hub.list(\"chenyaofo/pytorch-cifar-models\", force_reload=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e2OwEFlMKuws"
      },
      "source": [
        "### Downlaoding the Dataset and Creating the Dataloaders"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-01GV7HHKuws",
        "outputId": "428b3ea3-bcb0-4e87-d2aa-dfe2ab381588"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n"
          ]
        }
      ],
      "source": [
        "### Transformation\n",
        "transform = transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(\n",
        "       (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)\n",
        "    )\n",
        "])\n",
        "### Pytorch Datasets\n",
        "train_dataset = torchvision.datasets.CIFAR10(\n",
        "    root= './data', train = True,\n",
        "    download =True, transform = transform)\n",
        "test_dataset = torchvision.datasets.CIFAR10(\n",
        "    root= './data', train = False,\n",
        "    download =True, transform = transform)\n",
        "### Dataloaders\n",
        "trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=2)\n",
        "testloader = torch.utils.data.DataLoader(test_dataset, batch_size=128, shuffle=False, num_workers=2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cclKNSe-Kuws",
        "outputId": "94d263ec-838a-45e8-c338-9f298847ac15"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "50000"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(train_dataset)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bOZabokbKuwt"
      },
      "source": [
        "### Create the Forget Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "-wGG5RC2LW7m"
      },
      "outputs": [],
      "source": [
        "# Define the classes\n",
        "classes = ['forget', 'retain']\n",
        "\n",
        "# Create a dictionary to store datasets for each class\n",
        "class_datasets = {class_name: [] for class_name in classes}\n",
        "\n",
        "# Iterate through the CIFAR-10 dataset and split it into class-specific subsets\n",
        "for image, label in train_dataset:\n",
        "  if label == 1:\n",
        "    class_datasets['forget'].append((image, label))\n",
        "\n",
        "  else:\n",
        "      class_datasets['retain'].append((image, label))\n",
        "\n",
        "# You now have class-specific subsets in the class_datasets dict\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LqTeFK7hKuwu"
      },
      "source": [
        "#### Forget and Retain"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "RbO79fHuKuwu"
      },
      "outputs": [],
      "source": [
        "# Class split\n",
        "retain_dataloader = torch.utils.data.DataLoader(class_datasets['retain'], batch_size=128, shuffle=True, num_workers=2)\n",
        "forget_dataloader = torch.utils.data.DataLoader(class_datasets['forget'], batch_size=128, shuffle=True, num_workers=2)\n",
        "\n",
        "# Random split\n",
        "# train_split_dataset,forget_split_dataset=torch.utils.data.random_split(train_dataset,lengths=[45000,5000])\n",
        "# retain_dataloader = torch.utils.data.DataLoader(train_split_dataset,  batch_size=128, shuffle=True, num_workers=2)\n",
        "# forget_dataloader = torch.utils.data.DataLoader(forget_split_dataset, batch_size=128, shuffle=True, num_workers=2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Unlearner class\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Check that dumb model output random values\n",
        "We expect this values to be uniform in [0,9], thus accuracy to be 10%."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "ename": "NameError",
          "evalue": "name 'ignorant' is not defined",
          "output_type": "error",
          "traceback": [
            "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[1;32md:\\Sapienza academic\\SL\\Project\\ML_Unlearn\\Unlearn_test_DMM.ipynb Cell 24\u001b[0m line \u001b[0;36m<cell line: 19>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/Sapienza%20academic/SL/Project/ML_Unlearn/Unlearn_test_DMM.ipynb#X32sZmlsZQ%3D%3D?line=14'>15</a>\u001b[0m     n  \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m y\u001b[39m.\u001b[39msize(\u001b[39m0\u001b[39m)\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/Sapienza%20academic/SL/Project/ML_Unlearn/Unlearn_test_DMM.ipynb#X32sZmlsZQ%3D%3D?line=16'>17</a>\u001b[0m   \u001b[39mreturn\u001b[39;00m tp\u001b[39m/\u001b[39mn\n\u001b[1;32m---> <a href='vscode-notebook-cell:/d%3A/Sapienza%20academic/SL/Project/ML_Unlearn/Unlearn_test_DMM.ipynb#X32sZmlsZQ%3D%3D?line=18'>19</a>\u001b[0m test(ignorant, forget_dataloader)\n",
            "\u001b[1;31mNameError\u001b[0m: name 'ignorant' is not defined"
          ]
        }
      ],
      "source": [
        "# from tools.Unlearner import Unlearner\n",
        "\n",
        "# unlearner = Unlearner(model, lr = 1e-6, alpha=1)\n",
        "# ignorant = unlearner.reset_weights(model)\n",
        "\n",
        "def test(model, dataloader):\n",
        "  tp, n = 0,0\n",
        "  for X,y in dataloader:\n",
        "    X,y = X.to(device), y.to(device)\n",
        "\n",
        "    with torch.no_grad():\n",
        "      y_pred = model(X)\n",
        "\n",
        "    tp += (y_pred.argmax(axis=1) == y).sum().item()\n",
        "    n  += y.size(0)\n",
        "\n",
        "  return tp/n\n",
        "\n",
        "test(ignorant, forget_dataloader)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [],
      "source": [
        "### Congiguration file taken \n",
        "config={'max_epochs': 200,\n",
        "                    \n",
        "                     'data':\n",
        "                      [('type_', 'cifar100'),\n",
        "                                  ('image_size', 32),\n",
        "                                  ('num_classes', 100),\n",
        "                                  ('root', 'data/cifar100'),\n",
        "                                  ('mean', [0.507, 0.4865, 0.4409]),\n",
        "                                  ('std', [0.2673, 0.2564, 0.2761]),\n",
        "                                  ('batch_size', 256),\n",
        "                                  ('num_workers', 4)],\n",
        "                     'model':[('type_', 'PyTorchHub'),\n",
        "                                  ('force_reload', False),\n",
        "                                  ('repo', 'chenyaofo/pytorch-cifar-models'),\n",
        "                                  ('name', 'cifar100_vgg11_bn'),\n",
        "                                  ('pretrained', False)],\n",
        "                     'optimizer':\n",
        "                      [('type_', 'SGD'),\n",
        "                                  ('lr', 0.1),\n",
        "                                  ('momentum', 0.9),\n",
        "                                  ('dampening', 0),\n",
        "                                  ('weight_decay', 0.0005),\n",
        "                                  ('nesterov', True)],\n",
        "                     'scheduler':\n",
        "                      [('type_', 'CosineAnnealingLR'),\n",
        "                                  ('T_max', 200),\n",
        "                                  ('eta_min', 0)],\n",
        "                     }"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Run the unlearning\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Using cache found in C:\\Users\\nmura/.cache\\torch\\hub\\chenyaofo_pytorch-cifar-models_master\n",
            "  0%|          | 1/200 [00:39<2:10:02, 39.21s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.12561849815221987, 1.0, 0.0]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  1%|          | 2/200 [01:21<2:15:29, 41.06s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.0, 1.0, 0.0]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  2%|▏         | 3/200 [02:02<2:15:00, 41.12s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.0, 1.0, 0.0]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  2%|▏         | 4/200 [02:44<2:15:09, 41.38s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.0, 1.0, 0.0]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  2%|▎         | 5/200 [03:31<2:21:15, 43.46s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.0, 1.0, 0.0]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  3%|▎         | 6/200 [04:19<2:25:28, 44.99s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.0, 1.0, 0.0]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  3%|▎         | 6/200 [05:10<2:47:08, 51.70s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.0, 1.0, 0.0]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "from tools.Unlearner_DMM import UnlearnerDMM\n",
        "model = torch.hub.load(\"chenyaofo/pytorch-cifar-models\", \"cifar10_vgg19_bn\", pretrained=True)\n",
        "model=model.to(device)\n",
        "unlearner = UnlearnerDMM(model, config,alpha=0.2)\n",
        "\n",
        "\n",
        "unlearner.unlearn(forget_dataloader,retain_dataloader)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.11237777777777777"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test(unlearner.retained_model,retain_dataloader)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.0"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test(unlearner.retained_model,forget_dataloader)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "We can see that the unlearning process may need many epochs in the erasure phase to converge. We could also try to increase the learning rate for faster convergence."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "\n",
        "log_df = pd.DataFrame.from_records(unlearner.log, columns=['phase', 'epoch', 'batch', 'tp', 'n', 'loss'])\n",
        "unlearn_history = log_df.groupby(['phase', 'epoch']).agg({'tp':sum, 'n':sum, 'loss': 'mean'}).reset_index()\n",
        "unlearn_history['accuracy'] = unlearn_history.tp / unlearn_history.n\n",
        "unlearn_history.loc[unlearn_history.phase == 'erasure', 'loss_scale'] = unlearn_history.loc[unlearn_history.phase == 'erasure', 'loss'] /unlearn_history.loc[unlearn_history.phase == 'erasure', 'loss'].max() \n",
        "unlearn_history.loc[unlearn_history.phase == 'retrain', 'loss_scale'] = unlearn_history.loc[unlearn_history.phase == 'retrain', 'loss'] /unlearn_history.loc[unlearn_history.phase == 'retrain', 'loss'].max() \n",
        "\n",
        "sns.lineplot(data = unlearn_history, x = 'epoch', y='loss_scale', hue='phase')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Check model performance\n",
        "We now try check the model performance on the retain and forget set. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "models = {'original':model, 'dumb':unlearner.dumb_model, 'erased':unlearner.erased_model, 'retrained':unlearner.retrained_model}\n",
        "dataloaders = {'retain':retain_dataloader, 'forget':forget_dataloader, 'test':testloader}\n",
        "\n",
        "performances = []\n",
        "for model_name, m in models.items():\n",
        "  for dl_name, dl in dataloaders.items():\n",
        "    acc = test(m, dl)\n",
        "    performances.append((model_name, dl_name, acc))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Strangely the dumb network has 0% performance on the forget set, and this may have a negative impact, note that the performance on the forget set is even too low (we expected 10%, not less).\n",
        "\n",
        "Outside of that we note that this procedure works great for the retain and test which are fitted very nicely, and we can even see that there's a big leap in performance from the erased model to the retrained model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "perf_df = pd.DataFrame.from_records(performances, columns=['model', 'data_partition', 'accuracy'])\n",
        "\n",
        "tb = pd.pivot_table(perf_df, index='data_partition', columns='model', values='accuracy')\n",
        "sns.heatmap(tb, annot = True, fmt='.2%')"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}