Fine_Tuning_Roberta.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/redkhalil/DevOps-Python-tools/blob/master/Fine_Tuning_Roberta.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PEtDQuEiO9E8",
        "outputId": "af2fe873-9910-4911-dcfb-dc85bf668133"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mon Apr 11 03:12:40 2022       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   40C    P8    26W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gAuRVlCmP-ha",
        "outputId": "147012e1-866e-463a-923c-aed4d713c51e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.18.0-py3-none-any.whl (4.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.0 MB 5.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.5.1-py3-none-any.whl (77 kB)\n",
            "\u001b[K     |████████████████████████████████| 77 kB 6.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 47.7 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.49-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 10.0 MB/s \n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.5 MB 30.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.5.1 pyyaml-6.0 sacremoses-0.0.49 tokenizers-0.11.6 transformers-4.18.0\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AXvM-yqrnkaF",
        "outputId": "31c1d0ed-68ab-4703-cc83-a20677f4ba0e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla K80\n"
          ]
        }
      ],
      "source": [
        "#from torch import cuda\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hj_LGI65RPNj"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "import torch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OkJQxERuSRgJ"
      },
      "outputs": [],
      "source": [
        "df=pd.read_csv(\"data_combined_concat.csv\",encoding='utf-8')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "-TUtfnsCS2kE",
        "outputId": "a11d000c-b78e-41e7-8925-507cc3104e78"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-5223331d-8648-4850-906f-21ae06af7e26\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>author</th>\n",
              "      <th>post_history</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4thcuts</td>\n",
              "      <td>Feeling like heading for the fridge, taking my...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Apo-cone-lypse</td>\n",
              "      <td>I didnt think I could sink any lower but now I...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>BC2020uzn</td>\n",
              "      <td>Help me be a better parent to my autistic son ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Baal357</td>\n",
              "      <td>How can I ever hope to get out of this when I ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Busy-Estimate-3929</td>\n",
              "      <td>At what age did you start feeling depressed? A...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5223331d-8648-4850-906f-21ae06af7e26')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5223331d-8648-4850-906f-21ae06af7e26 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5223331d-8648-4850-906f-21ae06af7e26');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "               author                                       post_history  \\\n",
              "0             4thcuts  Feeling like heading for the fridge, taking my...   \n",
              "1      Apo-cone-lypse  I didnt think I could sink any lower but now I...   \n",
              "2           BC2020uzn  Help me be a better parent to my autistic son ...   \n",
              "3             Baal357  How can I ever hope to get out of this when I ...   \n",
              "4  Busy-Estimate-3929  At what age did you start feeling depressed? A...   \n",
              "\n",
              "   label  \n",
              "0      1  \n",
              "1      1  \n",
              "2      1  \n",
              "3      1  \n",
              "4      1  "
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.drop(labels='Unnamed: 0',axis=1,inplace=True)\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QWaia8flHzlR"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "\n",
        "def text_clean(x):\n",
        "\n",
        "    x = x.lower() # lowercase everything\n",
        "    x = x.encode('ascii', 'ignore').decode()  # remove unicode characters\n",
        "    x = re.sub(r'https*\\S+', ' ', x) # remove https links\n",
        "    x = re.sub(r'http*\\S+', ' ', x) # remove http links\n",
        "    \n",
        "    # cleaning up text\n",
        "    x = re.sub(r'\\'\\w+', '', x) # remove extra space\n",
        "    x = re.sub(r'\\w*\\d+\\w*', '', x) # remove numerical values \n",
        "    x = re.sub(r'\\s{2,}', ' ', x)\n",
        "    x = re.sub(r'\\s[^\\w\\s]\\s', '', x)\n",
        "    x = re.sub(r'\\[[^\\]]*\\]', '', x) # remove expression start end with []\n",
        "\n",
        "    # remove emojis\n",
        "    emoj = re.compile(\"[\"\n",
        "        u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "        u\"\\U00002500-\\U00002BEF\"  # chinese char\n",
        "        u\"\\U00002702-\\U000027B0\"\n",
        "        u\"\\U00002702-\\U000027B0\"\n",
        "        u\"\\U000024C2-\\U0001F251\"\n",
        "        u\"\\U0001f926-\\U0001f937\"\n",
        "        u\"\\U00010000-\\U0010ffff\"\n",
        "        u\"\\u2640-\\u2642\" \n",
        "        u\"\\u2600-\\u2B55\"\n",
        "        u\"\\u200d\"\n",
        "        u\"\\u23cf\"\n",
        "        u\"\\u23e9\"\n",
        "        u\"\\u231a\"\n",
        "        u\"\\ufe0f\"  # dingbats\n",
        "        u\"\\u3030\"\n",
        "                      \"]+\", re.UNICODE)\n",
        "    x = re.sub(emoj, '', x)\n",
        "    return x\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hwXPP8bmH7xx"
      },
      "outputs": [],
      "source": [
        "df['cleaned_text'] = df.post_history.apply(text_clean)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JEnxgRwuTDg7"
      },
      "outputs": [],
      "source": [
        "df1=df.copy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aw8_PBNyT7HQ"
      },
      "outputs": [],
      "source": [
        "df1.drop(['author'],axis=1,inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IDxsLI-wIM-J"
      },
      "outputs": [],
      "source": [
        "df1.drop(['post_history'],axis=1,inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "9tb53DidUZ1b",
        "outputId": "29c932d4-9ea3-4fb2-d338-c6cc0d86b39b"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-123663bc-e4e5-4506-b560-c6d6a972e64e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>cleaned_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>feeling like heading for the fridge, taking my...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>i didnt think i could sink any lower but now i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>help me be a better parent to my autistic son ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>how can i ever hope to get out of this when i ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>at what age did you start feeling depressed? a...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-123663bc-e4e5-4506-b560-c6d6a972e64e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-123663bc-e4e5-4506-b560-c6d6a972e64e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-123663bc-e4e5-4506-b560-c6d6a972e64e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   label                                       cleaned_text\n",
              "0      1  feeling like heading for the fridge, taking my...\n",
              "1      1  i didnt think i could sink any lower but now i...\n",
              "2      1  help me be a better parent to my autistic son ...\n",
              "3      1  how can i ever hope to get out of this when i ...\n",
              "4      1  at what age did you start feeling depressed? a..."
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df1.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qOXcUf-XRhat",
        "outputId": "f5595c93-4fb8-4e2f-9360-0a93ed97c6f9"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(269, 2)"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df1.describe()\n",
        "df1.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "mWqor06I-h7p",
        "outputId": "277d0ce7-c1d5-4ff8-f237-3b4bf6deacee"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-c6abe8fd-f6da-40c6-a62c-fcf85f31ea06\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cleaned_text</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>label</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>166</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>103</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c6abe8fd-f6da-40c6-a62c-fcf85f31ea06')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c6abe8fd-f6da-40c6-a62c-fcf85f31ea06 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c6abe8fd-f6da-40c6-a62c-fcf85f31ea06');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "       cleaned_text\n",
              "label              \n",
              "0               166\n",
              "1               103"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df1.groupby('label').count()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BGzaF6gcXDwh"
      },
      "outputs": [],
      "source": [
        "MAX_LEN = 256\n",
        "TRAIN_BATCH_SIZE = 4\n",
        "VALID_BATCH_SIZE = 4\n",
        "# EPOCHS = 1\n",
        "LEARNING_RATE = 1e-05"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NW1WJywhLU3j"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoTokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ERCwOxm5i-Vk"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import Dataset, DataLoader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TCzgxy6QLl0j"
      },
      "outputs": [],
      "source": [
        "model_name='roberta-base'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "7629f086a297402fb80367053cff853b",
            "7b1847b6d14f4dddab852d89d64f4538",
            "e034b2d598d441a9917ca9cd48e5662e",
            "9756ee46dc234671a1e336787124f864",
            "f4b6dbd8b6e34ced9458e4968ae95e69",
            "995ec40aa73d48468f808b2792882d28",
            "3239af4dc81a4ba297a85c852bb1ea14",
            "0efea41175d1436695a43cb87f6be173",
            "2dd3cb60fbbf4b7a9518f07c06f849ea",
            "994db462800c4cdf97fbf04f31b7c9ff",
            "e930b25539a246afb1e32ecf58ba0b90",
            "2366ee68e48f42c9b0c215a2e793b1e0",
            "655c0f0a9a9a409c9ad629d6d17ff2c2",
            "498990764daa4974b97bc3d35376ab77",
            "644dce8d87654462b2c203f025750022",
            "49ea079c791a404ea279ba228cab1b30",
            "234a046ae64f4748b80c4abd91dab67d",
            "c71315440d37416ebf42f39ce88ae9b8",
            "3987a1100ba44ef8a97bb7d2a165bce2",
            "b1e99ee4c0ca4bc094d8a87a6991e14e",
            "2f82654a33db46739118fa9f6042663f",
            "cce6f7d2e82e46de87fbeb9ee79e3443",
            "3cdf594aacfd41cc9339e2d2d647f4d8",
            "ba52a5e4ec1d4c819eea841f6edef795",
            "7f55e10cb2c8471c93ffe351fc3edea5",
            "32de4063970f4b6a9e6ebad750405fcb",
            "c5ae14ec773d409aaffeedf0a53b0a98",
            "02c38b2cf34347c2847bfcbea1de9727",
            "28f648efb6dc4157974165e392119db0",
            "eba32af18ec141cf94381359416eac88",
            "e5878c6610f6441885846490e59873a2",
            "359d889c7fac4e69b3ffa45dd90d8afe",
            "b3f076a092db43fca8b7eeae1e8f45c5",
            "49097fb4c3624d3ebe40f114626eeee4",
            "db4ec8612ebc45d1b39de87e98249777",
            "9ca8e485ab604eecb0a144f8efa5813a",
            "3efeca6a08f642d3a55ac2571702cddf",
            "5b1fb891b5524810820878c0797ffe4d",
            "665ec78e79ce407f9ed92598834887a0",
            "58028463413546caa0d41e9753a90844",
            "2d16a2868c8b43f4be9f2a76e899f946",
            "86fb7ccef8c349dc8fd8b899abd74925",
            "3a6270e065d6445eb25f8c44ec244ad6",
            "b33abc4dcd2243dab866433f9b132cbb"
          ]
        },
        "id": "ZeqR9luWLqw0",
        "outputId": "d312b968-2072-4302-a0ba-8c8bdbdce5af"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7629f086a297402fb80367053cff853b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/481 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2366ee68e48f42c9b0c215a2e793b1e0",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/878k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3cdf594aacfd41cc9339e2d2d647f4d8",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "49097fb4c3624d3ebe40f114626eeee4",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.29M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "tokenizer=AutoTokenizer.from_pretrained(model_name)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "okgkYQigIu00"
      },
      "outputs": [],
      "source": [
        "class SentimentData(Dataset):\n",
        "    def __init__(self, dataframe, tokenizer, max_len):\n",
        "        self.tokenizer = tokenizer\n",
        "        self.data = dataframe\n",
        "        self.text = dataframe.cleaned_text\n",
        "        self.targets = dataframe.label\n",
        "        self.max_len = max_len\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.text)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        text = str(self.text[index])\n",
        "        text = \" \".join(text.split())\n",
        "        \n",
        "        \n",
        "        inputs = self.tokenizer.encode_plus(\n",
        "            text,\n",
        "            None,\n",
        "            add_special_tokens=True,\n",
        "            max_length=self.max_len,\n",
        "            pad_to_max_length=True,\n",
        "            return_token_type_ids=True\n",
        "        )\n",
        "        ids = inputs['input_ids']\n",
        "        mask = inputs['attention_mask']\n",
        "        token_type_ids = inputs[\"token_type_ids\"]\n",
        "\n",
        "\n",
        "        return {\n",
        "            'ids': torch.tensor(ids, dtype=torch.long),\n",
        "            'mask': torch.tensor(mask, dtype=torch.long),\n",
        "            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n",
        "            'targets': torch.tensor(self.targets[index], dtype=torch.float)\n",
        "        }"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "he-BuR_qXNnp"
      },
      "outputs": [],
      "source": [
        "train_size = 0.8\n",
        "train_data=df1.sample(frac=train_size,random_state=200)\n",
        "test_data=df1.drop(train_data.index).reset_index(drop=True)\n",
        "train_data = train_data.reset_index(drop=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LLDHQRdZjnJI",
        "outputId": "0dce3ace-f551-4add-ed06-2a65ddd8e993"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(215, 2)"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AzI5o0NIcyTd",
        "outputId": "1b2c589c-4934-4106-afdd-74edb2a1c1bd"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(54, 2)"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test_data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "t7-yJklaJuG0"
      },
      "outputs": [],
      "source": [
        "training_set = SentimentData(train_data, tokenizer, 512)\n",
        "testing_set = SentimentData(test_data, tokenizer, 512)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qI2so5z5cZYt",
        "outputId": "b0c8fc9a-d9bb-4347-ac4b-718ee54933a4"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2269: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0])"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "training_set.__getitem__(1)['token_type_ids']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YbL6pOZXbkqf",
        "outputId": "20305884-d9a4-4389-f4d4-77a816ba19b6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2269: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([    0,  9178,    64,   939,   655,  1034,     7,   120,    66,     9,\n",
              "           42,    77,   939,    64,   892,   932,     6,   117,   633,  7688,\n",
              "            7,   162,     8,   939,  4889,    11,   317,    13,   107,   116,\n",
              "           25,   939,    64,  2045,     7,   465,    10,   699,  1948,   939,\n",
              "           40,  1394,   259,     4,    99,    16,     5,   921,    13,   951,\n",
              "         1403,    12, 29888,   740, 42964,     7,   120,    10,   633,   116,\n",
              "            5,  2014,    16,  9279,   239,  7938,   472,     5,  2014,    16,\n",
              "         9279,   239,  7938,   472,   784,   506,    10, 12594,  1757,   368,\n",
              "           54,    64,   244,    66,   162,     8,   127,  2837,   120,   357,\n",
              "           23,   313,     5,  2014,    16,  9279,   147,    64,   939,   465,\n",
              "         2185,    10,  2837,     7,   310,   313,    19,    15,   364,   257,\n",
              "        10228,   116,   141,   251,    74,    24,   185,   162,     7, 16655,\n",
              "           62,    13,  4549,    19,  9717,  5660,   116,   546,    13,   472,\n",
              "            7,   165,    62,    19,     8,  1920,   239,  7938,    25,    10,\n",
              "         8566,   546,     7,  1962,    50,  1045,    10,  2837,  2882,     7,\n",
              "         1920,    81,   332,    42,   191,   546,    13,   472,     7,  1169,\n",
              "         1026,    10,  2837,    50,     7,  8566,    73,  6588,   139,    19,\n",
              "            4,   114,    47,   236,     7, 16655,     7,  9591,   162,    62,\n",
              "            4,   546,    13,  5633,     7,  7938,    62,    19,     6,   236,\n",
              "            7, 16655,    42,   191,     7,    16,    89,   143,  4704,    66,\n",
              "           89,    13, 11134, 11303, 46247,   116,    99,    16,   120,    73,\n",
              "         7049,     8,   141,   109,    47,   146,    24,   116,    64,   951,\n",
              "         1137,   162,    99,   197,   939,  1532,    59,    98,    14,   939,\n",
              "           64,   244,    66,    15,    42,   695,   116,  2274,  3104,    16,\n",
              "          874,     4,     5,   695,  1495,    16,    10, 30295,  7208,    13,\n",
              "           10,   177,     4,    64,   951,  3922,    99,   197,   939,  1339,\n",
              "           62,     7,  1532,    98,    14,   939,    64,   244,    15,    42,\n",
              "          695,   116,  2274,  3104,    16,   874,     4,     5,   695,  1495,\n",
              "           16,    10, 30295,  1546,    13,    10,   177,     4,    64,   951,\n",
              "         3922,    99,    16,   956,    73,  9178,   109,  1423,  2160,   190,\n",
              "         1642,   447,    15,    42,  7208,    31,     5,  2170,   116,    24,\n",
              "           16,    13,    10, 30295, 11808,     4,    54,   197,   939,  1339,\n",
              "           62,     8, 33136,   116,   614, 12594,   869,   259,     4,   473,\n",
              "         1268,   216,     9,   143,   205, 38892,    73,   438, 33165,    13,\n",
              "          740, 42964, 25776,    11, 29692,  3819,   116,   295,   677,   261,\n",
              "          992,  1469,  2558,  4001,  1021, 22617,  3900,   740, 42964,     6,\n",
              "          821,  2794,   385,   337,  2359, 42661,   116,   475,   259,     6,\n",
              "          519,    41, 18544,  1280,     9,  6882,   235,   122,    98,   939,\n",
              "          240,     7, 13228,    24,    66,  6152,  7586,   939,   202,  1207,\n",
              "           19,   127,  1041,     6,  1882,    66,     9,   542,   118,   416,\n",
              "          528,     7,    10,  1762,     9,   773,    13,     5,  9352,     8,\n",
              "           33,  1240,   107,  2744,   122,    19,   608,  3668,  1085,    19,\n",
              "          127,   301,     4,   939,    57, 16658,     8, 23630,    13,    54,\n",
              "         2215,   141,   251,   122,     6,   685,  1349,     9,    24,    53,\n",
              "          127,   275,  4443,    74,    28, 12857,     4,  4420,     6,   939,\n",
              "           33,    57,     7,  3333,     4,  4420,     6,   939,    33,   551,\n",
              "         5679,    29,     4,  7105,   939,   190,   300, 12399,  2330,    13,\n",
              "           59,   377,     4,  1085,     4,  1147,     4,    23,     4,    70,\n",
              "            4,     5,   200,  1098,  1938,   888,   156,    24,  3007,    87,\n",
              "           24,    21,     4,    13,   162,    24,   269,   505,     7,  1339,\n",
              "           10,   756,  2718,     8,  2845,    15,    99,   939,   236,     7,\n",
              "           28,     2])"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ],
      "source": [
        "testing_set.__getitem__(1)['ids']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7bPMu6X3Nk2b",
        "outputId": "4e9b1b83-0dde-44c2-d487-fb2e73fc3808"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2269: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "tensor(0.)"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "testing_set.__getitem__(23)['targets']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1qNbOheAci9W",
        "outputId": "564b700e-a7fc-49fc-94e9-e4104e388694"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2269: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "tensor(0.)"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "training_set.__getitem__(1)['targets']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UdUxkhjJq6iZ"
      },
      "source": [
        "Dataloaders"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VvImEiQcrBpB"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoModel, AdamW, get_cosine_schedule_with_warmup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6-H8s8k7kyYU"
      },
      "outputs": [],
      "source": [
        "train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
        "                'shuffle': True,\n",
        "                'num_workers': 0\n",
        "                }\n",
        "\n",
        "test_params = {'batch_size': VALID_BATCH_SIZE,\n",
        "                'shuffle': True,\n",
        "                'num_workers': 0\n",
        "                }\n",
        "\n",
        "training_loader = DataLoader(training_set, **train_params)\n",
        "testing_loader = DataLoader(testing_set, **test_params)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sh7D8nD4k4-U"
      },
      "source": [
        "Creating the Neural Network for Fine Tuning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HMqQTafXEaei"
      },
      "outputs": [],
      "source": [
        "class RobertaClass(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(RobertaClass, self).__init__()##calling the super method of neural net\n",
        "        self.l1 = AutoModel.from_pretrained(\"roberta-base\")\n",
        "        self.pre_classifier = torch.nn.Linear(768,768)## input size, output size\n",
        "        self.dropout = torch.nn.Dropout(0.3)\n",
        "        self.classifier = torch.nn.Linear(768, 2)# final ouput layer( 2 for depressed or non-depressed)\n",
        "\n",
        "    def forward(self, input_ids, attention_mask, token_type_ids):\n",
        "        output_1 = self.l1(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
        "        hidden_state = output_1[0]### last_hidden_state- shape(batch_size,number of tokens,768(fixed for base bert model)\n",
        "        pooler = hidden_state[:, 0]## cls embeddings to be used for classfication. fisrt token for each batch- shape(batch_size,768)\n",
        "        pooler = self.pre_classifier(pooler) # Linear\n",
        "        pooler = torch.nn.ReLU()(pooler)#activation function\n",
        "        pooler = self.dropout(pooler)\n",
        "        output = self.classifier(pooler)\n",
        "        #output=torch.nn.Sigmoid(output)\n",
        "        return output"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dRc5zGf1LkGv"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nkZE8qgfok2E"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "8ce94b136eaf43fd9cd79ca1dc05f212",
            "1b7c389be0f14dddbdd7837485c42158",
            "1e5a334875f44fb4a259eac9fd74b33a",
            "eaf6b1a0ff834c9095f5536df6b2b9cd",
            "b2692eea5ee34d9c80c8fe6859536ce7",
            "dce2e39aa5634bb6a906f90f7e57ccb8",
            "de798c677d6f436a8bcb9c14a28801ae",
            "61f7eb32fe2d4a09b1846e2b416e772f",
            "d5d23b32668b4ce99ee26d0f7f4e4a9f",
            "c2336ea003c14f5dafdce6822768286e",
            "b574facc5455476187054586bec3ff75"
          ]
        },
        "id": "TymaMZyWlL9W",
        "outputId": "05e12ce0-6f87-419d-da0d-51cb3e15e40e"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8ce94b136eaf43fd9cd79ca1dc05f212",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/478M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaModel: ['lm_head.dense.weight', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "RobertaClass(\n",
              "  (l1): RobertaModel(\n",
              "    (embeddings): RobertaEmbeddings(\n",
              "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
              "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
              "      (token_type_embeddings): Embedding(1, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): RobertaEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): RobertaPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (pre_classifier): Linear(in_features=768, out_features=768, bias=True)\n",
              "  (dropout): Dropout(p=0.3, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model = RobertaClass()\n",
        "model.to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "saqmvCMPonnW"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S3gBvlTymr9r"
      },
      "source": [
        "Fine Tuning the Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9zlhXvJZmtrD"
      },
      "outputs": [],
      "source": [
        "# Creating the loss function and optimizer\n",
        "loss_function = torch.nn.CrossEntropyLoss()### have to change the entropy loss function\n",
        "optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BqvnDpd9PmZp"
      },
      "outputs": [],
      "source": [
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GrkiB0SSm620"
      },
      "outputs": [],
      "source": [
        "def calcuate_accuracy(preds, targets):\n",
        "    n_correct = (preds==targets).sum().item()\n",
        "   \n",
        "    return n_correct"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jRJJ4Eszp626"
      },
      "outputs": [],
      "source": [
        "def confusion_matrix(preds, targets):\n",
        "    tn,fp,fn,tp=confusion_matrix(targets,preds,labels=[0,1]).ravel()\n",
        "    print('confusion matrix:')\n",
        "    print('            predicted    ')\n",
        "    print('          0     |     1')\n",
        "    print('    ----------------------')\n",
        "    print('   0 | ',format(tn,'5d'),' | ',format(fp,'5d'))\n",
        "    print('gt -----------------------')\n",
        "    print('   1 | ',format(fn,'5d'),' | ',format(tp,'5d'))\n",
        "    print('---------------------------------------------------')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aTbXZ6jlnH_U"
      },
      "outputs": [],
      "source": [
        "# Defining the training function on the 80% of the dataset for tuning the distilbert model\n",
        "\n",
        "def train(epoch):\n",
        "    tr_loss = 0\n",
        "    n_correct = 0\n",
        "    nb_tr_steps = 0\n",
        "    nb_tr_examples = 0\n",
        "    model.train()\n",
        "    for _,data in tqdm(enumerate(training_loader, 0)):\n",
        "        ids = data['ids'].to(device, dtype = torch.long)\n",
        "        mask = data['mask'].to(device, dtype = torch.long)\n",
        "        token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
        "        targets = data['targets'].to(device, dtype = torch.long)\n",
        "        #targets=targets.view(-1,4)\n",
        "        #print(targets)\n",
        "        \n",
        "\n",
        "        outputs = model(ids, mask, token_type_ids)\n",
        "        print(outputs)\n",
        "        loss = loss_function(outputs, targets)### outputs can have any number of rows but columns corresponding to num_classes,targets have to corresonding to the class they belong\n",
        "        tr_loss += loss.item()\n",
        "        #print(tr_loss)\n",
        "        big_val, big_idx = torch.max(outputs.data, dim=1)\n",
        "        n_correct += calcuate_accuracy(big_idx, targets)\n",
        "\n",
        "        nb_tr_steps += 1\n",
        "        nb_tr_examples+=targets.size(0)\n",
        "        \n",
        "        if _%100==0:\n",
        "            loss_step = tr_loss/nb_tr_steps\n",
        "            accu_step = (n_correct*100)/nb_tr_examples \n",
        "            print(f\"Training Loss per 100 steps: {loss_step}\")\n",
        "            print(f\"Training Accuracy per 100 steps: {accu_step}\")\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()### when using data to retrain the weights\n",
        "        # # When using GPU\n",
        "        optimizer.step()\n",
        "\n",
        "    print(f'The Total Accuracy for Epoch {epoch}: {(n_correct*100)/nb_tr_examples}')\n",
        "    epoch_loss = tr_loss/nb_tr_steps\n",
        "    epoch_accu = (n_correct*100)/nb_tr_examples\n",
        "    print(f\"Training Loss Epoch: {epoch_loss}\")\n",
        "    print(f\"Training Accuracy Epoch: {epoch_accu}\")\n",
        "\n",
        "    return "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "Um3pMqZToakv",
        "outputId": "f28643fb-477b-4ec9-c68f-7c331c829df4"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2269: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 0, 1])\n",
            "tensor([[-0.0078,  0.0223],\n",
            "        [-0.0891, -0.0104],\n",
            "        [-0.0239, -0.0315],\n",
            "        [-0.0261, -0.0206]], grad_fn=<AddmmBackward0>)\n",
            "Training Loss per 100 steps: 0.7053422927856445\n",
            "Training Accuracy per 100 steps: 50.0\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r1it [00:27, 27.93s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 0, 0])\n",
            "tensor([[ 0.0490, -0.0517],\n",
            "        [-0.0360,  0.0022],\n",
            "        [-0.0318,  0.0206],\n",
            "        [ 0.0169, -0.0955]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r2it [00:54, 26.94s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 0])\n",
            "tensor([[-0.0222,  0.0386],\n",
            "        [ 0.0436,  0.1116],\n",
            "        [-0.0403, -0.2107],\n",
            "        [ 0.0028, -0.0956]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r3it [01:19, 26.07s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 0])\n",
            "tensor([[ 0.0036,  0.0084],\n",
            "        [ 0.0373, -0.0378],\n",
            "        [ 0.3116, -0.2557],\n",
            "        [ 0.0266,  0.0140]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r4it [01:43, 25.56s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 1])\n",
            "tensor([[ 0.0352, -0.1170],\n",
            "        [ 0.1327, -0.0380],\n",
            "        [-0.0127, -0.0394],\n",
            "        [ 0.0304,  0.0042]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r5it [02:08, 25.34s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 1, 0, 1])\n",
            "tensor([[ 0.0194, -0.0668],\n",
            "        [ 0.0997, -0.0753],\n",
            "        [ 0.0695, -0.1295],\n",
            "        [ 0.0616, -0.0337]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r6it [02:34, 25.26s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 0])\n",
            "tensor([[ 0.1364, -0.0321],\n",
            "        [-0.0261, -0.1106],\n",
            "        [ 0.0519, -0.1083],\n",
            "        [ 0.0629, -0.1462]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r7it [02:59, 25.21s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 1, 1])\n",
            "tensor([[ 0.0521, -0.1424],\n",
            "        [ 0.1664, -0.1185],\n",
            "        [ 0.0932, -0.0125],\n",
            "        [ 0.1847, -0.5254]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r8it [03:24, 25.20s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 0])\n",
            "tensor([[ 0.3086, -0.2972],\n",
            "        [ 0.0768, -0.0030],\n",
            "        [ 0.0855, -0.1346],\n",
            "        [ 0.1347, -0.0476]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r9it [03:49, 25.05s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 0, 0])\n",
            "tensor([[ 0.0066, -0.0775],\n",
            "        [ 0.0656, -0.3748],\n",
            "        [ 0.1085, -0.1194],\n",
            "        [-0.0094, -0.1155]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r10it [04:13, 24.98s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 1, 0, 1])\n",
            "tensor([[ 0.1523, -0.1351],\n",
            "        [ 0.2047, -0.1684],\n",
            "        [ 0.1059, -0.0359],\n",
            "        [ 0.0307, -0.1503]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r11it [04:38, 24.86s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 0, 1])\n",
            "tensor([[ 0.0867,  0.0462],\n",
            "        [ 0.0015, -0.0988],\n",
            "        [ 0.0197, -0.2175],\n",
            "        [ 0.5077, -0.3647]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r12it [05:03, 25.06s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 1, 0, 1])\n",
            "tensor([[ 0.0488, -0.0951],\n",
            "        [ 0.0866, -0.2022],\n",
            "        [ 0.1419, -0.1229],\n",
            "        [ 0.3027, -0.1216]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r13it [05:29, 25.33s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 1, 1, 0])\n",
            "tensor([[ 0.0477, -0.0653],\n",
            "        [ 0.0711, -0.2269],\n",
            "        [ 0.1515, -0.0987],\n",
            "        [ 0.2920, -0.3034]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r14it [05:54, 25.20s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 1, 1, 0])\n",
            "tensor([[ 0.1661, -0.0731],\n",
            "        [ 0.0243, -0.1517],\n",
            "        [ 0.0518, -0.0359],\n",
            "        [ 0.1680, -0.1110]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r15it [06:19, 25.06s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 1, 1, 0])\n",
            "tensor([[ 0.2351, -0.3496],\n",
            "        [ 0.0342, -0.2218],\n",
            "        [ 0.0667, -0.1112],\n",
            "        [ 0.1007, -0.1497]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r16it [06:44, 25.02s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 1, 1])\n",
            "tensor([[-0.0171, -0.1488],\n",
            "        [ 0.0878, -0.1672],\n",
            "        [ 0.3193, -0.3165],\n",
            "        [ 0.0919, -0.1961]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r17it [07:09, 24.99s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 1, 0])\n",
            "tensor([[ 0.2676, -0.6227],\n",
            "        [ 0.0370, -0.1381],\n",
            "        [ 0.2485, -0.3692],\n",
            "        [ 0.0167, -0.2518]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r18it [07:34, 24.97s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 0, 1])\n",
            "tensor([[ 0.1710, -0.1391],\n",
            "        [ 0.1095, -0.1072],\n",
            "        [ 0.0954, -0.0586],\n",
            "        [ 0.1480,  0.0171]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r19it [07:59, 24.98s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 1, 0])\n",
            "tensor([[ 0.1711, -0.1664],\n",
            "        [-0.0509, -0.1031],\n",
            "        [ 0.1473, -0.2237],\n",
            "        [ 0.1470, -0.1487]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r20it [08:24, 24.92s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 0])\n",
            "tensor([[ 0.1555, -0.2076],\n",
            "        [ 0.1115, -0.1562],\n",
            "        [ 0.4687, -0.2718],\n",
            "        [ 0.1503, -0.1148]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r21it [08:49, 24.92s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 1, 0])\n",
            "tensor([[ 0.0418, -0.0277],\n",
            "        [ 0.0425, -0.1633],\n",
            "        [ 0.0668, -0.0250],\n",
            "        [-0.0169, -0.1239]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r22it [09:13, 24.77s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 1, 0])\n",
            "tensor([[ 0.0788, -0.0346],\n",
            "        [ 0.1323, -0.2023],\n",
            "        [ 0.1388, -0.1602],\n",
            "        [ 0.4617, -0.3709]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r23it [09:38, 24.81s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 0])\n",
            "tensor([[ 0.0904, -0.1480],\n",
            "        [ 0.0740, -0.2221],\n",
            "        [ 0.0948, -0.2351],\n",
            "        [ 0.0930, -0.1854]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r24it [10:03, 24.85s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 1, 0, 0])\n",
            "tensor([[ 0.2306, -0.1251],\n",
            "        [ 0.0539, -0.1857],\n",
            "        [ 0.1234, -0.0881],\n",
            "        [ 0.0051, -0.0960]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r25it [10:28, 24.84s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 1, 1, 1])\n",
            "tensor([[ 0.2411, -0.5604],\n",
            "        [ 0.0319, -0.3903],\n",
            "        [ 0.0219, -0.0605],\n",
            "        [ 0.0236, -0.2465]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r26it [10:52, 24.84s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 1, 0])\n",
            "tensor([[ 0.5691, -0.4681],\n",
            "        [ 0.3320, -0.6274],\n",
            "        [ 0.0572, -0.1001],\n",
            "        [ 0.2192, -0.3596]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r27it [11:17, 24.87s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 1, 0, 0])\n",
            "tensor([[ 0.2144, -0.2396],\n",
            "        [ 0.1864, -0.1334],\n",
            "        [ 0.1259, -0.1335],\n",
            "        [ 0.0323, -0.2212]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r28it [11:42, 24.90s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 1, 0])\n",
            "tensor([[ 0.0712, -0.0962],\n",
            "        [ 0.1399, -0.0630],\n",
            "        [ 0.1242, -0.1416],\n",
            "        [ 0.0969, -0.1673]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r29it [12:07, 24.90s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 1, 0, 0])\n",
            "tensor([[ 0.0332, -0.0591],\n",
            "        [ 0.0550, -0.0677],\n",
            "        [ 0.1113, -0.2308],\n",
            "        [ 0.0884, -0.0866]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r30it [12:32, 24.92s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 0, 0])\n",
            "tensor([[ 0.0963, -0.1192],\n",
            "        [ 0.0792, -0.1191],\n",
            "        [ 0.0551, -0.0783],\n",
            "        [ 0.1648, -0.0877]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r31it [12:57, 24.91s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 0, 0])\n",
            "tensor([[ 0.2107, -0.3650],\n",
            "        [ 0.1298, -0.1221],\n",
            "        [ 0.0966, -0.1293],\n",
            "        [ 0.1462, -0.1924]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r32it [13:22, 24.95s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 1])\n",
            "tensor([[ 0.0867, -0.0602],\n",
            "        [ 0.0694, -0.0725],\n",
            "        [ 0.1335, -0.0760],\n",
            "        [ 0.1486, -0.2892]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r33it [13:47, 24.94s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 1, 0])\n",
            "tensor([[ 0.0858, -0.1592],\n",
            "        [ 0.0267, -0.1128],\n",
            "        [ 0.0334, -0.2268],\n",
            "        [ 0.1405, -0.0423]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r34it [14:12, 24.89s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 1, 1, 0])\n",
            "tensor([[ 0.1684, -0.0908],\n",
            "        [ 0.1252, -0.2729],\n",
            "        [ 0.1614, -0.1769],\n",
            "        [ 0.1490, -0.1462]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r35it [14:37, 24.91s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 0, 0])\n",
            "tensor([[ 0.0969, -0.2317],\n",
            "        [ 0.1961, -0.1572],\n",
            "        [ 0.4279, -0.2795],\n",
            "        [ 0.1080, -0.1478]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r36it [15:01, 24.78s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 0])\n",
            "tensor([[ 0.1553, -0.6028],\n",
            "        [ 0.1742, -0.1214],\n",
            "        [ 0.0434, -0.1958],\n",
            "        [ 0.0824, -0.2086]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r37it [15:29, 25.53s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 1, 1])\n",
            "tensor([[ 0.0987, -0.1591],\n",
            "        [ 0.1351, -0.1481],\n",
            "        [ 0.0892, -0.2031],\n",
            "        [ 0.1670, -0.1641]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r38it [15:54, 25.54s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 1])\n",
            "tensor([[ 0.1667, -0.1602],\n",
            "        [ 0.1510, -0.1435],\n",
            "        [ 0.1239, -0.1087],\n",
            "        [ 0.2115, -0.2129]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r39it [16:19, 25.39s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 1])\n",
            "tensor([[ 0.2862, -0.1813],\n",
            "        [ 0.1795, -0.2078],\n",
            "        [ 0.2586, -0.2776],\n",
            "        [ 0.1418, -0.2733]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r40it [16:44, 25.22s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 1, 0, 0])\n",
            "tensor([[ 0.0806, -0.1479],\n",
            "        [ 0.1630, -0.1626],\n",
            "        [ 0.2514, -0.2217],\n",
            "        [ 0.4460, -0.5770]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r41it [17:09, 25.11s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 0])\n",
            "tensor([[ 0.1654, -0.2039],\n",
            "        [ 0.2631, -0.2496],\n",
            "        [ 0.1839, -0.0545],\n",
            "        [ 0.0652, -0.1645]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r42it [17:34, 25.08s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 1, 1])\n",
            "tensor([[ 0.1252, -0.2108],\n",
            "        [ 0.3018, -0.2242],\n",
            "        [ 0.4545, -0.7383],\n",
            "        [ 0.1214, -0.1823]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r43it [17:59, 25.09s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 0, 0])\n",
            "tensor([[ 0.1995, -0.2096],\n",
            "        [ 0.0978, -0.2705],\n",
            "        [ 0.1600, -0.1707],\n",
            "        [ 0.2464, -0.3601]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r44it [18:24, 25.09s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 1, 1])\n",
            "tensor([[ 0.1948, -0.2253],\n",
            "        [ 0.2293, -0.2149],\n",
            "        [ 0.0939, -0.2307],\n",
            "        [ 0.1493, -0.2497]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r45it [18:49, 25.09s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 1, 0, 1])\n",
            "tensor([[ 0.1719, -0.2806],\n",
            "        [ 0.1145, -0.2703],\n",
            "        [ 0.1995, -0.1851],\n",
            "        [ 0.2609, -0.2336]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r46it [19:14, 25.00s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 1, 1])\n",
            "tensor([[ 0.5329, -0.7567],\n",
            "        [ 0.2097, -0.0923],\n",
            "        [ 0.0909, -0.2409],\n",
            "        [ 0.1192, -0.1597]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r47it [19:39, 25.02s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 1, 1, 1])\n",
            "tensor([[ 0.0478, -0.0752],\n",
            "        [ 0.1602, -0.2265],\n",
            "        [ 0.1934, -0.3189],\n",
            "        [ 0.2183, -0.1449]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r48it [20:04, 25.04s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 1])\n",
            "tensor([[ 0.1866, -0.1692],\n",
            "        [ 0.2217, -0.2307],\n",
            "        [ 0.0768, -0.6062],\n",
            "        [ 0.3275, -0.0954]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r49it [20:29, 25.06s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 0, 0, 1])\n",
            "tensor([[ 0.1537, -0.1738],\n",
            "        [ 0.1559, -0.1279],\n",
            "        [ 0.1180, -0.2210],\n",
            "        [ 0.2155, -0.2077]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r50it [20:54, 25.08s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 1, 0, 0])\n",
            "tensor([[ 0.1450, -0.1965],\n",
            "        [ 0.0536, -0.0902],\n",
            "        [ 0.1240, -0.1906],\n",
            "        [ 0.2105, -0.2088]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r51it [21:19, 25.06s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 0, 0])\n",
            "tensor([[ 0.3262, -0.7345],\n",
            "        [ 0.1544, -0.3238],\n",
            "        [ 0.0787, -0.2087],\n",
            "        [ 0.1888, -0.1014]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r52it [21:44, 25.07s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([1, 1, 0, 0])\n",
            "tensor([[ 0.1642, -0.2003],\n",
            "        [ 0.2495, -0.2742],\n",
            "        [ 0.1524, -0.2070],\n",
            "        [ 0.1697, -0.2760]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r53it [22:10, 25.07s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0, 0, 0])\n",
            "tensor([[ 0.2456, -0.3553],\n",
            "        [ 0.1792, -0.2388],\n",
            "        [ 0.0980, -0.1402]], grad_fn=<AddmmBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "54it [22:29, 24.99s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The Total Accuracy for Epoch 0: 60.93023255813954\n",
            "Training Loss Epoch: 0.6762156707269175\n",
            "Training Accuracy Epoch: 60.93023255813954\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "EPOCHS = 1\n",
        "for epoch in range(EPOCHS):\n",
        "    train(epoch)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r3RVJCnHzXXN"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "from sklearn.metrics import confusion_matrix"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aGmv2NMUJNDE"
      },
      "outputs": [],
      "source": [
        "def valid(model, testing_loader):\n",
        "    model.eval()\n",
        "    n_correct = 0; n_wrong = 0; total = 0; tr_loss=0; nb_tr_steps=0; nb_tr_examples=0;\n",
        "    preds = np.empty((54));### size of the validation set\n",
        "    out_label_ids = np.empty((54))### size of the validation set\n",
        "    n_batches = len(testing_loader)\n",
        "    with torch.no_grad():\n",
        "        for i, data in tqdm(enumerate(testing_loader, 0)):\n",
        "            ids = data['ids'].to(device, dtype = torch.long)\n",
        "            mask = data['mask'].to(device, dtype = torch.long)\n",
        "            token_type_ids = data['token_type_ids'].to(device, dtype=torch.long)\n",
        "            targets = data['targets'].to(device, dtype = torch.long)\n",
        "            #print(targets)## 1 D tensor of actual targets\n",
        "            outputs = model(ids, mask, token_type_ids).squeeze()\n",
        "            #print(outputs.data)## 2 D tensor of logits \n",
        "            loss = loss_function(outputs, targets)\n",
        "            tr_loss += loss.item()\n",
        "            #big_idx return the index of the \n",
        "            big_val, big_idx = torch.max(outputs.data, dim=1)## returns the position of the highest probability\n",
        "            #print(big_idx)\n",
        "            n_correct += calcuate_accuracy(big_idx, targets)### wherever the index matches\n",
        "            \n",
        "            nb_tr_steps += 1\n",
        "            nb_tr_examples+=targets.size(0)\n",
        "            start_index=4*i\n",
        "            end_index = start_index + 4 if i != (n_batches - 1) else 54\n",
        "            preds[start_index:end_index] = big_idx.cpu().detach().numpy()\n",
        "            print(preds)\n",
        "            out_label_ids[start_index:end_index] = targets.cpu().detach().numpy()\n",
        "            print(out_label_ids)\n",
        "\n",
        "            if i%20==0:\n",
        "                loss_step = tr_loss/nb_tr_steps\n",
        "                accu_step = (n_correct*100)/nb_tr_examples\n",
        "                #print(f\"Validation Loss per 20 steps: {loss_step}\")\n",
        "                #print(f\"Validation Accuracy per 20 steps: {accu_step}\")\n",
        "            \n",
        "    epoch_loss = tr_loss/nb_tr_steps\n",
        "    print(confusion_matrix(out_label_ids, preds, labels=[0, 1]))\n",
        "    # print('confusion matrix:')\n",
        "    # print('            predicted    ')\n",
        "    # print('          0     |     1')\n",
        "    # print('    ----------------------')\n",
        "    # print('   0 | ',format(tn,'5d'),' | ',format(fp,'5d'))\n",
        "    # print('gt -----------------------')\n",
        "    # print('   1 | ',format(fn,'5d'),' | ',format(tp,'5d'))\n",
        "    # print('---------------------------------------------------')\n",
        "    epoch_accu = (n_correct*100)/nb_tr_examples\n",
        "    preds= torch.max(outputs.data, dim=1)\n",
        "    print(f\"Validation Loss Epoch: {epoch_loss}\")\n",
        "    print(f\"Validation Accuracy Epoch: {epoch_accu}\")\n",
        "    \n",
        "    return epoch_accu"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G1FXU6iXJr6L",
        "outputId": "e1f64ecf-ac8c-4141-fda1-92897939de2b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2269: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "1it [00:00,  2.70it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 7.905e-323 9.881e-323\n",
            " 1.136e-322 1.729e-322 2.075e-322 2.520e-322 2.668e-322 2.717e-322\n",
            " 3.014e-322 3.458e-322 3.755e-322 3.903e-322 4.200e-322 4.397e-322\n",
            " 4.447e-322 4.694e-322 4.891e-322 5.188e-322 5.237e-322 5.731e-322\n",
            " 5.929e-322 6.324e-322 6.373e-322 6.472e-322 6.868e-322 7.065e-322\n",
            " 7.411e-322 7.510e-322 7.856e-322 7.905e-322 7.954e-322 8.547e-322\n",
            " 8.893e-322 9.041e-322 9.091e-322 9.140e-322 9.239e-322 9.338e-322\n",
            " 9.782e-322 9.881e-322 9.931e-322 1.013e-321 1.018e-321 1.082e-321\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0.e+000 0.e+000 1.e+000 0.e+000 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324\n",
            " 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324\n",
            " 5.e-324 5.e-324 5.e-324 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r2it [00:00,  2.96it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 2.075e-322 2.520e-322 2.668e-322 2.717e-322\n",
            " 3.014e-322 3.458e-322 3.755e-322 3.903e-322 4.200e-322 4.397e-322\n",
            " 4.447e-322 4.694e-322 4.891e-322 5.188e-322 5.237e-322 5.731e-322\n",
            " 5.929e-322 6.324e-322 6.373e-322 6.472e-322 6.868e-322 7.065e-322\n",
            " 7.411e-322 7.510e-322 7.856e-322 7.905e-322 7.954e-322 8.547e-322\n",
            " 8.893e-322 9.041e-322 9.091e-322 9.140e-322 9.239e-322 9.338e-322\n",
            " 9.782e-322 9.881e-322 9.931e-322 1.013e-321 1.018e-321 1.082e-321\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0.e+000 0.e+000 1.e+000 0.e+000 0.e+000 0.e+000 1.e+000 1.e+000 5.e-324\n",
            " 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324\n",
            " 5.e-324 5.e-324 5.e-324 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r3it [00:00,  3.14it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 3.014e-322 3.458e-322 3.755e-322 3.903e-322 4.200e-322 4.397e-322\n",
            " 4.447e-322 4.694e-322 4.891e-322 5.188e-322 5.237e-322 5.731e-322\n",
            " 5.929e-322 6.324e-322 6.373e-322 6.472e-322 6.868e-322 7.065e-322\n",
            " 7.411e-322 7.510e-322 7.856e-322 7.905e-322 7.954e-322 8.547e-322\n",
            " 8.893e-322 9.041e-322 9.091e-322 9.140e-322 9.239e-322 9.338e-322\n",
            " 9.782e-322 9.881e-322 9.931e-322 1.013e-321 1.018e-321 1.082e-321\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0.e+000 0.e+000 1.e+000 0.e+000 0.e+000 0.e+000 1.e+000 1.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324 5.e-324\n",
            " 5.e-324 5.e-324 5.e-324 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r4it [00:01,  3.26it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 4.200e-322 4.397e-322\n",
            " 4.447e-322 4.694e-322 4.891e-322 5.188e-322 5.237e-322 5.731e-322\n",
            " 5.929e-322 6.324e-322 6.373e-322 6.472e-322 6.868e-322 7.065e-322\n",
            " 7.411e-322 7.510e-322 7.856e-322 7.905e-322 7.954e-322 8.547e-322\n",
            " 8.893e-322 9.041e-322 9.091e-322 9.140e-322 9.239e-322 9.338e-322\n",
            " 9.782e-322 9.881e-322 9.931e-322 1.013e-321 1.018e-321 1.082e-321\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0.e+000 0.e+000 1.e+000 0.e+000 0.e+000 0.e+000 1.e+000 1.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 5.e-324 5.e-324\n",
            " 5.e-324 5.e-324 5.e-324 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r5it [00:01,  3.29it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 4.891e-322 5.188e-322 5.237e-322 5.731e-322\n",
            " 5.929e-322 6.324e-322 6.373e-322 6.472e-322 6.868e-322 7.065e-322\n",
            " 7.411e-322 7.510e-322 7.856e-322 7.905e-322 7.954e-322 8.547e-322\n",
            " 8.893e-322 9.041e-322 9.091e-322 9.140e-322 9.239e-322 9.338e-322\n",
            " 9.782e-322 9.881e-322 9.931e-322 1.013e-321 1.018e-321 1.082e-321\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0.e+000 0.e+000 1.e+000 0.e+000 0.e+000 0.e+000 1.e+000 1.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 1.e+000 1.e+000\n",
            " 0.e+000 0.e+000 5.e-324 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000\n",
            " 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000 0.e+000]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r6it [00:01,  3.38it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 5.929e-322 6.324e-322 6.373e-322 6.472e-322 6.868e-322 7.065e-322\n",
            " 7.411e-322 7.510e-322 7.856e-322 7.905e-322 7.954e-322 8.547e-322\n",
            " 8.893e-322 9.041e-322 9.091e-322 9.140e-322 9.239e-322 9.338e-322\n",
            " 9.782e-322 9.881e-322 9.931e-322 1.013e-321 1.018e-321 1.082e-321\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0. 0. 1. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 1. 0. 1. 0.\n",
            " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
            " 0. 0. 0. 0. 0. 0.]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r7it [00:02,  3.44it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 6.868e-322 7.065e-322\n",
            " 7.411e-322 7.510e-322 7.856e-322 7.905e-322 7.954e-322 8.547e-322\n",
            " 8.893e-322 9.041e-322 9.091e-322 9.140e-322 9.239e-322 9.338e-322\n",
            " 9.782e-322 9.881e-322 9.931e-322 1.013e-321 1.018e-321 1.082e-321\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0. 0. 1. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 1. 0. 1. 0.\n",
            " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
            " 0. 0. 0. 0. 0. 0.]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r8it [00:02,  3.46it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 7.856e-322 7.905e-322 7.954e-322 8.547e-322\n",
            " 8.893e-322 9.041e-322 9.091e-322 9.140e-322 9.239e-322 9.338e-322\n",
            " 9.782e-322 9.881e-322 9.931e-322 1.013e-321 1.018e-321 1.082e-321\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0. 0. 1. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 1. 0. 1. 0.\n",
            " 0. 0. 0. 0. 1. 1. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
            " 0. 0. 0. 0. 0. 0.]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r9it [00:02,  3.47it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 8.893e-322 9.041e-322 9.091e-322 9.140e-322 9.239e-322 9.338e-322\n",
            " 9.782e-322 9.881e-322 9.931e-322 1.013e-321 1.018e-321 1.082e-321\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0. 0. 1. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 1. 0. 1. 0.\n",
            " 0. 0. 0. 0. 1. 1. 1. 1. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
            " 0. 0. 0. 0. 0. 0.]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r10it [00:02,  3.47it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 9.239e-322 9.338e-322\n",
            " 9.782e-322 9.881e-322 9.931e-322 1.013e-321 1.018e-321 1.082e-321\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0. 0. 1. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 1. 0. 1. 0.\n",
            " 0. 0. 0. 0. 1. 1. 1. 1. 0. 1. 1. 0. 0. 1. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.\n",
            " 0. 0. 0. 0. 0. 0.]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r11it [00:03,  3.51it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 9.931e-322 1.013e-321 1.018e-321 1.082e-321\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0. 0. 1. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 1. 0. 1. 0.\n",
            " 0. 0. 0. 0. 1. 1. 1. 1. 0. 1. 1. 0. 0. 1. 0. 1. 1. 0. 1. 0. 0. 0. 0. 0.\n",
            " 0. 0. 0. 0. 0. 0.]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r12it [00:03,  3.50it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 1.141e-321 1.176e-321 1.201e-321 1.240e-321 1.250e-321 1.275e-321]\n",
            "[0. 0. 1. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 1. 0. 1. 0.\n",
            " 0. 0. 0. 0. 1. 1. 1. 1. 0. 1. 1. 0. 0. 1. 0. 1. 1. 0. 1. 0. 1. 1. 0. 0.\n",
            " 0. 0. 0. 0. 0. 0.]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "14it [00:04,  3.49it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000 0.000e+000\n",
            " 0.000e+000 0.000e+000 0.000e+000 0.000e+000 1.250e-321 1.275e-321]\n",
            "[0. 0. 1. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 1. 0. 1. 0.\n",
            " 0. 0. 0. 0. 1. 1. 1. 1. 0. 1. 1. 0. 0. 1. 0. 1. 1. 0. 1. 0. 1. 1. 0. 0.\n",
            " 0. 0. 1. 0. 0. 0.]\n",
            "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
            " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
            " 0. 0. 0. 0. 0. 0.]\n",
            "[0. 0. 1. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 1. 0. 1. 0.\n",
            " 0. 0. 0. 0. 1. 1. 1. 1. 0. 1. 1. 0. 0. 1. 0. 1. 1. 0. 1. 0. 1. 1. 0. 0.\n",
            " 0. 0. 1. 0. 1. 0.]\n",
            "[[33  0]\n",
            " [21  0]]\n",
            "Validation Loss Epoch: 0.6659979777676719\n",
            "Validation Accuracy Epoch: 61.111111111111114\n",
            "Accuracy on test data = 61.11%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "acc = valid(model, testing_loader)\n",
        "print(\"Accuracy on test data = %0.2f%%\" % acc)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mlARExyYJ6WZ"
      },
      "outputs": [],
      "source": [
        "#classifcation matrix"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wnB1KVJYgwls",
        "outputId": "7ffe73c0-2d64-4e7e-bf10-e23fd8f19ff9"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: UserWarning: torch.range is deprecated and will be removed in a future release because its behavior is inconsistent with Python's range builtin. Instead, use torch.arange, which produces values in [start, end).\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ]
        }
      ],
      "source": [
        "a= torch.range(1,16)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c_VIp6rSiyrI",
        "outputId": "4e17ba54-3d5f-4d3d-c7ba-1096cb98ce90"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([[ 1.,  2.,  3.,  4.],\n",
            "        [ 5.,  6.,  7.,  8.],\n",
            "        [ 9., 10., 11., 12.],\n",
            "        [13., 14., 15., 16.]])\n"
          ]
        }
      ],
      "source": [
        "print(a.view(-1,4))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BnizokkSSTls",
        "outputId": "9f868f94-ffd7-4abb-9b6f-369207bf956a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([0.3220, 1.0978, 0.4612], requires_grad=True)\n",
            "tensor([0., 0., 1.])\n",
            "tensor(0.9139, grad_fn=<BinaryCrossEntropyBackward0>)\n"
          ]
        }
      ],
      "source": [
        "m = torch.nn.Sigmoid()\n",
        "loss = torch.nn.BCELoss()\n",
        "input = torch.randn(3, requires_grad=True)\n",
        "print(input)\n",
        "target = torch.empty(3).random_(2)\n",
        "print(target)\n",
        "output = loss(m(input), target)\n",
        "print(output)\n",
        "output.backward()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "Fine_Tuning_Roberta.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "02c38b2cf34347c2847bfcbea1de9727": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0efea41175d1436695a43cb87f6be173": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1b7c389be0f14dddbdd7837485c42158": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dce2e39aa5634bb6a906f90f7e57ccb8",
            "placeholder": "​",
            "style": "IPY_MODEL_de798c677d6f436a8bcb9c14a28801ae",
            "value": "Downloading: 100%"
          }
        },
        "1e5a334875f44fb4a259eac9fd74b33a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_61f7eb32fe2d4a09b1846e2b416e772f",
            "max": 501200538,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d5d23b32668b4ce99ee26d0f7f4e4a9f",
            "value": 501200538
          }
        },
        "234a046ae64f4748b80c4abd91dab67d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2366ee68e48f42c9b0c215a2e793b1e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_655c0f0a9a9a409c9ad629d6d17ff2c2",
              "IPY_MODEL_498990764daa4974b97bc3d35376ab77",
              "IPY_MODEL_644dce8d87654462b2c203f025750022"
            ],
            "layout": "IPY_MODEL_49ea079c791a404ea279ba228cab1b30"
          }
        },
        "28f648efb6dc4157974165e392119db0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2d16a2868c8b43f4be9f2a76e899f946": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2dd3cb60fbbf4b7a9518f07c06f849ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2f82654a33db46739118fa9f6042663f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3239af4dc81a4ba297a85c852bb1ea14": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "32de4063970f4b6a9e6ebad750405fcb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_359d889c7fac4e69b3ffa45dd90d8afe",
            "placeholder": "​",
            "style": "IPY_MODEL_b3f076a092db43fca8b7eeae1e8f45c5",
            "value": " 446k/446k [00:00&lt;00:00, 1.57MB/s]"
          }
        },
        "359d889c7fac4e69b3ffa45dd90d8afe": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3987a1100ba44ef8a97bb7d2a165bce2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3a6270e065d6445eb25f8c44ec244ad6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3cdf594aacfd41cc9339e2d2d647f4d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ba52a5e4ec1d4c819eea841f6edef795",
              "IPY_MODEL_7f55e10cb2c8471c93ffe351fc3edea5",
              "IPY_MODEL_32de4063970f4b6a9e6ebad750405fcb"
            ],
            "layout": "IPY_MODEL_c5ae14ec773d409aaffeedf0a53b0a98"
          }
        },
        "3efeca6a08f642d3a55ac2571702cddf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3a6270e065d6445eb25f8c44ec244ad6",
            "placeholder": "​",
            "style": "IPY_MODEL_b33abc4dcd2243dab866433f9b132cbb",
            "value": " 1.29M/1.29M [00:00&lt;00:00, 1.05MB/s]"
          }
        },
        "49097fb4c3624d3ebe40f114626eeee4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_db4ec8612ebc45d1b39de87e98249777",
              "IPY_MODEL_9ca8e485ab604eecb0a144f8efa5813a",
              "IPY_MODEL_3efeca6a08f642d3a55ac2571702cddf"
            ],
            "layout": "IPY_MODEL_5b1fb891b5524810820878c0797ffe4d"
          }
        },
        "498990764daa4974b97bc3d35376ab77": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3987a1100ba44ef8a97bb7d2a165bce2",
            "max": 898823,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b1e99ee4c0ca4bc094d8a87a6991e14e",
            "value": 898823
          }
        },
        "49ea079c791a404ea279ba228cab1b30": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58028463413546caa0d41e9753a90844": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5b1fb891b5524810820878c0797ffe4d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61f7eb32fe2d4a09b1846e2b416e772f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "644dce8d87654462b2c203f025750022": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2f82654a33db46739118fa9f6042663f",
            "placeholder": "​",
            "style": "IPY_MODEL_cce6f7d2e82e46de87fbeb9ee79e3443",
            "value": " 878k/878k [00:00&lt;00:00, 1.02MB/s]"
          }
        },
        "655c0f0a9a9a409c9ad629d6d17ff2c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_234a046ae64f4748b80c4abd91dab67d",
            "placeholder": "​",
            "style": "IPY_MODEL_c71315440d37416ebf42f39ce88ae9b8",
            "value": "Downloading: 100%"
          }
        },
        "665ec78e79ce407f9ed92598834887a0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7629f086a297402fb80367053cff853b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7b1847b6d14f4dddab852d89d64f4538",
              "IPY_MODEL_e034b2d598d441a9917ca9cd48e5662e",
              "IPY_MODEL_9756ee46dc234671a1e336787124f864"
            ],
            "layout": "IPY_MODEL_f4b6dbd8b6e34ced9458e4968ae95e69"
          }
        },
        "7b1847b6d14f4dddab852d89d64f4538": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_995ec40aa73d48468f808b2792882d28",
            "placeholder": "​",
            "style": "IPY_MODEL_3239af4dc81a4ba297a85c852bb1ea14",
            "value": "Downloading: 100%"
          }
        },
        "7f55e10cb2c8471c93ffe351fc3edea5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eba32af18ec141cf94381359416eac88",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e5878c6610f6441885846490e59873a2",
            "value": 456318
          }
        },
        "86fb7ccef8c349dc8fd8b899abd74925": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8ce94b136eaf43fd9cd79ca1dc05f212": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1b7c389be0f14dddbdd7837485c42158",
              "IPY_MODEL_1e5a334875f44fb4a259eac9fd74b33a",
              "IPY_MODEL_eaf6b1a0ff834c9095f5536df6b2b9cd"
            ],
            "layout": "IPY_MODEL_b2692eea5ee34d9c80c8fe6859536ce7"
          }
        },
        "9756ee46dc234671a1e336787124f864": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_994db462800c4cdf97fbf04f31b7c9ff",
            "placeholder": "​",
            "style": "IPY_MODEL_e930b25539a246afb1e32ecf58ba0b90",
            "value": " 481/481 [00:00&lt;00:00, 10.7kB/s]"
          }
        },
        "994db462800c4cdf97fbf04f31b7c9ff": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "995ec40aa73d48468f808b2792882d28": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ca8e485ab604eecb0a144f8efa5813a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2d16a2868c8b43f4be9f2a76e899f946",
            "max": 1355863,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_86fb7ccef8c349dc8fd8b899abd74925",
            "value": 1355863
          }
        },
        "b1e99ee4c0ca4bc094d8a87a6991e14e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b2692eea5ee34d9c80c8fe6859536ce7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b33abc4dcd2243dab866433f9b132cbb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b3f076a092db43fca8b7eeae1e8f45c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b574facc5455476187054586bec3ff75": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ba52a5e4ec1d4c819eea841f6edef795": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_02c38b2cf34347c2847bfcbea1de9727",
            "placeholder": "​",
            "style": "IPY_MODEL_28f648efb6dc4157974165e392119db0",
            "value": "Downloading: 100%"
          }
        },
        "c2336ea003c14f5dafdce6822768286e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c5ae14ec773d409aaffeedf0a53b0a98": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c71315440d37416ebf42f39ce88ae9b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cce6f7d2e82e46de87fbeb9ee79e3443": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d5d23b32668b4ce99ee26d0f7f4e4a9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "db4ec8612ebc45d1b39de87e98249777": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_665ec78e79ce407f9ed92598834887a0",
            "placeholder": "​",
            "style": "IPY_MODEL_58028463413546caa0d41e9753a90844",
            "value": "Downloading: 100%"
          }
        },
        "dce2e39aa5634bb6a906f90f7e57ccb8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de798c677d6f436a8bcb9c14a28801ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e034b2d598d441a9917ca9cd48e5662e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0efea41175d1436695a43cb87f6be173",
            "max": 481,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2dd3cb60fbbf4b7a9518f07c06f849ea",
            "value": 481
          }
        },
        "e5878c6610f6441885846490e59873a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e930b25539a246afb1e32ecf58ba0b90": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "eaf6b1a0ff834c9095f5536df6b2b9cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2336ea003c14f5dafdce6822768286e",
            "placeholder": "​",
            "style": "IPY_MODEL_b574facc5455476187054586bec3ff75",
            "value": " 478M/478M [00:17&lt;00:00, 33.1MB/s]"
          }
        },
        "eba32af18ec141cf94381359416eac88": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f4b6dbd8b6e34ced9458e4968ae95e69": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}