chapter5/Chapter_5_new_api__temp.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Chapter_5_new_api__temp.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fz9NHzq5O46y"
      },
      "source": [
        "## First trial to harmonize twitter classification in chapter 5 with new torchtext api\n",
        "\n",
        "\n",
        "\n",
        "Some parts are taken from:\n",
        "\n",
        "https://colab.research.google.com/github/pytorch/tutorials/blob/gh-pages/_downloads/27bd42079e7f46673b53e90153168529/text_sentiment_ngrams_tutorial.ipynb\n",
        "\n",
        "\n",
        "Open issues:\n",
        "\n",
        "1.   use class OurFirstLSTM instead of class TextClassificationModel (LSTM missing, offsets have to be included in LSTM, or it's done before in collate(), and change of optimizer used) \n",
        "2.   padding/fields class process\n",
        "3.   Vocab size\n",
        "4.   dynamic no. of classes\n",
        "5.   vocab vs. Vocab\n",
        "6.   extensive testing\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jxRnnSt-QCfJ",
        "outputId": "46dd8ff8-4905-4dd4-b3c0-d8204c0372bb"
      },
      "source": [
        "!pip install torchtext==0.10.0\n",
        "!pip install torch==1.9.0"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: torchtext==0.10.0 in /usr/local/lib/python3.7/dist-packages (0.10.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchtext==0.10.0) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchtext==0.10.0) (2.23.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from torchtext==0.10.0) (4.62.3)\n",
            "Requirement already satisfied: torch==1.9.0 in /usr/local/lib/python3.7/dist-packages (from torchtext==0.10.0) (1.9.0+cu102)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.9.0->torchtext==0.10.0) (3.7.4.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.10.0) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.10.0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.10.0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.10.0) (2.10)\n",
            "Requirement already satisfied: torch==1.9.0 in /usr/local/lib/python3.7/dist-packages (1.9.0+cu102)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.9.0) (3.7.4.3)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OfELh6nfTvFY",
        "outputId": "b63abb96-9196-4a56-fe10-442375eb5592"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z6JQyeKQTyfM"
      },
      "source": [
        "import torch \n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import numpy as np\n",
        "import torchtext\n",
        "import pandas as pd\n",
        "import spacy\n",
        "\n",
        "from collections import Counter\n",
        "from pathlib import Path\n",
        "from torch.utils.data import DataLoader\n",
        "from torchtext.data.datasets_utils import (\n",
        "    _RawTextIterableDataset,\n",
        "    _wrap_split_argument,\n",
        "    _add_docstring_header,\n",
        "    _create_data_from_csv,\n",
        ")\n",
        "from torchtext.data.functional import to_map_style_dataset\n",
        "from torchtext.data.utils import get_tokenizer\n",
        "from torchtext.vocab import vocab, build_vocab_from_iterator # Vocab?\n",
        "\n",
        "\n",
        "SAMPLESIZE = 10000"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j4V3gcorfVi_",
        "outputId": "9a9c4435-286c-41de-a200-ce4a240b2c55"
      },
      "source": [
        "tweetsDF = pd.read_csv(\"/content/drive/My Drive/Colab Notebooks/beginners-pytorch-deep-learning/chapter5/training.1600000.processed.noemoticon.csv\", \n",
        "                       engine=\"python\", header=None)\n",
        "tweetsDF[\"sentiment_cat\"] = tweetsDF[0].astype('category')\n",
        "tweetsDF[\"sentiment\"] = tweetsDF[\"sentiment_cat\"].cat.codes\n",
        "tweetsDF.to_csv(\"/content/drive/My Drive/Colab Notebooks/beginners-pytorch-deep-learning/chapter5/train-processed.csv\", header=None, index=None)   \n",
        "tweetsDF.columns"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index([0, 1, 2, 3, 4, 5, 'sentiment_cat', 'sentiment'], dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ERaFSEnmgGNv"
      },
      "source": [
        "# Drop irrelevant columns and safe as a new df & csv file\n",
        "\n",
        "tweetsDF_ = pd.DataFrame(list(zip(tweetsDF[\"sentiment\"], tweetsDF[5])))\n",
        "tweetsDF_.sample(SAMPLESIZE).to_csv(\"/content/drive/My Drive/Colab Notebooks/beginners-pytorch-deep-learning/chapter5/train-processed-sample-newapi.csv\", header=None, index=None) "
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IHApOdJ-UTPf"
      },
      "source": [
        "# Creating the Dataset with torchtext.data.datasets_utils instead of deprecated data classes (also possible to define own Dataset class)\n",
        "\n",
        "DATASET_PATH=\"/content/drive/My Drive/Colab Notebooks/beginners-pytorch-deep-learning/chapter5/train-processed-sample-newapi.csv\"\n",
        "\n",
        "DATASET_NAME = \"Sample_sentiment_csv\"\n",
        "\n",
        "NUM_LINES = {\n",
        "    'train': 0.8*SAMPLESIZE,\n",
        "    'valid': 0.1*SAMPLESIZE,\n",
        "    'test': 0.1*SAMPLESIZE    \n",
        "}\n",
        "\n",
        "\n",
        "@_add_docstring_header(num_lines=NUM_LINES, num_classes=2)\n",
        "@_wrap_split_argument(('train', 'valid', 'test'))\n",
        "def SENTIMENT_CSV(root, split):\n",
        "    return _RawTextIterableDataset(DATASET_NAME, NUM_LINES[split],\n",
        "                                   _create_data_from_csv(DATASET_PATH))"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fukRuNwhUTUm"
      },
      "source": [
        "# Split dataset iterators & convert to map style datasets\n",
        "\n",
        "train_iter, val_iter, test_iter = SENTIMENT_CSV(split=('train', 'valid', 'test'))\n",
        "\n",
        "train_data = to_map_style_dataset(train_iter)\n",
        "val_data = to_map_style_dataset(val_iter)\n",
        "test_data = to_map_style_dataset(test_iter)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JRVEWzXc-4bL"
      },
      "source": [
        "# Build tokenized vocab\n",
        "\n",
        "def yield_tokens(data_iter):\n",
        "    for _, text in data_iter:\n",
        "        yield tokenizer(text)\n",
        "        \n",
        "\n",
        "tokenizer = get_tokenizer('spacy', language='en_core_web_sm')\n",
        "vocab = build_vocab_from_iterator(yield_tokens(train_data), specials=[\"<unk>\"])\n",
        "vocab.set_default_index(vocab[\"<unk>\"])"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F1HI2lny--5m",
        "outputId": "a6dadbf0-819c-4ae9-e946-9de21cf03d2b"
      },
      "source": [
        "vocab(['here', 'is', 'an', 'example'])"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[113, 14, 94, 14375]"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-TB5DFz9-_Zj"
      },
      "source": [
        "# Create text & label pipeline\n",
        "\n",
        "text_pipeline = lambda x: vocab(tokenizer(x))\n",
        "label_pipeline = lambda x: int(x)"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9sLwF789_R_B"
      },
      "source": [
        "# Set up Dataloader (with collate_batch - padding still missing, maybe adding another dimension instead of offsets)\n",
        "\n",
        "BATCH_SIZE = 32\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "\n",
        "def collate_batch(batch):\n",
        "    label_list, text_list, offsets = [], [], [0]\n",
        "    for (_label, _text) in batch:\n",
        "         label_list.append(label_pipeline(_label))\n",
        "         processed_text = torch.tensor(text_pipeline(_text), dtype=torch.int64)\n",
        "         text_list.append(processed_text)\n",
        "         offsets.append(processed_text.size(0))\n",
        "    label_list = torch.tensor(label_list, dtype=torch.int64)\n",
        "    offsets = torch.tensor(offsets[:-1]).cumsum(dim=0)\n",
        "    text_list = torch.cat(text_list)\n",
        "    return label_list.to(device), text_list.to(device), offsets.to(device)\n",
        "\n",
        "\n",
        "train_data_loader = DataLoader(train_data, batch_size=BATCH_SIZE, \n",
        "                               shuffle=False, collate_fn=collate_batch)\n",
        "val_data_loader  = DataLoader(val_data, batch_size=BATCH_SIZE, \n",
        "                              shuffle=False, collate_fn=collate_batch) \n",
        "test_data_loader  = DataLoader(test_data, batch_size=BATCH_SIZE, \n",
        "                               shuffle=False, collate_fn=collate_batch) "
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hpx-gQfYJppN"
      },
      "source": [
        "# Use of TextClassificationModel (to be changed, since no use of LSTM due to offsets)\n",
        "\n",
        "class TextClassificationModel(nn.Module):\n",
        "\n",
        "    def __init__(self, vocab_size, embed_dim, num_class):\n",
        "        super(TextClassificationModel, self).__init__()\n",
        "        self.embedding = nn.EmbeddingBag(vocab_size, embed_dim, sparse=True)\n",
        "        self.fc = nn.Linear(embed_dim, num_class)\n",
        "        self.init_weights()\n",
        "\n",
        "    def init_weights(self):\n",
        "        initrange = 0.5\n",
        "        self.embedding.weight.data.uniform_(-initrange, initrange)\n",
        "        self.fc.weight.data.uniform_(-initrange, initrange)\n",
        "        self.fc.bias.data.zero_()\n",
        "\n",
        "    def forward(self, text, offsets):\n",
        "        embedded = self.embedding(text, offsets)\n",
        "        return self.fc(embedded)\n",
        "\n",
        "model = TextClassificationModel(20002, 300, 2).to(device)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R0VQnYVeAMA4"
      },
      "source": [
        "# Define adjusted train function\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "def train(epochs, model, optimizer, criterion, train_iterator, valid_iterator):\n",
        "    for epoch in range(1, epochs+1):\n",
        "     \n",
        "        training_loss = 0.0\n",
        "        valid_loss = 0.0\n",
        "        model.train()\n",
        "        for batch_idx, (label, tweet, offsets) in enumerate(train_iterator):\n",
        "            optimizer.zero_grad()\n",
        "            predict = model(tweet, offsets)\n",
        "            loss = criterion(predict, label)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "            training_loss += loss.data.item() * tweet.size(0)\n",
        "        training_loss /= len(train_iterator.dataset)\n",
        " \n",
        "        \n",
        "        model.eval()\n",
        "        for batch_idx, (label, tweet, offsets) in enumerate(valid_iterator):\n",
        "            predict = model(tweet, offsets)\n",
        "            loss = criterion(predict, label)\n",
        "            valid_loss += loss.data.item() * tweet.size(0)\n",
        " \n",
        "        valid_loss /= len(valid_iterator.dataset) #\n",
        "        print('Epoch: {}, Training Loss: {:.2f}, Validation Loss: {:.2f}'.format(epoch, training_loss, valid_loss))"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4UWBXXSmAMLY",
        "outputId": "81d2f443-5d80-4786-906c-24671154fa71"
      },
      "source": [
        "# take different optimizer, since Adam does not work properly\n",
        "\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=2e-2)\n",
        "\n",
        "train(5, model, optimizer, criterion, train_data_loader, val_data_loader) "
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1, Training Loss: 12.28, Validation Loss: 11.71\n",
            "Epoch: 2, Training Loss: 11.62, Validation Loss: 11.22\n",
            "Epoch: 3, Training Loss: 11.31, Validation Loss: 10.94\n",
            "Epoch: 4, Training Loss: 11.12, Validation Loss: 10.75\n",
            "Epoch: 5, Training Loss: 10.97, Validation Loss: 10.60\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "csABR0JGLdxI",
        "outputId": "601fca79-6e7b-4562-9b54-6b2c3b6218fd"
      },
      "source": [
        "# Classify tweet (example)\n",
        "\n",
        "categories = {0: \"Negative\", 1:\"Positive\"}\n",
        "\n",
        "def predict(text, text_pipeline):\n",
        "    with torch.no_grad():\n",
        "        text = torch.tensor(text_pipeline(text))\n",
        "        text = text.to(device)\n",
        "        output = model(text, torch.tensor([0]).to(device))\n",
        "        return output.argmax(1).item()\n",
        "\n",
        "categories[predict('This is an awesome burger!\"', text_pipeline)]"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Positive'"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    }
  ]
}