NLP_Paper_Judge_Sections.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP Paper Judge Sections.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lym_glPd23n7"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "61KeHP4iDHOc"
      },
      "source": [
        "%cd /content/\n",
        "!git clone -b reduced-data https://github.com/westphal-jan/peer-data\n",
        "%cd /content/peer-data\n",
        "# !git checkout huggingface\n",
        "!git submodule update --init --recursive"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZzxkH1uGFGKb"
      },
      "source": [
        "# !pip install pytorch-lightning wandb python-dotenv catalyst sentence-transformers numpy requests\n",
        "!pip install wandb transformers nltk pytorch-lightning"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W1ludytPTqS4"
      },
      "source": [
        "import os\n",
        "import torch\n",
        "import json\n",
        "import glob\n",
        "from pathlib import Path\n",
        "from tqdm import tqdm\n",
        "# from sklearn.model_selection import train_test_split\n",
        "from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification, Trainer, TrainingArguments, TrainerCallback, TrainerState, TrainerControl, AdamW, AutoTokenizer, AutoModel, AutoModelForSequenceClassification, AutoModelWithLMHead, T5ForConditionalGeneration\n",
        "import wandb\n",
        "from datetime import datetime\n",
        "import pickle\n",
        "import numpy as np\n",
        "# import nlpaug.augmenter.word as naw\n",
        "from torch.utils.data import DataLoader, WeightedRandomSampler\n",
        "from torch import nn, optim\n",
        "import copy\n",
        "from datetime import datetime\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.ticker import PercentFormatter\n",
        "from collections import defaultdict, Counter\n",
        "from nltk.corpus import stopwords\n",
        "import nltk\n",
        "nltk.download(\"stopwords\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5IZktYlUTlqq"
      },
      "source": [
        "class PaperDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]).float() for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx]).float()\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "def raw_read_dataset(data_dir: Path, num_texts=None):\n",
        "    file_paths = glob.glob(f\"{data_dir}/*.json\")\n",
        "    if num_texts != None:\n",
        "        file_paths = file_paths[:num_texts]\n",
        "    raws = []\n",
        "    for i, file_path in enumerate(tqdm(file_paths)):\n",
        "        with open(file_path) as f:\n",
        "            paper_json = json.load(f)\n",
        "            raws.append(paper_json)\n",
        "    return raws\n",
        "\n",
        "def read_dataset(data_dirs, num_texts=None, restrict_file=None):\n",
        "    if not isinstance(data_dirs, list):\n",
        "        data_dirs = [data_dirs]\n",
        "\n",
        "    # with open(restrict_file, \"r\") as f:\n",
        "    #     filter_file_names = f.read().splitlines()\n",
        "    #     for data_dir in data_dirs:\n",
        "    #         file_paths = glob.glob(f\"{data_dir}/*.json\")\n",
        "    #         file_paths = [p for p in file_paths if p.split(\"/\")[-1] in filter_file_names]\n",
        "    #         print(data_dir, len(file_paths))\n",
        "\n",
        "    file_paths = []\n",
        "    for data_dir in data_dirs:\n",
        "        file_paths.extend(glob.glob(f\"{data_dir}/*.json\"))\n",
        "        \n",
        "    if restrict_file:\n",
        "        with open(restrict_file, \"r\") as f:\n",
        "            filter_file_names = f.read().splitlines()\n",
        "            file_paths = [p for p in file_paths if p.split(\"/\")[-1] in filter_file_names]\n",
        "\n",
        "    if num_texts != None:\n",
        "        file_paths = file_paths[:num_texts]\n",
        "    \n",
        "    abstracts = []\n",
        "    sections = []\n",
        "    labels = []\n",
        "    for i, file_path in enumerate(tqdm(file_paths)):\n",
        "        with open(file_path) as f:\n",
        "            paper_json = json.load(f)\n",
        "            accepted = paper_json[\"review\"][\"accepted\"]\n",
        "            abstract = paper_json[\"review\"][\"abstract\"]\n",
        "            _sections = paper_json[\"pdf\"][\"metadata\"][\"sections\"]\n",
        "            _sections = _sections if _sections else []\n",
        "            \n",
        "            abstracts.append(abstract)\n",
        "            labels.append(int(accepted))\n",
        "            sections.append(_sections)\n",
        "    return abstracts, sections, labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ZzdLZ2ppaGw"
      },
      "source": [
        "data_dir = \"data/original\"\n",
        "augmented = [\"data/back-translations-train-accepted\", \"data/back-translations-train-rejected\"]\n",
        "data_dirs = [data_dir]# + augmented\n",
        "#8121, 122194\n",
        "_, train_sections, train_labels = read_dataset(data_dirs, restrict_file=\"data/train.txt\")\n",
        "_, val_sections, val_labels = read_dataset(data_dir, restrict_file=\"data/val.txt\")\n",
        "_, test_sections, test_labels = read_dataset(data_dir, restrict_file=\"data/test.txt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ge-mQB1zS2M0"
      },
      "source": [
        "# def label_distribution(labels):\n",
        "#     num_rejected, num_accepted = labels.count(0), labels.count(1)\n",
        "#     print(num_rejected, num_rejected / len(labels), num_accepted, num_accepted / len(labels))\n",
        "\n",
        "# label_distribution(train_labels)\n",
        "# label_distribution(val_labels)\n",
        "# label_distribution(test_labels)\n",
        "\n",
        "# print(len(np.nonzero(list(map(lambda x: len(x), train_sections)))[0]))\n",
        "# print(len(np.nonzero(list(map(lambda x: len(x), val_sections)))[0]))\n",
        "# print(len(np.nonzero(list(map(lambda x: len(x), test_sections)))[0]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3fQpyNN7X1J7"
      },
      "source": [
        "# num_sections = list(map(lambda x: len(x), train_sections))\n",
        "# num_sections = sorted(num_sections)\n",
        "# len(num_sections)\n",
        "# q = 0.95\n",
        "# index = int(len(num_sections) * q)\n",
        "# percentile = num_sections[index]\n",
        "# print(q, percentile, index, len(num_sections) - index)\n",
        "# print(num_sections[index:])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Phhp-sMYpdss"
      },
      "source": [
        "def extract_sections(sections, labels):\n",
        "    all_sections = []\n",
        "    all_labels = []\n",
        "    assignments = []\n",
        "    section_counter = 0\n",
        "    for _sections, label in zip(sections, labels):\n",
        "        if len(sections) == 0:\n",
        "            continue\n",
        "        texts = list(map(lambda x: x[\"text\"], _sections))\n",
        "        all_sections.extend(texts)\n",
        "        all_labels.extend([label] * len(_sections))\n",
        "\n",
        "        # Create mapping from original submission to flattened sections \n",
        "        new_section_counter = section_counter + len(_sections)\n",
        "        assignments.append((section_counter, new_section_counter))\n",
        "        section_counter = new_section_counter\n",
        "    return all_sections, all_labels, assignments\n",
        "\n",
        "flattened_train_sections, flattened_train_labels, _ = extract_sections(train_sections, train_labels)\n",
        "flattened_val_sections, flattened_val_labels, val_assignemnts = extract_sections(val_sections, val_labels)\n",
        "flattened_test_sections, flattened_test_labels, test_assignemnts = extract_sections(test_sections, test_labels)\n",
        "print(len(flattened_train_sections), len(flattened_val_sections), len(flattened_test_sections))\n",
        "\n",
        "num_accepted, num_rejected = flattened_train_labels.count(1), flattened_train_labels.count(0)\n",
        "print(num_accepted, num_rejected)\n",
        "label_weight = num_rejected / np.array([num_rejected, num_accepted])\n",
        "print(label_weight)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E-B3-ZhNp4Jf"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/paraphrase-TinyBERT-L6-v2')\n",
        "# tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-cased')\n",
        "# tokenizer = AutoTokenizer.from_pretrained('allenai/scibert_scivocab_uncased')\n",
        "train_encodings = tokenizer(flattened_train_sections, truncation=True, padding=\"max_length\", max_length=512)\n",
        "val_encodings = tokenizer(flattened_val_sections, truncation=True, padding=\"max_length\", max_length=512)\n",
        "test_encodings = tokenizer(flattened_test_sections, truncation=True, padding=\"max_length\", max_length=512)\n",
        "\n",
        "# print(np.array(train_encodings[\"input_ids\"]).shape)\n",
        "\n",
        "train_dataset = PaperDataset(train_encodings, flattened_train_labels)\n",
        "val_dataset = PaperDataset(val_encodings, flattened_val_labels)\n",
        "test_dataset = PaperDataset(test_encodings, flattened_test_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NewpWgWk8PPP"
      },
      "source": [
        "# special_characters = set(['-', '\\'', '.', ',', '!','\"','#','$','%','&','(',')','*','+','/',':',';','<','=','>','@','[','\\\\',']','^','`','{','|','}','~','\\t'])\n",
        "# english_stopwords = set(stopwords.words('english'))\n",
        "\n",
        "# def tokenize(text: str, token_to_id=None, encode=False):\n",
        "#     for c in special_characters:\n",
        "#         text = text.replace(c, ' ')\n",
        "#     text = text.lower()\n",
        "#     tokens = [t for t in text.split(' ') if t]\n",
        "#     tokens = [t for t in tokens if t.isalpha()]\n",
        "#     tokens = [t for t in tokens if not t in english_stopwords]\n",
        "#     if token_to_id:\n",
        "#         tokens = [t for t in tokens if t in token_to_id]\n",
        "#         if encode:\n",
        "#             tokens = [token_to_id[t] for t in tokens]\n",
        "#             onehot = np.zeros(len(token_to_id))\n",
        "#             onehot[tokens] = 1\n",
        "#             return onehot\n",
        "#     return tokens\n",
        "\n",
        "# train_tokens = list(map(tokenize, flattened_train_sections))\n",
        "# flattened_tokens = [item for sublist in train_tokens for item in sublist]\n",
        "# vocab = sorted(list(set(flattened_tokens)))\n",
        "# token_to_id = {t: i for i, t in enumerate(vocab)}\n",
        "# val_tokens = list(map(lambda x: tokenize(x, token_to_id), flattened_val_sections))\n",
        "# test_tokens = list(map(lambda x: tokenize(x, token_to_id), flattened_test_sections))\n",
        "# print(\"Vocab Size:\", len(vocab))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q5JJ-iuG8WHM"
      },
      "source": [
        "train_encodings = list(map(lambda x: tokenize(x, token_to_id, True), flattened_train_sections))\n",
        "val_encodings = list(map(lambda x: tokenize(x, token_to_id, True), flattened_val_sections))\n",
        "test_encodings = list(map(lambda x: tokenize(x, token_to_id, True), flattened_test_sections))\n",
        "\n",
        "train_dataset = PaperDataset({\"tokens\": train_encodings}, flattened_train_labels)\n",
        "val_dataset = PaperDataset({\"tokens\": val_encodings}, flattened_val_labels)\n",
        "test_dataset = PaperDataset({\"tokens\": test_encodings}, flattened_test_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "muf4YwxMaeeT"
      },
      "source": [
        "samples_weight = [label_weight[t] for t in flattened_train_labels]\n",
        "samples_weight = torch.Tensor(samples_weight)\n",
        "\n",
        "sampler = WeightedRandomSampler(samples_weight, len(samples_weight))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PgxRi1Rcf2_5"
      },
      "source": [
        "def compute_metrics(logits, labels, prefix=\"eval\"):\n",
        "    # predictions = np.argmax(logits, axis=1)\n",
        "    predictions = np.array(logits) >= 0\n",
        "    actual = np.array(labels)\n",
        "\n",
        "    tp = ((predictions == 1) & (actual == 1)).sum()\n",
        "    fp = ((predictions == 1) & (actual == 0)).sum()\n",
        "    fn = ((predictions == 0) & (actual == 1)).sum()\n",
        "    tn = ((predictions == 0) & (actual == 0)).sum()\n",
        "\n",
        "    precision = tp / (tp + fp) if tp + fp > 0 else 0.0\n",
        "    recall = tp / (tp + fn) if tp + fn > 0 else 0.0\n",
        "    f1 = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0.0\n",
        "    accuracy = (tp + tn) / (tp + tn + fp + fn)\n",
        "    mcc = (tp * tn - fp * fn) / (((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))**0.5)\n",
        "    mcc = mcc if np.isnan(mcc) else 0.0\n",
        "    p = tp + fn\n",
        "    n = tn + fp\n",
        "\n",
        "    metrics = {\"metric/accuracy\": accuracy, \"metric/precision\": precision, \"metric/recall\": recall, \"metric/f1\": f1, \"metric/mcc\": mcc,\n",
        "            \"classification/tp\": tp, \"classification/fp\": fp, \"classification/fn\": fn, \"classification/tn\": tn, \"classification/n\": n, \"classification/p\": p}\n",
        "    metrics = {f\"{prefix}/{key}\": metric for key, metric in metrics.items()}\n",
        "    # \"augmentation/train\": train_dataset.num_augmentations, \"augmentation/val\": val_dataset.num_augmentations\n",
        "    return metrics\n",
        "\n",
        "def compute_original_metrics(logits, original_labels, assignments, prefix=\"eval\"):\n",
        "    assert len(original_labels) == len(assignments)\n",
        "    new_logits = []\n",
        "    for start_idx, end_idx in assignments:\n",
        "        _assignment = np.array(logits[start_idx:end_idx])\n",
        "        # TODO: Sum could also be possible but should only change confidence and not the outcome\n",
        "        reduced_logits = _assignment.mean(axis=0)\n",
        "        new_logits.append(reduced_logits.tolist())\n",
        "    return compute_metrics(new_logits, original_labels, prefix)\n",
        "\n",
        "logits = [[1.0, 2.5], [1.3, 0.7]]\n",
        "labels = [0, 1]\n",
        "\n",
        "result = compute_metrics(logits, labels)\n",
        "print(result)\n",
        "\n",
        "logits = [[1.0, 2.5], [1.3, 0.7], [1.0, 2.5], [1.0, 2.5]]\n",
        "assignments = [(0, 2), (2, 4)]\n",
        "result = compute_original_metrics(logits, labels, assignments)\n",
        "print(result)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bn9V7QNw4Pdv"
      },
      "source": [
        "# class MyModel(nn.Module):\n",
        "#     def __init__(self):\n",
        "#         super().__init__()\n",
        "#         self.base_model = T5ForConditionalGeneration.from_pretrained(\"t5-base\")\n",
        "#         self.classifier = nn.Linear(768, 2, bias=False)\n",
        "    \n",
        "#     def forward(self, x):\n",
        "#         print(x)\n",
        "#         emb = self.base_model(**x)\n",
        "#         print(emb.shape)\n",
        "#         y = self.classifier(emb)\n",
        "#         print(y.shape)\n",
        "#         return y\n",
        "\n",
        "# device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "# model = MyModel()\n",
        "# model.to(device)\n",
        "\n",
        "# loader = DataLoader(train_dataset, batch_size=4, shuffle=False)\n",
        "# for _batch in loader:\n",
        "#     inputs = {key: val.to(device) for key, val in _batch.items()}\n",
        "#     labels = inputs.pop(\"labels\")\n",
        "#     y = model(inputs)\n",
        "#     break"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NTTuzfhv8EWJ"
      },
      "source": [
        "# from klib.misc import kdict\n",
        "\n",
        "# class BowClassifier(nn.Module):\n",
        "#     def __init__(self, input_size, output_size=1):\n",
        "#         super().__init__()\n",
        "#         self.input_size = input_size\n",
        "#         self.output_size = output_size\n",
        "\n",
        "#         self.classifier = nn.Linear(self.input_size, self.output_size, bias=False)\n",
        "\n",
        "#     def forward(self, tokens):\n",
        "#         y = self.classifier(tokens)\n",
        "#         y = y.squeeze()\n",
        "#         return kdict(logits=y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D7iddk0kKS7x"
      },
      "source": [
        "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "\n",
        "# model = BowClassifier(len(vocab)).float()\n",
        "model = AutoModelForSequenceClassification.from_pretrained('sentence-transformers/paraphrase-TinyBERT-L6-v2', num_labels=2)\n",
        "# model = AutoModelForSequenceClassification.from_pretrained('allenai/scibert_scivocab_uncased', num_labels=2)\n",
        "# model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-cased', num_labels=2)\n",
        "model.to(device)\n",
        "\n",
        "train_batch_size, val_batch_size = 64, 64\n",
        "# train_loader = DataLoader(train_dataset, batch_size=train_batch_size, shuffle=True)\n",
        "train_loader = DataLoader(train_dataset, batch_size=train_batch_size, sampler=sampler)\n",
        "val_loader = DataLoader(val_dataset, batch_size=val_batch_size, shuffle=False)\n",
        "\n",
        "# optimizer = AdamW(model.parameters(), lr=5e-5)\n",
        "optimizer = optim.AdamW(model.parameters(), lr=0.001)\n",
        "# _label_weight = torch.from_numpy(label_weight).float().to(device)\n",
        "# loss_func = torch.nn.CrossEntropyLoss(_label_weight, reduction=\"sum\")\n",
        "loss_func = torch.nn.CrossEntropyLoss()\n",
        "\n",
        "pos_weight = torch.tensor(label_weight[1]).float().to(device)\n",
        "weighted_loss_func = nn.BCEWithLogitsLoss(pos_weight=pos_weight)\n",
        "loss_func = nn.BCEWithLogitsLoss()\n",
        "\n",
        "num_epochs = 3\n",
        "\n",
        "wandb_logging = False\n",
        "\n",
        "run_name = datetime.now().strftime('%d-%m-%Y_%H_%M_%S')\n",
        "print(\"Run name:\", run_name)\n",
        "if wandb_logging:\n",
        "    wandb.login()\n",
        "    wandb.init(entity=\"paper-judging\", project=\"huggingface\", name=run_name)\n",
        "\n",
        "output_dir=f'results/{run_name}'\n",
        "os.makedirs(output_dir)\n",
        "# logging_steps, eval_steps = 200, 2000\n",
        "logging_steps, eval_steps = 400, 2000\n",
        "# logging_steps, eval_steps = 5, 10\n",
        "_steps = 0\n",
        "\n",
        "def run_model(_model, _batch):\n",
        "    inputs = {key: val.to(device) for key, val in _batch.items()}\n",
        "    labels = inputs.pop(\"labels\")\n",
        "    outputs = _model(**inputs)\n",
        "    \n",
        "    logits = outputs.logits\n",
        "    loss = loss_func(logits, labels)\n",
        "    return loss, logits\n",
        "\n",
        "min_val_loss, best_model, best_metrics, best_loss_epoch = None, None, None, None\n",
        "best_f1, best_epoch = None, None\n",
        "for epoch in range(num_epochs):\n",
        "    print(f\"Epoch: {epoch + 1}/{num_epochs}\")\n",
        "    train_losses = []\n",
        "    for batch in tqdm(train_loader, position=0, leave=True):\n",
        "        model.train()\n",
        "        optimizer.zero_grad()\n",
        "        train_loss, _ = run_model(model, batch)\n",
        "        train_losses.append(train_loss)\n",
        "        train_loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        metrics = {}\n",
        "        if _steps % logging_steps == 0:\n",
        "            _train_loss = sum(train_losses) / len(train_losses)\n",
        "            metrics.update({\"train/loss\": _train_loss.item(), \"steps\": _steps})\n",
        "            train_losses = []\n",
        "        \n",
        "        if _steps % eval_steps == 0:\n",
        "            print(\"EVAL\")\n",
        "            model.eval()\n",
        "            val_losses = []\n",
        "            logits = []\n",
        "            with torch.no_grad():\n",
        "                for val_batch in val_loader:\n",
        "                    _val_loss, _logits = run_model(model, val_batch)\n",
        "                    val_losses.append(_val_loss.item())\n",
        "                    logits.extend(_logits.tolist())\n",
        "\n",
        "            val_loss = sum(val_losses) / len(val_losses)\n",
        "            _metrics = compute_metrics(logits, flattened_val_labels, \"eval/flattened\")\n",
        "            _original_metrics = compute_original_metrics(logits, val_labels, val_assignemnts, \"eval\")\n",
        "            metrics[\"eval/loss\"] = val_loss\n",
        "            metrics.update(_metrics)\n",
        "            metrics.update(_original_metrics)\n",
        "\n",
        "            if min_val_loss == None:\n",
        "                min_val_loss = val_loss\n",
        "            elif min_val_loss > val_loss:\n",
        "                min_val_loss = val_loss\n",
        "                best_loss_epoch = epoch\n",
        "\n",
        "            f1 = metrics[\"eval/metric/f1\"]\n",
        "            if best_f1 == None:\n",
        "                best_f1 = f1\n",
        "                best_model = copy.deepcopy(model)\n",
        "            elif f1 > best_f1:\n",
        "                best_f1 = f1\n",
        "                best_model = copy.deepcopy(model)\n",
        "                best_metrics = metrics\n",
        "                best_epoch = epoch\n",
        "        \n",
        "        if metrics:\n",
        "            print(metrics) \n",
        "            if wandb_logging:\n",
        "                wandb.log(metrics)\n",
        "\n",
        "        _steps += 1\n",
        "\n",
        "snapshot_dir = f\"results/{run_name}/network-snapshot-latest\"\n",
        "# best_model.save_pretrained(snapshot_dir)\n",
        "\n",
        "print(f\"Best val metrics during training epoch {best_epoch}:\")\n",
        "print(best_metrics)\n",
        "test_loader = DataLoader(test_dataset, batch_size=val_batch_size, shuffle=False)\n",
        "best_model.eval()\n",
        "\n",
        "with torch.no_grad():\n",
        "    logits = []\n",
        "    for val_batch in val_loader:\n",
        "        _, _logits = run_model(best_model, val_batch)\n",
        "        logits.extend(_logits.tolist())\n",
        "    metrics = compute_metrics(logits, flattened_val_labels, \"eval/flattened\")\n",
        "    original_metrics = compute_original_metrics(logits, val_labels, val_assignemnts, \"eval\")\n",
        "    print(\"Val metrics:\")\n",
        "    print(metrics)\n",
        "    print(original_metrics)\n",
        "\n",
        "    logits = []\n",
        "    for test_batch in test_loader:\n",
        "        _, _logits = run_model(best_model, test_batch)\n",
        "        logits.extend(_logits.tolist())\n",
        "    metrics = compute_metrics(logits, flattened_test_labels, \"test/flattened\")\n",
        "    original_metrics = compute_original_metrics(logits, test_labels, test_assignemnts, \"test\")\n",
        "    print(\"Test metrics:\")\n",
        "    print(metrics)\n",
        "    print(original_metrics)\n",
        "\n",
        "if wandb_logging:\n",
        "    wandb.save(f\"{snapshot_dir}/*\", os.path.dirname(snapshot_dir))\n",
        "    wandb.finish()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UW6Q0AmmAE3m"
      },
      "source": [
        "# wandb.finish()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dZPWnjVxU0or"
      },
      "source": [
        "# device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "\n",
        "# model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')\n",
        "# model.to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pOjvyeLkTUBz"
      },
      "source": [
        "# val_batch_size = 16\n",
        "# val_loader = DataLoader(val_dataset, batch_size=val_batch_size, shuffle=False)\n",
        "# _label_weight = torch.from_numpy(label_weight).float().to(device)\n",
        "# loss_func = torch.nn.CrossEntropyLoss(_label_weight, reduction=\"sum\")\n",
        "\n",
        "# model.eval()\n",
        "# val_losses = []\n",
        "# logits = []\n",
        "# with torch.no_grad():\n",
        "#     for val_batch in tqdm(val_loader):\n",
        "#         inputs = {key: val.to(device) for key, val in val_batch.items()}\n",
        "#         _labels = inputs.pop(\"labels\")\n",
        "#         _outputs = model(**inputs)\n",
        "\n",
        "#         _logits = _outputs.logits\n",
        "#         _val_loss = loss_func(_logits, _labels)\n",
        "#         val_losses.append(_val_loss.item())\n",
        "#         logits.extend(_logits.tolist())\n",
        "\n",
        "# val_loss = sum(val_losses) / len(val_losses)\n",
        "# metrics = compute_metrics((logits, val_labels))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M2AZLJF9zFCo"
      },
      "source": [
        "# snapshot_dir = f\"results/{run_name}/network-snapshot-latest\"\n",
        "# model.save_pretrained(snapshot_dir)\n",
        "\n",
        "# if wandb_logging:\n",
        "#     wandb.save(f\"{snapshot_dir}/*\", os.path.dirname(snapshot_dir))\n",
        "#     wandb.finish()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q6gdbuMQsaZT"
      },
      "source": [
        "# !pip install wandb\n",
        "# import wandb\n",
        "# wandb.login()\n",
        "# wandb.finish()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s-2Ct3_vc921"
      },
      "source": [
        "# from transformers import AutoTokenizer, AutoModelWithLMHead\n",
        "\n",
        "# t5_tokenizer = AutoTokenizer.from_pretrained(\"t5-base\")\n",
        "\n",
        "# t5_model = AutoModelWithLMHead.from_pretrained(\"t5-base\").to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Md8S9addYunG"
      },
      "source": [
        "# input = \"That's great\"\n",
        "# input_enc = t5_tokenizer(input, truncation=True, padding=True, return_tensors=\"pt\")\n",
        "# output = t5_model(**input_enc.to(device))\n",
        "# print(output)\n",
        "# print(torch.softmax(output.logits, dim=1))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "odxHty8Cd9Xj"
      },
      "source": [
        "# model_path = f\"network-snapshot-latest-279194.pt\"\n",
        "# model = BowClassifier(len(vocab))\n",
        "# model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu')))\n",
        "\n",
        "# def find_word(word, tokens, labels):\n",
        "#     total = [0, 0]\n",
        "#     classes = [0, 0]\n",
        "#     for i, _tokens in enumerate(tokens):\n",
        "#         _tokens = set(_tokens)\n",
        "#         label = labels[i]\n",
        "#         if word in _tokens:\n",
        "#             classes[label] += 1\n",
        "#         total[label] += 1\n",
        "#     # print(f\"Not accepted: {classes[0]}/{total[0]} ({classes[0]/total[0]}), Accepted: {classes[1]}/{total[1]} ({classes[1]/total[1]})\")\n",
        "#     return classes[0]/total[0], classes[1]/total[1]\n",
        "\n",
        "# def analyze(params, k, tokens, labels):\n",
        "#     val, ind = params.topk(k)\n",
        "#     for i in ind:\n",
        "#         word = vocab[i]\n",
        "#         rejected, accepted = find_word(word, tokens, labels)\n",
        "#         n = 4\n",
        "#         print(f\"{word} & {np.round(params[i].item(), 3)} & {np.round(rejected*100, n)} & {np.round(accepted*100, n)}\")\n",
        "\n",
        "# tokens = train_tokens + val_tokens + test_tokens\n",
        "# print(len(train_tokens), len(val_tokens), len(test_tokens))\n",
        "# labels = flattened_train_labels + flattened_val_labels + flattened_test_labels\n",
        "# print(len(flattened_train_labels), len(flattened_val_labels), len(flattened_test_labels))\n",
        "# params = list(model.parameters())[0][0]\n",
        "# k = 8\n",
        "# print(len(tokens), len(labels))\n",
        "\n",
        "# print(\"Positive:\")\n",
        "# analyze(params, k, tokens, labels)\n",
        "# print()\n",
        "\n",
        "# print(\"Negative:\")\n",
        "# analyze(-params, k, tokens, labels)\n",
        "# print()\n",
        "\n",
        "# print(\"Unnecessary:\")\n",
        "# val, ind = (-(params.abs())).topk(k)\n",
        "# for i in ind:\n",
        "#     print(vocab[i], params[i].item())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iXJP6lSnhSJU"
      },
      "source": [
        "# paper_abstract = \"\"\"Generative adversarial networks (GANs) have shown\n",
        "# outstanding performance on a wide range of problems in\n",
        "# computer vision, graphics, and machine learning, but often require numerous training data and heavy computational resources. To tackle this issue, several methods introduce a transfer learning technique in GAN training. They,\n",
        "# however, are either prone to overfitting or limited to learning small distribution shifts. In this paper, we show that\n",
        "# simple fine-tuning of GANs with frozen lower layers of\n",
        "# the discriminator performs surprisingly well. This simple\n",
        "# baseline, FreezeD, significantly outperforms previous techniques used in both unconditional and conditional GANs.\n",
        "# We demonstrate the consistent effect using StyleGAN and\n",
        "# SNGAN-projection architectures on several datasets of Animal Face, Anime Face, Oxford Flower, CUB-200-2011, and\n",
        "# Caltech-256 datasets. The code and results are available at\n",
        "# https://github.com/sangwoomo/FreezeD.\"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BDqbwDCReV9a"
      },
      "source": [
        "# input = [\"It's incredibly bad\", paper_abstract, \"hello\", \"darkness\"]\n",
        "# input_enc = tokenizer(input, truncation=True, padding=True, return_tensors=\"pt\")\n",
        "# output = model(**input_enc.to(device))\n",
        "# print(output)\n",
        "# # print(torch.softmax(output.logits, dim=1))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ti9x-w_erAaX"
      },
      "source": [
        "# prediction = output.logits.argmax(dim=1)\n",
        "# actual = prediction\n",
        "# n = tp = fp = fn = tn = 0\n",
        "# tp += (prediction == 1) & (actual == 1)\n",
        "# fp += (prediction == 1) & (actual == 0)\n",
        "# fn += (prediction == 0) & (actual == 1)\n",
        "# tn += (prediction == 0) & (actual == 0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dy500R68NdLG"
      },
      "source": [
        "# LIME\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jnEQ1UWBNy5k"
      },
      "source": [
        "# !pip install lime transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N-vIZ4vhNfTX"
      },
      "source": [
        "# import numpy as np\n",
        "# import lime\n",
        "# import torch\n",
        "# import torch.nn.functional as F\n",
        "# from lime.lime_text import LimeTextExplainer\n",
        "\n",
        "# from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
        "\n",
        "# tokenizer = AutoTokenizer.from_pretrained(\"ProsusAI/finbert\")\n",
        "# model = AutoModelForSequenceClassification.from_pretrained(\"ProsusAI/finbert\")\n",
        "# class_names = ['positive','negative', 'neutral']\n",
        "\n",
        "# def predictor(texts):\n",
        "#     print(len(texts))\n",
        "#     outputs = model(**tokenizer(texts, return_tensors=\"pt\", padding=True))\n",
        "#     probas = F.softmax(outputs.logits).detach().numpy()\n",
        "#     return probas\n",
        "\n",
        "# explainer = LimeTextExplainer(class_names=class_names)\n",
        "\n",
        "# str_to_predict = \"surprising increase in revenue in spite of decrease in market share\"\n",
        "# exp = explainer.explain_instance(str_to_predict, predictor, num_features=20, num_samples=100)\n",
        "# exp.show_in_notebook(text=str_to_predict)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}