Huggingface_finetune_gpt2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Huggingface finetune gpt2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "mount_file_id": "1lkuYC06chnKHyMAeD3TpZwV7DrAPNPj0",
      "authorship_tag": "ABX9TyN8G2fEXqGFmINEabSD8stH",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/GeorgeDittmar/reddit-scan-datasets/blob/master/Huggingface_finetune_gpt2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MZMbGCw0Qxfg"
      },
      "source": [
        "First begin setup by cloning transformers repo. We need to store the training script locally since there isnt an easier way to train tf based gpt2 models as far as I can see.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "91rmSAUQVIUP"
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers\n",
        "!git clone https://github.com/GeorgeDittmar/reddit-scan-datasets.git"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RFzEr4Y1VJKP"
      },
      "source": [
        "!ls"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M-ke4920tDqv"
      },
      "source": [
        "merge data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BPIgByLqmI81"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K2M2Oz9CYB4P"
      },
      "source": [
        "import os\n",
        "os.chdir(\"transformers\")\n",
        "os.chdir(\"./examples\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "04rBGxwiYnep"
      },
      "source": [
        "!pip install -r requirements.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iB29mAKjaNIQ"
      },
      "source": [
        "!ls"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eo5gRmXaWx0m"
      },
      "source": [
        "!pip install pyarrow --upgrade"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P54JMsgZYrEK"
      },
      "source": [
        "!ls /content/reddit-scan-datasets/data_sets/investing/submission_titles_investing_100k.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y5sdYSpAWY1S"
      },
      "source": [
        "import os\n",
        "os.chdir(\"/content/transformers/examples/\")\n",
        "os.chdir(\"./language-modeling\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a6G6WINaYmwx"
      },
      "source": [
        "! pip install git+git://github.com/huggingface/transformers/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uU4ckPTf9T-2"
      },
      "source": [
        "\"\"\"\n",
        "Now load in the reddit conspiracy merged data set and split it so we have a train and eval set\n",
        "\"\"\"\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "with open('/content/reddit-scan-datasets/data_sets/conspiracy/submission_titles_conspiracy_merged.txt', 'r') as data:\n",
        "  dataset = [\"<|title|>\" + x.strip() for x in data.readlines()]\n",
        "\n",
        "with open('/content/reddit-scan-datasets/data_sets/conspiracy/submission_titles_conspiracy_merged.txt', 'r') as data:\n",
        "  dataset2 = [\"<|title|>\" + x.strip() for x in data.readlines()]\n",
        "\n",
        "# # train, eval = train_test_split(dataset + dataset2, train_size=.8, random_state=2021)\n",
        "# with open('/content/comments_cloud_city_pdx.txt', 'r') as data:\n",
        "#   comments = data.read()\n",
        "\n",
        "# examples = [\"<|comment|>\" + x  for x in comments.split(\"<|comment|>\") if len(x) > 0]\n",
        "# print(len(examples))\n",
        "train, eval = train_test_split(dataset + dataset2, train_size=.9, random_state=2020)\n",
        "print(len(train))\n",
        "print(len(eval))\n",
        "#print(dataset[:10])\n",
        "with open('train_tmp.txt', 'w') as file_handle:\n",
        "  file_handle.write(\"<|endoftext|>\".join(train))\n",
        "\n",
        "with open('eval_tmp.txt', 'w') as file_handle:\n",
        "  file_handle.write(\"<|endoftext|>\".join(eval))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Hz8h71-lahn"
      },
      "source": [
        "!tail train_tmp.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yyV_rTL3ZE_H"
      },
      "source": [
        "!python run_clm.py \\\n",
        "--model_type gpt2-medium \\\n",
        "--model_name_or_path gpt2-medium \\\n",
        "--train_file \"train_tmp.txt\" \\\n",
        "--do_train \\\n",
        "--validation_file \"eval_tmp.txt\" \\\n",
        "--do_eval \\\n",
        "--per_gpu_train_batch_size 1 \\\n",
        "--save_steps -1 \\\n",
        "--num_train_epochs 5 \\\n",
        "--fp16 \\\n",
        "--output_dir=\"/content/conspiracy_conservative_model_fp16_v6\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JnD2v-wiACEg"
      },
      "source": [
        "!cat run_clm.py\n",
        "!pwd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DToY33kMVzt-"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}