TrainGPT2_FromScratch_02_encode_dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TrainGPT2-FromScratch-02-encode-dataset.ipynb",
      "provenance": [],
      "private_outputs": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "2mpcM2mU39Ez",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s2xLi78w4BS3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd /content\n",
        "!cp -r '/gdrive/My Drive/colab_data/gpt2/Hebrew_Bible-k' './Hebrew_Bible-k'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i67L4m-lh6hc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd /content\n",
        "\n",
        "\n",
        "!ln -s /content/Hebrew_Bible-k/vocab.json /content/Hebrew_Bible-k/encoder.json\n",
        "!ln -s /content/Hebrew_Bible-k/merges.txt /content/Hebrew_Bible-k/vocab.bpe\n",
        "\n",
        "!mkdir checkpoint\n",
        "!ln -s /content/Hebrew_Bible-k /content/checkpoint/Hebrew_Bible-k\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vQ9kl_fcgr--",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd /content\n",
        "!wget https://storage.googleapis.com/dadler_gpt2_data/Hebrew_Bible-k_all_clean.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5uYti5gMheKK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%tensorflow_version 1.x\n",
        "!pip install -q gpt-2-simple\n",
        "import gpt_2_simple as gpt2\n",
        "from datetime import datetime"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kb4fHRjbjal7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "gpt2.download_gpt2(model_name=\"124M\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ng4wBTrljcWb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ln -s /content/models/124M/checkpoint /content/Hebrew_Bible-k/checkpoint\n",
        "!ln -s /content/models/124M/model.ckpt.data-00000-of-00001 /content/Hebrew_Bible-k/model.ckpt.data-00000-of-00001\n",
        "!ln -s /content/models/124M/model.ckpt.index /content/Hebrew_Bible-k/model.ckpt.index\n",
        "!ln -s /content/models/124M/model.ckpt.meta /content/Hebrew_Bible-k/model.ckpt.meta\n",
        "!ln -s /content/models/124M/hparams.json /content/Hebrew_Bible-k/hparams.json"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vey6-OERhv92",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sess = gpt2.start_tf_sess()\n",
        "gpt2.load_gpt2(sess, run_name='Hebrew_Bible-k')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IQuL8peUuhSF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "gpt2.encode_dataset(\"/content/Hebrew_Bible-k_all_clean.txt\", \n",
        "                    model_dir='/content/checkpoint',\n",
        "                    out_path='Hebrew_Bible-k_encoded.npz',\n",
        "                    model_name='Hebrew_Bible-k')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x3xc52NXkCX0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "gpt2.reset_session(sess)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3jOvtjwA4pEQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp '/content/Hebrew_Bible-k_encoded.npz' '/gdrive/My Drive/colab_data/gpt2/'\n",
        "!cp '/content/Hebrew_Bible-k/encoder.json' '/gdrive/My Drive/colab_data/gpt2/Hebrew_Bible-k/'\n",
        "!cp '/content/Hebrew_Bible-k/vocab.bpe' '/gdrive/My Drive/colab_data/gpt2/Hebrew_Bible-k/'\n",
        "!cp '/content/Hebrew_Bible-k/hparams.json' '/gdrive/My Drive/colab_data/gpt2/Hebrew_Bible-k/'"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}