mamba 4chan train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pytorch_lightning as pl\n",
    "import torch\n",
    "from lightning.pytorch import loggers as pl_loggers\n",
    "from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint\n",
    "from torch.utils.data import DataLoader, Dataset, random_split\n",
    "\n",
    "from model import mamba_4chan\n",
    "from model_config import mamba_4chan_370m_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load base mamba model\n",
    "model = mamba_4chan(mamba_4chan_370m_config())\n",
    "state_dict = torch.load(\"path_to.bin\")\n",
    "model.load_state_dict(state_dict)\n",
    "\n",
    "# load mamba 4chan model\n",
    "# model = mamba_4chan.load_from_checkpoint(\"path_to.ckpt\")\n",
    "\n",
    "model.learning_rate = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class pol_dataset(Dataset):\n",
    "    def __init__(\n",
    "        self,\n",
    "        memmap_path: str,\n",
    "        context_size: int = 2048,\n",
    "        eos_token: int = 0,\n",
    "        stride: int = 2047,\n",
    "    ):\n",
    "        self.memmap = np.memmap(memmap_path, dtype=\"uint16\")\n",
    "        self.context_size = context_size\n",
    "        self.eos_token = eos_token\n",
    "        self.stride = stride\n",
    "\n",
    "    def __len__(self):\n",
    "        return ((len(self.memmap) - self.context_size) // self.stride) + 1\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        start = idx * self.stride\n",
    "        end = start + self.context_size\n",
    "\n",
    "        if end > len(self.memmap):\n",
    "            padding_size = end - len(self.memmap)\n",
    "            data = np.concatenate(\n",
    "                (\n",
    "                    self.memmap[start:],\n",
    "                    np.full(padding_size, self.eos_token, dtype=\"uint16\"),\n",
    "                )\n",
    "            )\n",
    "        else:\n",
    "            data = self.memmap[start:end]\n",
    "\n",
    "        return torch.tensor(data, dtype=torch.long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class data_module(pl.LightningDataModule):\n",
    "    def __init__(\n",
    "        self,\n",
    "        memmap_path: str,\n",
    "        batch_size: int,\n",
    "        context_size: int = 2048,\n",
    "        eos_token: int = 0,\n",
    "        stride: int = 2047,\n",
    "        train_val_ratio: float = 0.95,\n",
    "    ):\n",
    "        super().__init__()\n",
    "        self.save_hyperparameters()\n",
    "        self.memmap_path = memmap_path\n",
    "        self.batch_size = batch_size\n",
    "        self.context_size = context_size\n",
    "        self.eos_token = eos_token\n",
    "        self.stride = stride\n",
    "        self.train_val_ratio = train_val_ratio\n",
    "\n",
    "    def setup(self, stage: str = None):\n",
    "        dataset = pol_dataset(\n",
    "            self.memmap_path, self.context_size, self.eos_token, self.stride\n",
    "        )\n",
    "\n",
    "        train_size = int(len(dataset) * self.train_val_ratio)\n",
    "        val_size = len(dataset) - train_size\n",
    "\n",
    "        self.train_set, self.val_set = random_split(dataset, [train_size, val_size])\n",
    "\n",
    "    def train_dataloader(self):\n",
    "        return DataLoader(\n",
    "            self.train_set,\n",
    "            batch_size=self.batch_size,\n",
    "            shuffle=True,\n",
    "            num_workers=8,\n",
    "            pin_memory=True,\n",
    "        )\n",
    "\n",
    "    def val_dataloader(self):\n",
    "        return DataLoader(\n",
    "            self.val_set,\n",
    "            batch_size=self.batch_size,\n",
    "            num_workers=8,\n",
    "            pin_memory=True,\n",
    "        )\n",
    "\n",
    "\n",
    "data = data_module(memmap_path=\"dataset.dat\", batch_size=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = pl.Trainer(\n",
    "    callbacks=[\n",
    "        ModelCheckpoint(\n",
    "            dirpath=\"models/\",\n",
    "            save_top_k=-1,\n",
    "        ),\n",
    "        LearningRateMonitor(logging_interval=\"step\"),\n",
    "    ],\n",
    "    logger=pl_loggers.WandbLogger(project=\"Mamba 4chan 370m\", name=\"Fine-tuning\"),\n",
    "    precision=\"bf16-mixed\",\n",
    "    max_epochs=1,\n",
    "    accumulate_grad_batches=250,\n",
    ")\n",
    "\n",
    "trainer.fit(model, data)\n",
    "# trainer.fit(model, data, ckpt_path=\"ckpt to resume training\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mamba-4chan",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}