demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\projects\\tools\\transformerlens-model-table\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "f:\\projects\\tools\\transformerlens-model-table\\get_model_table.py:43: UserWarning: Failed to get Hugging Face token -- info about certain models will be limited\n",
      "'NoneType' object has no attribute 'startswith'\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import transformer_lens\n",
    "from get_model_table import get_model_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TransformerLens version: 2.8.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mivan\\AppData\\Local\\Temp\\ipykernel_24220\\2128677661.py:2: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html\n",
      "  import pkg_resources\n"
     ]
    }
   ],
   "source": [
    "# this notebook is for demo purposes and not automatically re-run -- see the live page for up-to-date version\n",
    "import pkg_resources\n",
    "print('TransformerLens version:', pkg_resources.get_distribution('transformer_lens').version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running in parallel with n_processes = 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading model info: 100%|██████████| 190/190 [01:10<00:00,  2.70it/s]\n"
     ]
    }
   ],
   "source": [
    "TABLE = get_model_table(\n",
    "\tverbose=True,\n",
    "\tforce_reload=True,\n",
    "\tparallelize=4,\n",
    "\tdo_write=True,\n",
    "\tallow_except=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name.default_alias</th>\n",
       "      <th>name.huggingface</th>\n",
       "      <th>name.aliases</th>\n",
       "      <th>model_type</th>\n",
       "      <th>name.from_cfg</th>\n",
       "      <th>n_params.as_str</th>\n",
       "      <th>n_params.as_int</th>\n",
       "      <th>n_params.from_name</th>\n",
       "      <th>cfg.n_params</th>\n",
       "      <th>cfg.n_layers</th>\n",
       "      <th>...</th>\n",
       "      <th>config</th>\n",
       "      <th>tokenizer.name</th>\n",
       "      <th>tokenizer.vocab_size</th>\n",
       "      <th>tokenizer.max_len</th>\n",
       "      <th>tokenizer.class</th>\n",
       "      <th>tokenizer.vocab_hash</th>\n",
       "      <th>tensor_shapes.state_dict</th>\n",
       "      <th>tensor_shapes.state_dict.raw__</th>\n",
       "      <th>tensor_shapes.activation_cache</th>\n",
       "      <th>tensor_shapes.activation_cache.raw__</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>gpt2</td>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>gpt2</td>\n",
       "      <td>gpt2</td>\n",
       "      <td>85M</td>\n",
       "      <td>84934656</td>\n",
       "      <td>None</td>\n",
       "      <td>84934656</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>n_layers: 12\\nd_model: 768\\nn_ctx: 1024\\nd_hea...</td>\n",
       "      <td>gpt2</td>\n",
       "      <td>50257.0</td>\n",
       "      <td>1024</td>\n",
       "      <td>GPT2TokenizerFast</td>\n",
       "      <td>v8xfIj5kwZX5RwgLU66lZNZUlE4=</td>\n",
       "      <td>embed:\\n  W_E: (50257, 768)\\npos_embed:\\n  W_p...</td>\n",
       "      <td>{'embed': {'W_E': '(50257, 768)'}, 'pos_embed'...</td>\n",
       "      <td>blocks:\\n  '[0-11]':\\n    ln1:\\n      hook_sca...</td>\n",
       "      <td>{'blocks': {'[0-11]': {'ln1': {'hook_scale': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gpt2-medium</td>\n",
       "      <td>gpt2-medium</td>\n",
       "      <td></td>\n",
       "      <td>gpt2</td>\n",
       "      <td>gpt2-medium</td>\n",
       "      <td>302M</td>\n",
       "      <td>301989888</td>\n",
       "      <td>None</td>\n",
       "      <td>301989888</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>n_layers: 24\\nd_model: 1024\\nn_ctx: 1024\\nd_he...</td>\n",
       "      <td>gpt2-medium</td>\n",
       "      <td>50257.0</td>\n",
       "      <td>1024</td>\n",
       "      <td>GPT2TokenizerFast</td>\n",
       "      <td>v8xfIj5kwZX5RwgLU66lZNZUlE4=</td>\n",
       "      <td>embed:\\n  W_E: (50257, 1024)\\npos_embed:\\n  W_...</td>\n",
       "      <td>{'embed': {'W_E': '(50257, 1024)'}, 'pos_embed...</td>\n",
       "      <td>blocks:\\n  '[0-23]':\\n    ln1:\\n      hook_sca...</td>\n",
       "      <td>{'blocks': {'[0-23]': {'ln1': {'hook_scale': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt2-large</td>\n",
       "      <td>gpt2-large</td>\n",
       "      <td></td>\n",
       "      <td>gpt2</td>\n",
       "      <td>gpt2-large</td>\n",
       "      <td>708M</td>\n",
       "      <td>707788800</td>\n",
       "      <td>None</td>\n",
       "      <td>707788800</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>n_layers: 36\\nd_model: 1280\\nn_ctx: 1024\\nd_he...</td>\n",
       "      <td>gpt2-large</td>\n",
       "      <td>50257.0</td>\n",
       "      <td>1024</td>\n",
       "      <td>GPT2TokenizerFast</td>\n",
       "      <td>v8xfIj5kwZX5RwgLU66lZNZUlE4=</td>\n",
       "      <td>embed:\\n  W_E: (50257, 1280)\\npos_embed:\\n  W_...</td>\n",
       "      <td>{'embed': {'W_E': '(50257, 1280)'}, 'pos_embed...</td>\n",
       "      <td>blocks:\\n  '[0-35]':\\n    ln1:\\n      hook_sca...</td>\n",
       "      <td>{'blocks': {'[0-35]': {'ln1': {'hook_scale': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt2-xl</td>\n",
       "      <td>gpt2-xl</td>\n",
       "      <td></td>\n",
       "      <td>gpt2</td>\n",
       "      <td>gpt2-xl</td>\n",
       "      <td>1.5B</td>\n",
       "      <td>1474560000</td>\n",
       "      <td>None</td>\n",
       "      <td>1474560000</td>\n",
       "      <td>48</td>\n",
       "      <td>...</td>\n",
       "      <td>n_layers: 48\\nd_model: 1600\\nn_ctx: 1024\\nd_he...</td>\n",
       "      <td>gpt2-xl</td>\n",
       "      <td>50257.0</td>\n",
       "      <td>1024</td>\n",
       "      <td>GPT2TokenizerFast</td>\n",
       "      <td>v8xfIj5kwZX5RwgLU66lZNZUlE4=</td>\n",
       "      <td>embed:\\n  W_E: (50257, 1600)\\npos_embed:\\n  W_...</td>\n",
       "      <td>{'embed': {'W_E': '(50257, 1600)'}, 'pos_embed...</td>\n",
       "      <td>blocks:\\n  '[0-47]':\\n    ln1:\\n      hook_sca...</td>\n",
       "      <td>{'blocks': {'[0-47]': {'ln1': {'hook_scale': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>distillgpt2</td>\n",
       "      <td>distilgpt2</td>\n",
       "      <td>distillgpt2, distill-gpt2, distil-gpt2, gpt2-xs</td>\n",
       "      <td>gpt2</td>\n",
       "      <td>distilgpt2</td>\n",
       "      <td>42M</td>\n",
       "      <td>42467328</td>\n",
       "      <td>None</td>\n",
       "      <td>42467328</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>n_layers: 6\\nd_model: 768\\nn_ctx: 1024\\nd_head...</td>\n",
       "      <td>distilgpt2</td>\n",
       "      <td>50257.0</td>\n",
       "      <td>1024</td>\n",
       "      <td>GPT2TokenizerFast</td>\n",
       "      <td>v8xfIj5kwZX5RwgLU66lZNZUlE4=</td>\n",
       "      <td>embed:\\n  W_E: (50257, 768)\\npos_embed:\\n  W_p...</td>\n",
       "      <td>{'embed': {'W_E': '(50257, 768)'}, 'pos_embed'...</td>\n",
       "      <td>blocks:\\n  '[0-5]':\\n    ln1:\\n      hook_scal...</td>\n",
       "      <td>{'blocks': {'[0-5]': {'ln1': {'hook_scale': '(...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185</th>\n",
       "      <td>yi-34b-chat</td>\n",
       "      <td>01-ai/Yi-34B-Chat</td>\n",
       "      <td>yi-34b-chat, Yi-34B-Chat</td>\n",
       "      <td>yi</td>\n",
       "      <td>Yi-34B-Chat</td>\n",
       "      <td>39B</td>\n",
       "      <td>38755368960</td>\n",
       "      <td>34b</td>\n",
       "      <td>38755368960</td>\n",
       "      <td>60</td>\n",
       "      <td>...</td>\n",
       "      <td>n_layers: 60\\nd_model: 7168\\nn_ctx: 4096\\nd_he...</td>\n",
       "      <td>01-ai/Yi-34B-Chat</td>\n",
       "      <td>63992.0</td>\n",
       "      <td>4096</td>\n",
       "      <td>LlamaTokenizerFast</td>\n",
       "      <td>VGXAFrTzytwGdUlX6AWH0NacncM=</td>\n",
       "      <td>embed:\\n  W_E: (64000, 7168)\\nblocks:\\n  '[0-5...</td>\n",
       "      <td>{'embed': {'W_E': '(64000, 7168)'}, 'blocks': ...</td>\n",
       "      <td>blocks:\\n  '[0-59]':\\n    ln1:\\n      hook_sca...</td>\n",
       "      <td>{'blocks': {'[0-59]': {'ln1': {'hook_scale': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>t5-small</td>\n",
       "      <td>google-t5/t5-small</td>\n",
       "      <td>t5-small</td>\n",
       "      <td>t5</td>\n",
       "      <td>t5-small</td>\n",
       "      <td>19M</td>\n",
       "      <td>18874368</td>\n",
       "      <td>None</td>\n",
       "      <td>18874368</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>n_layers: 6\\nd_model: 512\\nn_ctx: 20\\nd_head: ...</td>\n",
       "      <td>google-t5/t5-small</td>\n",
       "      <td>32100.0</td>\n",
       "      <td>512</td>\n",
       "      <td>T5TokenizerFast</td>\n",
       "      <td>jQeywCyCMVL_vza2wKfpuwjNVys=</td>\n",
       "      <td>embed:\\n  W_E: (32128, 512)\\npos_embed:\\n  W_p...</td>\n",
       "      <td>{'embed': {'W_E': '(32128, 512)'}, 'pos_embed'...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>t5-base</td>\n",
       "      <td>google-t5/t5-base</td>\n",
       "      <td>t5-base</td>\n",
       "      <td>t5</td>\n",
       "      <td>t5-base</td>\n",
       "      <td>85M</td>\n",
       "      <td>84934656</td>\n",
       "      <td>None</td>\n",
       "      <td>84934656</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>n_layers: 12\\nd_model: 768\\nn_ctx: 20\\nd_head:...</td>\n",
       "      <td>google-t5/t5-base</td>\n",
       "      <td>32100.0</td>\n",
       "      <td>1000000000000000019884624838656</td>\n",
       "      <td>T5TokenizerFast</td>\n",
       "      <td>jQeywCyCMVL_vza2wKfpuwjNVys=</td>\n",
       "      <td>embed:\\n  W_E: (32128, 768)\\npos_embed:\\n  W_p...</td>\n",
       "      <td>{'embed': {'W_E': '(32128, 768)'}, 'pos_embed'...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>t5-large</td>\n",
       "      <td>google-t5/t5-large</td>\n",
       "      <td>t5-large</td>\n",
       "      <td>t5</td>\n",
       "      <td>t5-large</td>\n",
       "      <td>302M</td>\n",
       "      <td>301989888</td>\n",
       "      <td>None</td>\n",
       "      <td>301989888</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>n_layers: 24\\nd_model: 1024\\nn_ctx: 20\\nd_head...</td>\n",
       "      <td>google-t5/t5-large</td>\n",
       "      <td>32100.0</td>\n",
       "      <td>1000000000000000019884624838656</td>\n",
       "      <td>T5TokenizerFast</td>\n",
       "      <td>jQeywCyCMVL_vza2wKfpuwjNVys=</td>\n",
       "      <td>embed:\\n  W_E: (32128, 1024)\\npos_embed:\\n  W_...</td>\n",
       "      <td>{'embed': {'W_E': '(32128, 1024)'}, 'pos_embed...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>mGPT</td>\n",
       "      <td>None</td>\n",
       "      <td></td>\n",
       "      <td>None</td>\n",
       "      <td>mGPT</td>\n",
       "      <td>1.2B</td>\n",
       "      <td>1207959552</td>\n",
       "      <td>None</td>\n",
       "      <td>1207959552</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>n_layers: 24\\nd_model: 2048\\nn_ctx: 2048\\nd_he...</td>\n",
       "      <td>ai-forever/mGPT</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>2048</td>\n",
       "      <td>GPT2TokenizerFast</td>\n",
       "      <td>8j6CU_p3zgyeEBZ1Z3lu358tiy0=</td>\n",
       "      <td>embed:\\n  W_E: (100000, 2048)\\npos_embed:\\n  W...</td>\n",
       "      <td>{'embed': {'W_E': '(100000, 2048)'}, 'pos_embe...</td>\n",
       "      <td>blocks:\\n  '[0-23]':\\n    ln1:\\n      hook_sca...</td>\n",
       "      <td>{'blocks': {'[0-23]': {'ln1': {'hook_scale': '...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>190 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    name.default_alias    name.huggingface  \\\n",
       "0           gpt2-small                gpt2   \n",
       "1          gpt2-medium         gpt2-medium   \n",
       "2           gpt2-large          gpt2-large   \n",
       "3              gpt2-xl             gpt2-xl   \n",
       "4          distillgpt2          distilgpt2   \n",
       "..                 ...                 ...   \n",
       "185        yi-34b-chat   01-ai/Yi-34B-Chat   \n",
       "186           t5-small  google-t5/t5-small   \n",
       "187            t5-base   google-t5/t5-base   \n",
       "188           t5-large  google-t5/t5-large   \n",
       "189               mGPT                None   \n",
       "\n",
       "                                        name.aliases model_type name.from_cfg  \\\n",
       "0                                         gpt2-small       gpt2          gpt2   \n",
       "1                                                          gpt2   gpt2-medium   \n",
       "2                                                          gpt2    gpt2-large   \n",
       "3                                                          gpt2       gpt2-xl   \n",
       "4    distillgpt2, distill-gpt2, distil-gpt2, gpt2-xs       gpt2    distilgpt2   \n",
       "..                                               ...        ...           ...   \n",
       "185                         yi-34b-chat, Yi-34B-Chat         yi   Yi-34B-Chat   \n",
       "186                                         t5-small         t5      t5-small   \n",
       "187                                          t5-base         t5       t5-base   \n",
       "188                                         t5-large         t5      t5-large   \n",
       "189                                                        None          mGPT   \n",
       "\n",
       "    n_params.as_str  n_params.as_int n_params.from_name  cfg.n_params  \\\n",
       "0               85M         84934656               None      84934656   \n",
       "1              302M        301989888               None     301989888   \n",
       "2              708M        707788800               None     707788800   \n",
       "3              1.5B       1474560000               None    1474560000   \n",
       "4               42M         42467328               None      42467328   \n",
       "..              ...              ...                ...           ...   \n",
       "185             39B      38755368960                34b   38755368960   \n",
       "186             19M         18874368               None      18874368   \n",
       "187             85M         84934656               None      84934656   \n",
       "188            302M        301989888               None     301989888   \n",
       "189            1.2B       1207959552               None    1207959552   \n",
       "\n",
       "     cfg.n_layers  ...                                             config  \\\n",
       "0              12  ...  n_layers: 12\\nd_model: 768\\nn_ctx: 1024\\nd_hea...   \n",
       "1              24  ...  n_layers: 24\\nd_model: 1024\\nn_ctx: 1024\\nd_he...   \n",
       "2              36  ...  n_layers: 36\\nd_model: 1280\\nn_ctx: 1024\\nd_he...   \n",
       "3              48  ...  n_layers: 48\\nd_model: 1600\\nn_ctx: 1024\\nd_he...   \n",
       "4               6  ...  n_layers: 6\\nd_model: 768\\nn_ctx: 1024\\nd_head...   \n",
       "..            ...  ...                                                ...   \n",
       "185            60  ...  n_layers: 60\\nd_model: 7168\\nn_ctx: 4096\\nd_he...   \n",
       "186             6  ...  n_layers: 6\\nd_model: 512\\nn_ctx: 20\\nd_head: ...   \n",
       "187            12  ...  n_layers: 12\\nd_model: 768\\nn_ctx: 20\\nd_head:...   \n",
       "188            24  ...  n_layers: 24\\nd_model: 1024\\nn_ctx: 20\\nd_head...   \n",
       "189            24  ...  n_layers: 24\\nd_model: 2048\\nn_ctx: 2048\\nd_he...   \n",
       "\n",
       "         tokenizer.name  tokenizer.vocab_size  \\\n",
       "0                  gpt2               50257.0   \n",
       "1           gpt2-medium               50257.0   \n",
       "2            gpt2-large               50257.0   \n",
       "3               gpt2-xl               50257.0   \n",
       "4            distilgpt2               50257.0   \n",
       "..                  ...                   ...   \n",
       "185   01-ai/Yi-34B-Chat               63992.0   \n",
       "186  google-t5/t5-small               32100.0   \n",
       "187   google-t5/t5-base               32100.0   \n",
       "188  google-t5/t5-large               32100.0   \n",
       "189     ai-forever/mGPT              100000.0   \n",
       "\n",
       "                   tokenizer.max_len     tokenizer.class  \\\n",
       "0                               1024   GPT2TokenizerFast   \n",
       "1                               1024   GPT2TokenizerFast   \n",
       "2                               1024   GPT2TokenizerFast   \n",
       "3                               1024   GPT2TokenizerFast   \n",
       "4                               1024   GPT2TokenizerFast   \n",
       "..                               ...                 ...   \n",
       "185                             4096  LlamaTokenizerFast   \n",
       "186                              512     T5TokenizerFast   \n",
       "187  1000000000000000019884624838656     T5TokenizerFast   \n",
       "188  1000000000000000019884624838656     T5TokenizerFast   \n",
       "189                             2048   GPT2TokenizerFast   \n",
       "\n",
       "             tokenizer.vocab_hash  \\\n",
       "0    v8xfIj5kwZX5RwgLU66lZNZUlE4=   \n",
       "1    v8xfIj5kwZX5RwgLU66lZNZUlE4=   \n",
       "2    v8xfIj5kwZX5RwgLU66lZNZUlE4=   \n",
       "3    v8xfIj5kwZX5RwgLU66lZNZUlE4=   \n",
       "4    v8xfIj5kwZX5RwgLU66lZNZUlE4=   \n",
       "..                            ...   \n",
       "185  VGXAFrTzytwGdUlX6AWH0NacncM=   \n",
       "186  jQeywCyCMVL_vza2wKfpuwjNVys=   \n",
       "187  jQeywCyCMVL_vza2wKfpuwjNVys=   \n",
       "188  jQeywCyCMVL_vza2wKfpuwjNVys=   \n",
       "189  8j6CU_p3zgyeEBZ1Z3lu358tiy0=   \n",
       "\n",
       "                              tensor_shapes.state_dict  \\\n",
       "0    embed:\\n  W_E: (50257, 768)\\npos_embed:\\n  W_p...   \n",
       "1    embed:\\n  W_E: (50257, 1024)\\npos_embed:\\n  W_...   \n",
       "2    embed:\\n  W_E: (50257, 1280)\\npos_embed:\\n  W_...   \n",
       "3    embed:\\n  W_E: (50257, 1600)\\npos_embed:\\n  W_...   \n",
       "4    embed:\\n  W_E: (50257, 768)\\npos_embed:\\n  W_p...   \n",
       "..                                                 ...   \n",
       "185  embed:\\n  W_E: (64000, 7168)\\nblocks:\\n  '[0-5...   \n",
       "186  embed:\\n  W_E: (32128, 512)\\npos_embed:\\n  W_p...   \n",
       "187  embed:\\n  W_E: (32128, 768)\\npos_embed:\\n  W_p...   \n",
       "188  embed:\\n  W_E: (32128, 1024)\\npos_embed:\\n  W_...   \n",
       "189  embed:\\n  W_E: (100000, 2048)\\npos_embed:\\n  W...   \n",
       "\n",
       "                        tensor_shapes.state_dict.raw__  \\\n",
       "0    {'embed': {'W_E': '(50257, 768)'}, 'pos_embed'...   \n",
       "1    {'embed': {'W_E': '(50257, 1024)'}, 'pos_embed...   \n",
       "2    {'embed': {'W_E': '(50257, 1280)'}, 'pos_embed...   \n",
       "3    {'embed': {'W_E': '(50257, 1600)'}, 'pos_embed...   \n",
       "4    {'embed': {'W_E': '(50257, 768)'}, 'pos_embed'...   \n",
       "..                                                 ...   \n",
       "185  {'embed': {'W_E': '(64000, 7168)'}, 'blocks': ...   \n",
       "186  {'embed': {'W_E': '(32128, 512)'}, 'pos_embed'...   \n",
       "187  {'embed': {'W_E': '(32128, 768)'}, 'pos_embed'...   \n",
       "188  {'embed': {'W_E': '(32128, 1024)'}, 'pos_embed...   \n",
       "189  {'embed': {'W_E': '(100000, 2048)'}, 'pos_embe...   \n",
       "\n",
       "                        tensor_shapes.activation_cache  \\\n",
       "0    blocks:\\n  '[0-11]':\\n    ln1:\\n      hook_sca...   \n",
       "1    blocks:\\n  '[0-23]':\\n    ln1:\\n      hook_sca...   \n",
       "2    blocks:\\n  '[0-35]':\\n    ln1:\\n      hook_sca...   \n",
       "3    blocks:\\n  '[0-47]':\\n    ln1:\\n      hook_sca...   \n",
       "4    blocks:\\n  '[0-5]':\\n    ln1:\\n      hook_scal...   \n",
       "..                                                 ...   \n",
       "185  blocks:\\n  '[0-59]':\\n    ln1:\\n      hook_sca...   \n",
       "186                                                NaN   \n",
       "187                                                NaN   \n",
       "188                                                NaN   \n",
       "189  blocks:\\n  '[0-23]':\\n    ln1:\\n      hook_sca...   \n",
       "\n",
       "                  tensor_shapes.activation_cache.raw__  \n",
       "0    {'blocks': {'[0-11]': {'ln1': {'hook_scale': '...  \n",
       "1    {'blocks': {'[0-23]': {'ln1': {'hook_scale': '...  \n",
       "2    {'blocks': {'[0-35]': {'ln1': {'hook_scale': '...  \n",
       "3    {'blocks': {'[0-47]': {'ln1': {'hook_scale': '...  \n",
       "4    {'blocks': {'[0-5]': {'ln1': {'hook_scale': '(...  \n",
       "..                                                 ...  \n",
       "185  {'blocks': {'[0-59]': {'ln1': {'hook_scale': '...  \n",
       "186                                                NaN  \n",
       "187                                                NaN  \n",
       "188                                                NaN  \n",
       "189  {'blocks': {'[0-23]': {'ln1': {'hook_scale': '...  \n",
       "\n",
       "[190 rows x 29 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(TABLE)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}