diff --git a/tokenizer_ts/.npmignore b/tokenizer_ts/.npmignore index 0363f59..10160e0 100644 --- a/tokenizer_ts/.npmignore +++ b/tokenizer_ts/.npmignore @@ -8,4 +8,6 @@ dist/test/* debug.ts *.map *.tiktoken -.eslintrc.js \ No newline at end of file +.eslintrc.js +/perf/* +*.map diff --git a/tokenizer_ts/package.json b/tokenizer_ts/package.json index 3e258b4..e52441d 100644 --- a/tokenizer_ts/package.json +++ b/tokenizer_ts/package.json @@ -34,6 +34,7 @@ "scripts": { "test": "mocha -u tdd --require ts-node/register test/**/*.ts", "build": "tsc -p ./tsconfig.json", + "watch": "tsc -p ./tsconfig.json --watch", "eslint": "eslint src --ext ts", "format": "prettier --write \"./**/*.{ts,tsx}\"" }, diff --git a/tokenizer_ts/perf/.gitignore b/tokenizer_ts/perf/.gitignore new file mode 100644 index 0000000..ff67559 --- /dev/null +++ b/tokenizer_ts/perf/.gitignore @@ -0,0 +1,3 @@ +package.json +package-lock.json +*.cpuprofile diff --git a/tokenizer_ts/perf/benchmark-folder.js b/tokenizer_ts/perf/benchmark-folder.js new file mode 100644 index 0000000..1047065 --- /dev/null +++ b/tokenizer_ts/perf/benchmark-folder.js @@ -0,0 +1,65 @@ +const fs = require('fs/promises'); +const path = require('path'); +const inspector = require('inspector'); +const { promisify } = require('util'); + +const [,, encoderName, folderPath, method, modulePath] = process.argv; +const { createByEncoderName } = require(modulePath); +const minTime = 10_000; +const minCycles = 5; + +const fileExtensions = ['.ts', '.js', '.py']; + +async function readAllFilesInFolder(folderPath) { + const files = await fs.readdir(folderPath, { withFileTypes: true }); + const fileContents = await Promise.all(files.map(async (file) => { + const res = path.resolve(folderPath, file.name); + if (file.isDirectory()) { + return readAllFilesInFolder(res); + } else if (fileExtensions.some(f => res.endsWith(f))) { + return fs.readFile(res, 'utf8'); + } else { + return []; + } + })); + + return fileContents.flat(); +} + +Promise.all([ + readAllFilesInFolder(folderPath), + createByEncoderName(encoderName) +]).then(async ([files, tokenizer]) => { + let totalSize = 0; + for (const file of files) { + totalSize += file.length; + } + + const session = new inspector.Session(); + session.connect(); + const post = promisify(session.post).bind(session); + await post('Profiler.enable'); + await post('Profiler.start'); + + const start = performance.now(); + let cycles = []; + while (performance.now() - start < minTime || cycles.length < minCycles) { + const cycleStart = performance.now(); + switch (method) { + case 'encode': + files.forEach(file => tokenizer.encode(file)); + break; + case 'encodeTrimSuffix': + files.forEach(file => tokenizer.encodeTrimSuffix(file, 1337)); + break; + default: + throw new Error(`unknown method ${method}`); + } + cycles.push(performance.now() - cycleStart); + } + + const data = await post('Profiler.stop'); + await fs.writeFile('profile.cpuprofile', JSON.stringify(data.profile)); + + process.stdout.write(JSON.stringify({ totalSize, cycles })); +}); diff --git a/tokenizer_ts/perf/notebook.ipynb b/tokenizer_ts/perf/notebook.ipynb new file mode 100644 index 0000000..89c926f --- /dev/null +++ b/tokenizer_ts/perf/notebook.ipynb @@ -0,0 +1,188 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# TS Tiktokenizer Performance\n", + "\n", + "This notebook is used for analyzing the performance of and performance improvements to the Tokenizer. It uses the VS Code repo as its corpus. First, let's grab the last released version of `@microsoft/tiktokenizer`, and get a baseline." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "import os\n", + "import subprocess\n", + "import json\n", + "\n", + "vscode_repo_path = \"../../../vscode\"\n", + "if not os.path.exists(vscode_repo_path):\n", + " print(\"The repo does not exist.\")\n", + "\n", + "def run_benchmark(module_path, encoder_name = 'cl100k_base', method = 'encode'):\n", + " command = f\"node ./benchmark-folder.js {encoder_name} {vscode_repo_path}/src {method} {module_path}\"\n", + " result = subprocess.check_output(command, shell=True)\n", + " parsed = json.loads(result)\n", + " return parsed\n", + "\n", + "#os.system('npm install @microsoft/tiktokenizer --prefix ./')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Performance can vary machine to machine, make sure to collect a baseline before you start working. Every time you run a benchmark, there'll be a `profile.cpuprofile` written out that you can inspect." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "metadata": {} + }, + "outputs": [], + "source": [ + "# This can take a minute, make some tea 🍵\n", + "baseline = run_benchmark('@microsoft/tiktokenizer')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZwAAAEICAYAAABrtkJsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAbKUlEQVR4nO3deVTVdf7H8dcllEwKUVxASRR3ltJgEmvUyqVFTRsrChPb1BptN5uxGc0sGxudpvHMKWc0tRzrlJYokTalNpmpaJMpiWXhguYyYoSpbO/fH56+vyiuiMoHpOfjHM7xftfX/Rzuffld7sVnZiYAAKpYQHUHAAD8MlA4AAAnKBwAgBMUDgDACQoHAOAEhQMAcILCQY0XExOjFStWVHeMM27YsGF6/PHHz/h2g4OD9dVXX53x7dZUVTWOOPMoHFS74OBg7ycgIED16tXzHs+bN0+bN29Wz549qzzHhAkTNGTIkFNev6a88RUUFKh169ZnZFtHjx5VgwYN9P777/9s3oMPPqjBgwdLkj788EN169ZNISEhatiwoS677DKtW7fO73a3bt2qG2+8UWFhYQoJCVF8fLymTZumkpKSM5IbNROFg2pXUFDg/Vx44YVavHix9zglJaW64/2inXvuubr55ps1d+7cMtNLSko0f/58paamKj8/X/369dPo0aN18OBB5ebmavz48QoKCip3m9u2bdOll16qyMhIffbZZ/r222/1+uuvKzMzU999952Lp4XqYkAN0rJlS3v33Xf9Ths/frwNHjzYUlJSLDg42GJjYy07O9uefvppa9y4sbVo0cKWLl3qrXvo0CG74447rFmzZhYREWHjxo2z4uLin+03IyPD6tSpY4GBgVa/fn2Lj483M7Pc3Fzr37+/hYaGWnR0tM2YMaPc3C+++KIFBgZanTp1rH79+tavXz8zM8vKyrIePXpYSEiIderUyRYtWuStk5qaauPGjTMzs/z8fOvZs6eNHj3aSktL7fPPP7devXpZaGiotWvXzl577bUy691777127bXXWnBwsP3qV7+yL7/80psvyb744gvLzc21+vXrez/16tWzH7/kZ86caR06dLAGDRpYnz59LCcnp9zntmrVKgsODrbDhw9709LT061x48ZWVFRk69ats5CQkHLXLU9KSopde+21J1xm0aJF1qlTJwsJCbEePXpYVlaWN2/Dhg3WuXNnCw4OtptuusluvvlmbxzNzBYvXmwXXXSRhYSEWFJSkn366acnnQ1Vi8JBjXIyhRMUFGTvvPOOFRUV2W233WZRUVE2adIkKywstBkzZlhUVJS37vXXX2/Dhw+3goIC27t3ryUmJtoLL7xQ7r7Hjx9vKSkpZaZ1797d7rnnHjty5Ih98sknFhYWZv/+97/LXf/HBWJmVlhYaNHR0fbUU0/ZsWPH7L333rPg4GDbsmVLmeUPHDhgiYmJ3roFBQXWokULmzVrlhUVFdn69eutUaNGtmnTJm+90NBQW7NmjRUVFdmtt95qN998s7ffHwrnp2699VZLTk42M7M333zToqOjLSsry4qKiuzJJ5+0pKSkcp+XmVnbtm3t5Zdf9h4nJyfb/fffb2Zm3377rTVs2NCGDh1qb7/9th08eNDvdszMmjZtarNmzfI7Pzs728477zxbtmyZFRYW2p/+9CeLjo62Y8eO2bFjx+zCCy+0adOmWWFhob3++usWGBjojd369eutcePG9vHHH1txcbHNnj3bWrZsaUePHj1hJrhB4aBGOZnC6dWrlzcvLS3N6tev7x215OfnmyTLy8uzb775xurWrWvff/+9t/y//vUv69mzZ7n7/mnh7NixwwICAiw/P9+b9thjj1lqamq56/+0cD744ANr2rSplZSUeNOSk5Nt/Pjx3vK33367xcTE2JQpU7xlXn31Vbv88svLbHv48OE2YcIEb70777zTm5eenm7t27f3HpdXOM8884x16dLFG4urr77a/vnPf3rzS0pKrF69en6Pcp588knr3bu3mR0vmHr16tmGDRu8+VlZWZaammrNmze3c845x/r372/ffPNNudsKDAy0jIyMcueZmU2cONFuvPHGMtkiIiJs+fLltnLlSgsPD7fS0lJvflJSkjfuI0eOtMcff7zM9tq1a2crVqzwuz+4wzUcnHWaNm3q/btevXoKCwvTOeec4z2Wjl8X2r59u4qKihQeHq4GDRqoQYMGGjFihPbt23dS+9m9e7caNmyo888/35vWsmVL5ebmnvT6kZGRCgj4/5fZT9dPT0/XkSNHNHLkSG/a9u3btWbNGi9zgwYNNG/ePH3zzTfeMs2aNfP+fd5556mgoMBvjoyMDP31r3/VW2+95Y3P9u3bdf/993vbb9iwoczM73MbOnSoli9frtzcXL3xxhtq06aNOnfu7M3v2LGjZs+erV27dmnTpk3avXu3HnjggXK31ahRI+3Zs8dv3t27d6tly5be44CAAEVGRio3N1e7d+9W8+bN5fP5vPk/Xnb79u2aOnVqmbHbuXOndu/e7Xd/cCewugMAVSUyMlJBQUE6cOCAAgMr/lX/8ZuYJEVEROjgwYP67rvvvNLZsWOHmjdvftLr79y5U6WlpV7p7NixQ+3atfOWufvuu5WXl6drr71W77zzjurXr6/IyEj16NFD7777bqWeb3mys7OVmpqqhQsXKjIy0pseGRmpcePGnfRNGRdeeKF+/etfa968ecrIyNDQoUP9LtuhQwcNGzZML774Yrnze/XqpQULFuj2228vd35ERIQ+++wz77GZaefOnV7R5Obmysy88d6xY4eio6PLPK9x48ad1POCWxzhoNYKDw9Xnz599PDDDys/P1+lpaXatm2bVq5cWe7yTZs2VU5OjkpLSyUdf/Pq1q2bfve73+no0aPauHGjZs6c6fdNumnTpmU+/3LppZeqfv36mjJlioqKirRixQotXrxYycnJZdabPn262rdvr379+unIkSPq16+ftm7dqpdffllFRUUqKirSunXr9Pnnn1fq+efn5+v666/XpEmTdPnll5eZN3LkSE2ePFmbN2+WJO9OsRNJTU3V9OnTtWrVqjJjsGXLFk2dOlW7du2SJO3cuVPz589X165dy93OE088oY8++khjxozxjtq+/PJLDRkyRIcOHdJNN92k9PR0vffeeyoqKtLUqVMVFBSkbt26KSkpSYGBgXr++edVXFyshQsXau3atd627777br3wwgtas2aNzEyHDx9Weno6d7/VEBQOarW5c+eqsLBQnTp1UmhoqAYPHuz3dM6NN94o6fgpny5dukiS5s+fr5ycHEVERGjQoEF64okn1Lt373LXv/POO5WVlaUGDRpo4MCBqlu3rtLS0pSRkaGwsDDde++9mjt3rjp06FBmPZ/PpxkzZigyMlLXX3+96tSpo2XLlunVV19VRESEmjVrprFjx+rYsWOVeu4bNmxQdna2HnrooTKfdZKkQYMGaezYsUpOTtYFF1yg2NhYZWRknHB7gwcPVl5enq666iqFh4d7088//3ytWbPGK9iuXbsqNjZWU6dOLXc70dHRWr16tXJychQTE6OQkBD95je/UUJCgs4//3y1b99er7zyikaPHq2wsDAtXrxYixcvVt26dVW3bl0tXLhQs2fPVmhoqF577TXdcMMN3rYTEhL0j3/8Q6NGjVJoaKjatGmj2bNnV2rcUHV8ZvwBNgBA1eMIBwDgBIUDAHCCwgEAOEHhAACc4HM4foSFhSkqKqq6YwDAWSUnJ0cHDhwodx6F40dUVJQyMzOrOwYAnFUSEhL8zuOUGgDACQoHAOAEhQMAcILCAQA4QeEAAJygcAAATlA4AAAnKBwAgBMUDgDACQoHAOAEhQMAcILCAQA4QeEAAJygcAAATlA4AAAnKBwAgBMUDgDACQoHAOAEhQMAcILCAQA4QeEAAJygcAAATlA4AAAnKBwAgBMUDgDACQoHAOAEhQMAcILCAQA4QeEAAJygcAAATlA4AAAnKBwAgBMUDgDACQoHAOAEhQMAcILCAQA4QeEAAJygcAAATlA4AAAnKBwAgBMUDgDACQoHAOAEhQMAcILCAQA4QeEAAJygcAAATlA4AAAnKBwAgBMUDgDACQoHAOAEhQMAcILCAQA4QeEAAJygcAAATlA4AAAnKBwAgBMUDgDACQoHAOAEhQMAcILCAQA4QeEAAJygcAAATlA4AAAnKBwAgBMUDgDACQoHAOAEhQMAcILCAQA4QeEAAJygcAAATlA4AAAnKBwAgBMUDgDACQoHAOAEhQMAcILCAQA4QeEAAJygcAAATlA4AAAnKBwAgBMUDgDACQoHAOAEhQMAcILCAQA4UWHhHD58WKWlpZKkrVu3Ki0tTUVFRVUeDABQu1RYON27d9fRo0eVm5urq666Si+99JKGDRvmIBoAoDapsHDMTOedd54WLlyo0aNH680331RWVpaLbACAWuSkCmf16tWaN2+errvuOklScXFxlQcDANQuFRbOc889p8mTJ2vQoEGKiYnRV199pSuuuMJFNgBALeIzM6vuEDVRQkKCMjMzqzsGAJxVTvTeGVjRypmZmXr66aeVk5NT5lTaxo0bz1xCAECtV2HhpKSk6Nlnn1VcXJwCAvjYDqpGw4YNlZeX52x/Nv4C+Z7Id7Y/1D6hoaE6ePBgdcc4q1RYOI0bN9aAAQNcZMEvWF5enpye3Z0Q4nZ/qHV8Pl91RzjrVFg4TzzxhO666y5dddVVCgoK8qbfcMMNVRoMAFC7VFg4L730krZs2aKioiLvlJrP56NwAACVUmHhfPrpp/rss89cZAEA1GIV3gXQtWtXvlkAAHDaKjzC+fDDDzVnzhy1atVKQUFBMjP5fD5uiwYAVEqFhfPOO++4yAEAqOUqLJyWLVu6yFGr+Hw+brkFcNaqqvcwPskJAHCCwgEAOOG3cPr27au//OUv2rJli8s8AIBaym/hzJkzR6GhoZowYYK6dOmie+65R4sWLVJBQYHLfACAWsJv4TRr1kzDhg3Tq6++qszMTA0dOlTr169X37591atXL02ZMuWEG87JyVFsbOwZDyxJK1asUL9+/SRJaWlpeuaZZ6pkPwCAM6fCu9QkKSAgQElJSUpKStLEiRN14MABLV26tKqznZQBAwbw5aIAcBY4pZsGwsLClJKSUuFyxcXFSk1NVXx8vAYPHqzvv/9eEydOVGJiomJjYzV8+HDv1rvnn39enTp1Unx8vJKTkyVJhw8f1h133KHExER17txZixYt+tk+Zs+erVGjRkmShg0bpvvuu0/dunVT69at9cYbb3jLPfvss0pMTFR8fLzGjx9/Kk8bAHAaqvQutezsbA0fPlwbN27UBRdcoL///e8aNWqU1q1bp02bNunIkSNasmSJJOmZZ57RJ598oo0bN+qFF16QJD311FO68sortW7dOi1fvlxjxozR4cOHT7jPPXv26MMPP9SSJUv02GOPSZKWLVumL774QmvXrtV///tfrV+/Xh988MHP1p0xY4YSEhKUkJCg/fv3n9Zz9/l8/FTiBzgbVffr5mx7PZ7UKbVTFRkZqcsuu0ySNGTIED3//PNq1aqVpkyZou+//14HDx5UTEyM+vfvr/j4eKWkpGjgwIEaOHCgpONFkZaWpj//+c+SpKNHj2rHjh0n3OfAgQMVEBCgTp06ae/evd52li1bps6dO0uSCgoK9MUXX6h79+5l1h0+fLiGDx8u6fifST0dfPCzcigdnI1q6+u8ql6PFRbO3r179fvf/167d+9WRkaGsrKytHr1at15550VbvynoX0+n+69915lZmYqMjJSEyZM0NGjRyVJ6enp+uCDD5SWlqYnn3xSmzdvlplpwYIFat++/c8y+fPjv9nzwy+Dmel3v/udRowYUWFmAEDVqPCU2rBhw9S3b1/t3r1bktSuXTs999xzJ7XxHTt2aPXq1ZKk+fPn6/LLL5d0/BpQQUGBd42ltLRUO3fu1BVXXKEpU6bo0KFDKigoUN++ffW3v/3NK45PPvmk0k9QOv6ZolmzZnm3dOfm5mrfvn2ntC0AwKmp8AjnwIEDuummmzR58uTjKwQG6pxzzjmpjXfs2FFz5szRiBEj1LZtW91zzz3Ky8tTXFycoqKilJiYKEkqKSnRkCFD9O2338rM9OCDD6pBgwb6wx/+oAceeEDx8fEyM0VFRXnXfCqjT58++vzzz5WUlCRJCg4O1iuvvKImTZpUelsAgFPjswpOQvbs2VMLFixQ7969tWHDBn388ccaO3asVq5c6SpjtUhISFBmZuYprevz8eWdleV8zCaESBO+dbc/1Dq1+XV+Os/tRO+dFR7hTJs2TQMGDNC2bdt02WWXaf/+/WVuNwYA4GRUWDhdunTRypUrlZ2dLTNT+/btVadOHRfZAAC1SIWFU1JSorfffls5OTkqLi7WsmXLJEkPPfRQlYcDANQeFRZO//79de655youLk4BAfw1g5NRW8/rAvhlqKr3sAoLZ9euXdq4cWOV7BwA8MtR4SHLNddc451GAwDgVFV4hNO1a1cNGjRIpaWlqlOnjsxMPp9P+fn5LvIBAGqJCgvn4Ycf1urVqxUXF8f3XQEATlmFp9Tatm2r2NhYygYAcFoqPMIJDw9Xz549dc0115T5YkxuiwYAVEaFhdOqVSu1atVKhYWFKiwsdJEJv1Auj6Jt/AUcteO0hIaGVneEs06FhcNfx4QL1fHZJZvgfJfAL5rfwhk1apSmT5+u/v37l/s/wbS0tCoNBgCoXfwWzty5czV9+nQ98sgjLvMAAGopv4UTHR0tSerRo4ezMACA2stv4ezfv1/Tpk3zuyJ3qQEAKsNv4ZSUlKigoIAvogQAnBF+Cyc8PFx//OMfXWYBANRifr9pgCMbAMCZ5Ldw3nvvPZc5AAC1nN/CadiwocscAIBajj/hCQBwgsIBADhB4QAAnKBwAABOUDgAACcoHACAExQOAMAJCgcA4ASFAwBwgsIBADhB4QAAnKBwAABOUDgAACcoHACAExQOAMAJCgcA4ASFAwBwgsIBADhB4QAAnKBwAABOUDgAACcoHACAExQOAMAJCgcA4ASFAwBwgsIBADhB4QAAnKBwAABOUDgAACcoHACAExQOAMAJCgcA4ASFAwBwgsIBADhB4QAAnKBwAABOUDgAACcoHACAExQOAMAJCgcA4ASFAwBwgsIBADhB4QAAnKBwAABOUDgAACcoHACAExQOAMAJCgcA4ASFAwBwgsIBADhB4QAAnKBwAABOUDgAACcoHACAExQOAMAJCgcA4ASFAwBwgsIBADhB4QAAnKBwAABOUDgAACcoHACAExQOAMAJCgcA4ASFAwBwgsIBADhB4QAAnKBwAABOUDgAACcoHACAExQOAMAJCgcA4ASFAwBwgsIBADhB4QAAnKBwAABOUDgAACcoHACAExQOAMAJCgcA4ASFAwBwgsIBADjhMzOr7hA1UVhYmKKioqps+/v371fjxo2rbPuni3ynr6ZnJN/pIV/5cnJydODAgXLnUTjVJCEhQZmZmdUdwy/ynb6anpF8p4d8lccpNQCAExQOAMAJCqeaDB8+vLojnBD5Tl9Nz0i+00O+yuMaDgDACY5wAABOUDgAACconDOspKREnTt3Vr9+/SRJn376qZKSkhQXF6f+/fsrPz/fW3by5Mlq06aN2rdvr6VLl3rT169fr7i4OLVp00b33XefzuRZz5PNl5OTo3r16uniiy/WxRdfrJEjR1Z5vqioKMXFxeniiy9WQkKCJOngwYPq3bu32rZtq969eysvL89bvjrGrzIZa8oYvv7664qJiVFAQMDPbpN1PYaVyVdTxm/MmDHq0KGD4uPjNWjQIB06dMhbviaMn7981TF+FTKcUVOnTrVbbrnFrrvuOjMzS0hIsBUrVpiZ2cyZM+3xxx83M7PNmzdbfHy8HT161L766itr3bq1FRcXm5lZYmKiffTRR1ZaWmpXX321vf32287zff311xYTE1PuNqoqX8uWLW3//v1lpo0ZM8YmT55sZmaTJ0+2Rx991Myqb/wqk7GmjGFWVpZt2bLFevToYevWrfOmV8cYViZfTRm/pUuXWlFRkZmZPfroo9X6O1iZfNUxfhXhCOcM2rVrl9LT03XXXXd507Kzs9W9e3dJUu/evbVgwQJJ0qJFi5ScnKygoCC1atVKbdq00dq1a7Vnzx7l5+crKSlJPp9PQ4cO1VtvveU8nz9Vma88ixYtUmpqqiQpNTXV21d1jF9lM/rjOmPHjh3Vvn37n02vKWPoL58/rvP16dNHgYGBkqSuXbtq165dkmrO+PnL5091vEZ+QOGcQQ888ICmTJmigID/H9bY2FilpaVJOn7qYOfOnZKk3NxcRUZGesu1aNFCubm5ys3NVYsWLX423XU+Sfr666/VuXNn9ejRQ//5z3+83FWVz+fzqU+fPrrkkks0Y8YMSdLevXsVHh4uSQoPD9e+ffu8HK7Hr7IZpZoxhv5UxxhWJp9U88Zv1qxZuuaaa7wcNW38fpxPcj9+FQl0spdfgCVLlqhJkya65JJLtGLFCm/6rFmzdN9992nixIkaMGCA6tatK0nlnjP1+Xx+p7vOFx4erh07dqhRo0Zav369Bg4cqM2bN1dZPklatWqVIiIitG/fPvXu3VsdOnTwu6zr8TuVjDVlDH84gv2p6hjDyuSraeP31FNPKTAwUCkpKZJq3vj9NF91jF9FOMI5Q1atWqW0tDRFRUUpOTlZ77//voYMGaIOHTpo2bJlWr9+vW655RZFR0dLOv6/ih8fTezatUsRERFq0aJFmUPiH6a7zhcUFKRGjRpJki655BJFR0dr69atVZZPkredJk2aaNCgQVq7dq2aNm2qPXv2SDp+KqBJkyaS3I/fqWSsKWPoT3WMYWXy1aTxmzNnjpYsWaJ58+Z5b841afzKy1cd41chJ1eKfmGWL1/uXZTfu3evmZmVlJTYbbfdZjNnzjQzs02bNpW54NiqVSvvgmNCQoKtXr3au6CXnp7uPN++ffu8PNu2bbOIiAj73//+V2X5CgoKLD8/3/t3UlKSZWRk2COPPFLmgvyYMWPMrHrGr7IZa8oY/uCnF+Vdj2Fl89WU8cvIyLCOHTvavn37yixfU8bPXz7X43cyKJwq8OM39Oeee87atm1rbdu2tbFjx1ppaam33KRJk6x169bWrl27MneJrFu3zmJiYqx169b229/+tsw6rvK98cYb1qlTJ4uPj7fOnTtbWlpalebbtm2bxcfHW3x8vHXq1MkmTZpkZmYHDhywK6+80tq0aWNXXnml94Ixcz9+lc1YU8Zw4cKF1rx5c6tbt641adLE+vTp463jcgwrm6+mjF90dLS1aNHCLrroIrvoootsxIgR3jo1Yfz85XM9fieDr7YBADjBNRwAgBMUDgDACQoHAOAEhQMAcILCAQA4QeEAAJygcAAATvwfym6iFz1VaH4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "cycles = baseline['cycles']\n", + "fig, ax = plt.subplots()\n", + "ax.boxplot(cycles, vert=False, labels=[\"baseline\"])\n", + "ax.set_title('Time to tokenize VS Code')\n", + "ax.set_ylabel('Time / ms')\n", + "fig.patch.set_facecolor('white')\n", + "ax.set_facecolor('white')\n", + "plt.show()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's the current performance of the repo. Make sure to `npm run build` or `npm run watch` first!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "updated = run_benchmark('../')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZwAAAEICAYAAABrtkJsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfMElEQVR4nO3de1RU9foG8GcQJITiEkggKIqKCoxCg4J5BDMxSy07XjBMLAuro2UXzRYVaJIeSyu1Ms7R1OSQK7NEkbQIMkkFvGSKotlBLpqKooSJDMz7+6Nf+0iBaMJ3YHo+a81azL585303sh/3DXQiIiAiImpmVuYugIiI/hoYOEREpAQDh4iIlGDgEBGREgwcIiJSgoFDRERKMHCoxfP390dWVpa5y2hykyZNwksvvdTk4zo4OODHH39s8nFbqubajtT0GDhkdg4ODtrLysoKdnZ22vvk5GQcPHgQERERzV5HQkICJkyY8KfXbyk7vsrKSnTp0qVJxqqqqoKTkxO++uqrP8x75plnMHr0aADA9u3b0b9/fzg6OsLFxQV33HEHcnNzGxz3yJEjGDNmDFxdXeHo6Ai9Xo9Fixahtra2SeqmlomBQ2ZXWVmpvTp27IiNGzdq76Ojo81d3l/aTTfdhHHjxmH16tV1ptfW1iIlJQUxMTGoqKjA8OHDMW3aNJw7dw6lpaWIj4+Hra1tvWMeO3YM/fr1g7e3N77//ntcuHABH3/8MfLy8vDzzz+raIvMRYhakE6dOskXX3zR4LT4+HgZPXq0REdHi4ODgwQEBEhBQYG89tpr4ubmJl5eXrJlyxZt3fPnz8sjjzwit912m3h6ekpcXJzU1NT84XPT09PFxsZGrK2txd7eXvR6vYiIlJaWyogRI8TZ2Vl8fX0lKSmp3rrff/99sba2FhsbG7G3t5fhw4eLiEh+fr6Eh4eLo6Oj9OrVSzZs2KCtExMTI3FxcSIiUlFRIRERETJt2jQxmUxy6NAhueuuu8TZ2Vm6d+8ua9eurbPek08+Kffcc484ODhI37595YcfftDmA5CjR49KaWmp2Nvbay87Ozu58kd++fLl0qNHD3FycpLIyEgpLCyst7fs7GxxcHCQixcvatPS0tLEzc1NjEaj5ObmiqOjY73r1ic6Olruueeeqy6zYcMG6dWrlzg6Okp4eLjk5+dr8/bs2SNBQUHi4OAgY8eOlXHjxmnbUURk48aN0rt3b3F0dJSwsDD57rvvrrk2al4MHGpRriVwbG1t5fPPPxej0SgPPfSQ+Pj4yNy5c6W6ulqSkpLEx8dHW/e+++6T2NhYqayslFOnTklISIgsW7as3s+Oj4+X6OjoOtMGDhwoTzzxhFy6dEn27t0rrq6u8uWXX9a7/pUBIiJSXV0tvr6+kpiYKJcvX5aMjAxxcHCQw4cP11m+rKxMQkJCtHUrKyvFy8tLVqxYIUajUXbv3i233nqrHDhwQFvP2dlZdu3aJUajUR588EEZN26c9rm/Bc7vPfjggxIVFSUiIp9++qn4+vpKfn6+GI1GefXVVyUsLKzevkREunXrJh9++KH2PioqSp5++mkREblw4YK4uLjIxIkTZfPmzXLu3LkGxxERcXd3lxUrVjQ4v6CgQNq1aydbt26V6upq+ec//ym+vr5y+fJluXz5snTs2FEWLVok1dXV8vHHH4u1tbW27Xbv3i1ubm6yc+dOqampkZUrV0qnTp2kqqrqqjWRGgwcalGuJXDuuusubV5qaqrY29trRy0VFRUCQMrLy+Wnn36Stm3byi+//KIt/5///EciIiLq/ezfB05RUZFYWVlJRUWFNm3WrFkSExNT7/q/D5xt27aJu7u71NbWatOioqIkPj5eW/7hhx8Wf39/WbBggbbMRx99JAMGDKgzdmxsrCQkJGjrTZ48WZuXlpYmfn5+2vv6Amf+/PkSHBysbYu7775b/v3vf2vza2trxc7OrsGjnFdffVWGDBkiIr8GjJ2dnezZs0ebn5+fLzExMdKhQwdp06aNjBgxQn766ad6x7K2tpb09PR654mIzJkzR8aMGVOnNk9PT8nMzJSvv/5aPDw8xGQyafPDwsK07f7444/LSy+9VGe87t27S1ZWVoOfR+rwGg61Ou7u7trXdnZ2cHV1RZs2bbT3wK/XhY4fPw6j0QgPDw84OTnByckJU6ZMwenTp6/pc06cOAEXFxfcfPPN2rROnTqhtLT0mtf39vaGldX/fsx+v35aWhouXbqExx9/XJt2/Phx7Nq1S6vZyckJycnJ+Omnn7RlbrvtNu3rdu3aobKyssE60tPT8fbbb+Ozzz7Tts/x48fx9NNPa+O7uLhARBrsbeLEicjMzERpaSnWrVuHrl27IigoSJvfs2dPrFy5EiUlJThw4ABOnDiB6dOn1zvWrbfeipMnTzZY74kTJ9CpUyftvZWVFby9vVFaWooTJ06gQ4cO0Ol02vwrlz1+/DgWLlxYZ9sVFxfjxIkTDX4eqWNt7gKImou3tzdsbW1RVlYGa+vG/6lfuRMDAE9PT5w7dw4///yzFjpFRUXo0KHDNa9fXFwMk8mkhU5RURG6d++uLfPYY4+hvLwc99xzDz7//HPY29vD29sb4eHh+OKLL66r3/oUFBQgJiYG69evh7e3tzbd29sbcXFx13xTRseOHfG3v/0NycnJSE9Px8SJExtctkePHpg0aRLef//9euffdddd+OSTT/Dwww/XO9/T0xPff/+99l5EUFxcrAVNaWkpRETb3kVFRfD19a3TV1xc3DX1RWrxCIcsloeHByIjI/Hcc8+hoqICJpMJx44dw9dff13v8u7u7igsLITJZALw686rf//+ePHFF1FVVYX9+/dj+fLlDe6k3d3d6zz/0q9fP9jb22PBggUwGo3IysrCxo0bERUVVWe9pUuXws/PD8OHD8elS5cwfPhwHDlyBB9++CGMRiOMRiNyc3Nx6NCh6+q/oqIC9913H+bOnYsBAwbUmff4449j3rx5OHjwIABod4pdTUxMDJYuXYrs7Ow62+Dw4cNYuHAhSkpKAADFxcVISUlBaGhovePMnj0b3377LWbMmKEdtf3www+YMGECzp8/j7FjxyItLQ0ZGRkwGo1YuHAhbG1t0b9/f4SFhcHa2hqLFy9GTU0N1q9fj5ycHG3sxx57DMuWLcOuXbsgIrh48SLS0tJ491sLwcAhi7Z69WpUV1ejV69ecHZ2xujRoxs8nTNmzBgAv57yCQ4OBgCkpKSgsLAQnp6eGDVqFGbPno0hQ4bUu/7kyZORn58PJycn3H///Wjbti1SU1ORnp4OV1dXPPnkk1i9ejV69OhRZz2dToekpCR4e3vjvvvug42NDbZu3YqPPvoInp6euO222/DCCy/g8uXL19X7nj17UFBQgGeffbbOs04AMGrUKLzwwguIiorCLbfcgoCAAKSnp191vNGjR6O8vByDBw+Gh4eHNv3mm2/Grl27tIANDQ1FQEAAFi5cWO84vr6+2LFjBwoLC+Hv7w9HR0f8/e9/h8FgwM033ww/Pz+sWbMG06ZNg6urKzZu3IiNGzeibdu2aNu2LdavX4+VK1fC2dkZa9euxQMPPKCNbTAY8K9//QtTp06Fs7MzunbtipUrV17XdqPmoxPhH2AjIqLmxyMcIiJSgoFDRERKMHCIiEgJBg4RESnB53Aa4OrqCh8fH3OXQUTUqhQWFqKsrKzeeQycBvj4+CAvL8/cZRARtSoGg6HBeTylRkRESjBwiIhICQYOEREpwcAhIiIlGDhERKQEA4eIiJRg4BARkRIMHCIiUoKBQ0RESjBwiIhICQYOEREpwcAhIiIlGDhERKQEA4eIiJRg4BARkRIMHCIiUoKBQ0RESjBwiIhICQYOEREpwcAhIiIlGDhERKQEA4eIiJRg4BARkRIMHCIiUoKBQ0RESjBwiIhICQYOEREpwcAhIiIlGDhERKQEA4eIiJRg4BARkRIMHCIiUoKBQ0RESjBwiIhICQYOEREpwcAhIiIlGDhERKQEA4eIiJRg4BARkRIMHCIiUoKBQ0RESjBwiIhICQYOEREpwcAhIiIlGDhERKQEA6eFcXFxgU6na/SFBMdrWk6n08HFxcXcbRERMXBamvLycohIoy8A17SciKC8vNzMXRERMXCIiEgRBg4RESnBwGkGOp3O3CUQEbU4LTpwfHx8UFZWdtVlXnvttesed+XKlZg6deqfLYv+pJSUFAQEBKBNmzYICAhASkqKuUsiIoVadOBciz8TOKReSkoK4uLisGTJElRVVWHJkiWIi4tj6BD9hTRr4BQWFiIgIEB7/8YbbyAhIQERERGYPn06+vfvj4CAAOTk5AAAzp49i8jISAQFBWHKlCna3VgAcP/99+P222+Hv78/kpKSAACzZs3CpUuX0KdPH0RHRwMA1qxZg759+6JPnz6YMmUKamtrAQAffPABunfvjvDwcGRnZzdn21SPxMRELF++HIMGDYKNjQ0GDRqE5cuXIzEx0dylEZEiZjvCuXjxIr799lu8++67eOSRRwAAs2fPxoABA7B3716MHDkSRUVF2vIrVqzA7t27kZeXh8WLF+Ps2bOYP38+7OzssG/fPiQnJ+PQoUNYu3YtsrOzsW/fPrRp0wbJyck4efIk4uPjkZ2djS+++AL5+fn11pSUlASDwQCDwYAzZ87cUH/X+ozMH56vsVCHDh3CgAED6kwbMGAADh06ZKaKiEg1a3N98Pjx4wEAAwcOREVFBc6fP49t27Zh/fr1AIB7770Xzs7O2vKLFy/Gp59+CgAoLi7G0aNHceutt9YZMyMjA7t370ZISAgA4NKlS2jfvj127dqFiIgIuLm5AQDGjRuHI0eO/KGm2NhYxMbGAgAMBsMN9Xfl0dn1sNTQ6dmzJ7Zv345BgwZp07Zv346ePXuasSoiUqlZj3Csra1hMpm091VVVdrXv9+x/va+vh1uVlYWvvzyS+zYsQPfffcdgoKC6oz1GxFBTEwM9u3bh3379qGgoAAJCQkNjkvqxMXFYfLkycjMzITRaERmZiYmT56MuLg4c5dGRIo0a+C4u7vj9OnTOHv2LC5fvoxNmzZp89auXQvg1//lOjo6wtHREQMHDkRycjIAID09XXtC/sKFC3B2dka7du1w+PBh7Ny5UxvHxsYGRqMRADB48GCsW7cOp0+fBgCcO3cOx48fR79+/ZCVlYWzZ8/CaDTi448/bs62qR7jx49HYmIipk2bhptuugnTpk1DYmKidqRLRJavWU+p2djY4JVXXkG/fv3QuXNn9OjRQ5vn7OyM/v37o6KiAitWrAAAxMfHY/z48QgODkZ4eDg6duwIALj77ruxbNky6PV6+Pn5ITQ0VBsnNjYWer0ewcHBSE5Oxty5cxEZGQmTyQQbGxu88847CA0NRUJCAsLCwuDh4YHg4GDtZgJSZ/z48QwYor8wnfzZiw03ICIiAm+88cYNXydpTgaDAXl5eco/V6fTXdv1nwRHIOFC045JRHSDrrbvbPXP4RARUetglrvUsrKyzPGxRERkRjzCaYGu9Xmda32258rby4mIzMVsz+FQ/a7nWoskNF8dRERNjUc4RESkBAOHiIiUYOAQEZESDBwiIlKCgUNEREowcIiISAkGDhERKcHAISIiJRg4RESkBAOHiIiUYOAQEZESDBwiIlKCgUNEREowcIiISAkGDhERKcHAISIiJRg4RESkBAOHiIiUYOAQEZESDBwiIlKCgUNEREo0GjgXL16EyWQCABw5cgSpqakwGo3NXhgREVmWRgNn4MCBqKqqQmlpKQYPHowPPvgAkyZNUlAaERFZkkYDR0TQrl07rF+/HtOmTcOnn36K/Px8FbUREZEFuabA2bFjB5KTk3HvvfcCAGpqapq9MCIisiyNBs5bb72FefPmYdSoUfD398ePP/6IQYMGqaiNiIgsiE5ExNxFtEQGgwF5eXnmLoOIqFW52r7TurGV8/Ly8Nprr6GwsLDOqbT9+/c3XYVERGTxGg2c6OhovP766wgMDISVFR/bISKiP6fRwHFzc8PIkSNV1EJERBas0cCZPXs2Hn30UQwePBi2trba9AceeKBZCyMiIsvSaOB88MEHOHz4MIxGo3ZKTafTMXCIiOi6NBo43333Hb7//nsVtRARkQVr9C6A0NBQ/mYBIiK6YY0e4Wzfvh2rVq1C586dYWtrCxGBTqfjbdFERHRdGg2czz//XEUdRERk4RoNnE6dOqmog4iILByf5CQiIiUYOEREpESDgTN06FC8+eabOHz4sMp6iIjIQjUYOKtWrYKzszMSEhIQHByMJ554Ahs2bEBlZaXK+oiIyEJc058nMJlM2LVrF9LT05GRkQE7OztERkZi5syZKmo0C/55AiKi63dDf54AAKysrBAWFoawsDDMmTMHZWVl2LJlS5MWSURElu1P3TTg6uqK6Ojopq6FiIgsGO9SIyIiJRg4RESkRKOBc+rUKUyePBnDhg0DAOTn52P58uXNXhgREVmWRgNn0qRJGDp0KE6cOAEA6N69O956663mrouIiCxMo4FTVlaGsWPHan98zdraGm3atGn2woiIyLI0Gjj29vY4e/YsdDodAGDnzp1wdHRs9sKIiMiyNPoczqJFizBy5EgcO3YMd9xxB86cOYN169apqI2IiCxIo4ETHByMr7/+GgUFBRAR+Pn5wcbGRkVtRERkQRoNnNraWmzevBmFhYWoqanB1q1bAQDPPvtssxdHRESWo9HAGTFiBG666SYEBgZqNw4QERFdr0YDp6SkBPv371dRCxERWbBGD1mGDRumnUYjIiL6sxo9wgkNDcWoUaNgMplgY2MDEYFOp0NFRYWK+oiIyEI0GjjPPfccduzYgcDAQO1ZHCIiouvV6Cm1bt26ISAggGFDREQ3pNEjHA8PD0RERGDYsGGwtbXVpvO2aCIiuh6NBk7nzp3RuXNnVFdXo7q6WkVNRERkgRoNnPj4eBV1EBGRhWswcKZOnYqlS5dixIgR9V6/SU1NbdbCiIjIsjQYOKtXr8bSpUvx/PPPq6yHiIgsVIOB4+vrCwAIDw9XVgwREVmuBgPnzJkzWLRoUYMr8i41IiK6Hg0GTm1tLSorKyEiKushIiIL1WDgeHh44JVXXlFZCxERWbAGf9MAj2yIiKgpNRg4GRkZKusgIiIL12DguLi4qKyDiIgsHP+EJxERKcHAISIiJRg4RESkBAOHiIiUYOAQEZESDBwiIlKCgUNEREowcIiISAkGDhERKcHAISIiJRg4RESkBAOHiIiUYOAQEZESDBwiIlKCgUNEREowcIiISAkGDhERKcHAISIiJRg4RESkBAOHiCySi4sLdDrdVV9IcGx0mdbycnFxMfcmb5S1uQsgImoO5eXlEJGrL5Tg2PgyrYROpzN3CY3iEQ4RESnBwCEiIiUYOESkTGs47UPN931qtsApLCxEQEBAs4ydlZWF4cOHAwBSU1Mxf/78ZvkcIiJqOq3+poGRI0di5MiR5i6DiIga0ayn1GpqahATEwO9Xo/Ro0fjl19+wZw5cxASEoKAgADExsZqd4gsXrwYvXr1gl6vR1RUFADg4sWLeOSRRxASEoKgoCBs2LDhD5+xcuVKTJ06FQAwadIkPPXUU+jfvz+6dOmCdevWacu9/vrrCAkJgV6vR3x8fHO2TURE9WjWwCkoKEBsbCz279+PW265Be+++y6mTp2K3NxcHDhwAJcuXcKmTZsAAPPnz8fevXuxf/9+LFu2DACQmJiIO++8E7m5ucjMzMSMGTNw8eLFq37myZMnsX37dmzatAmzZs0CAGzduhVHjx5FTk4O9u3bh927d2Pbtm1/WDcpKQkGgwEGgwFnzpxp4q1BRACUPZfyV9TSt12zBo63tzfuuOMOAMCECROwfft2ZGZmol+/fggMDMRXX32FgwcPAgD0ej2io6OxZs0aWFv/eqZv69atmD9/Pvr06YOIiAhUVVWhqKjoqp95//33w8rKCr169cKpU6e0cbZu3YqgoCAEBwfj8OHDOHr06B/WjY2NRV5eHvLy8uDm5taUm4KI/p+IKHn9FbX0bdes13B+n5Q6nQ5PPvkk8vLy4O3tjYSEBFRVVQEA0tLSsG3bNqSmpuLVV1/FwYMHISL45JNP4OfnV2ec34KkPra2ttrXv204EcGLL76IKVOmNFVrRER0nZr1CKeoqAg7duwAAKSkpGDAgAEAAFdXV1RWVmrXWEwmE4qLizFo0CAsWLAA58+fR2VlJYYOHYolS5ZowbF3794/VcfQoUOxYsUKVFZWAgBKS0tx+vTpG22PiIiuQ7Me4fTs2ROrVq3ClClT0K1bNzzxxBMoLy9HYGAgfHx8EBISAgCora3FhAkTcOHCBYgInnnmGTg5OeHll1/G9OnTodfrISLw8fHRrvlcj8jISBw6dAhhYWEAAAcHB6xZswbt27dv0n6JiKhhOvmrnuxshMFgQF5enrnLILIoOp1O2fWVa/qsBEcg4YKSeppbU27bGxnravtO/qYBIlKG/79tHZrr+8TAISIiJRg4RESkRKv/1TZERA1p7CFGib/FYh4SdXZ2NncJjWLgEJFFutbrEJLQvHXQ//CUGhERKcHAISIiJRg4RESkBAOHiIiUYOAQEZESDBwiIlKCgUNEREowcIiISAkGDhERKcHAISIiJRg4RESkBAOHiIiUYOAQEZESDBwiIlKCgUNEREowcIiISAkGDhERKcHAISIiJRg4RESkBAOHiIiUYOAQEZESDBwiIlKCgUNEREowcIiISAkGDhERKcHAISIiJRg4RESkBAOHiIiUYOAQEZESDBwiIlKCgUNEREowcIiISAkGDhERKcHAISIiJRg4RESkBAOHiIiUYOAQEZESDBwiIlKCgUNEREowcIiISAkGDhERKcHAISIiJRg4RESkBAOHiIiU0ImImLuIlsjV1RU+Pj5NPu6ZM2fg5ubW5OO2BJbcG8D+Wjv2p0ZhYSHKysrqncfAUcxgMCAvL8/cZTQLS+4NYH+tHfszP55SIyIiJRg4RESkBANHsdjYWHOX0GwsuTeA/bV27M/8eA2HiIiU4BEOEREpwcAhIiIlGDhNoLa2FkFBQRg+fDgAYMaMGejRowf0ej1GjRqF8+fPa8vOmzcPXbt2hZ+fH7Zs2aJN3717NwIDA9G1a1c89dRTaElnOn/f32/eeOMN6HS6OvfcW0p/S5YsgZ+fH/z9/TFz5kxtuiX0t2/fPoSGhqJPnz4wGAzIycnRlm1t/fn4+CAwMFDrBQDOnTuHIUOGoFu3bhgyZAjKy8u15S2hv1a9fxG6YQsXLpTx48fLvffeKyIiW7ZsEaPRKCIiM2fOlJkzZ4qIyMGDB0Wv10tVVZX8+OOP0qVLF6mpqRERkZCQEPn222/FZDLJ3XffLZs3bzZPM/X4fX8iIkVFRRIZGSkdO3aUM2fOiIjl9PfVV1/J4MGDpaqqSkRETp06JSKW09+QIUO0+tLS0iQ8PFxEWmd/nTp10v79/WbGjBkyb948ERGZN29eq/75q6+/1rx/4RHODSopKUFaWhoeffRRbVpkZCSsra0BAKGhoSgpKQEAbNiwAVFRUbC1tUXnzp3RtWtX5OTk4OTJk6ioqEBYWBh0Oh0mTpyIzz77zBzt/EF9/QHAM888gwULFkCn02nTLKW/9957D7NmzYKtrS0AoH379gAspz+dToeKigoAwIULF+Dp6QmgdfZXnw0bNiAmJgYAEBMTo9VqKf215v0LA+cGTZ8+HQsWLICVVf2bcsWKFRg2bBgAoLS0FN7e3to8Ly8vlJaWorS0FF5eXn+Y3hLU119qaio6dOiA3r1711nWUvo7cuQIvvnmG/Tr1w/h4eHIzc0FYDn9vfXWW5gxYwa8vb3x/PPPY968eQBaZ386nQ6RkZG4/fbbkZSUBAA4deoUPDw8AAAeHh44ffo0AMvp70qtbf/CwLkBmzZtQvv27XH77bfXOz8xMRHW1taIjo4GgHrPm+p0uganm1t9/f3yyy9ITEzEnDlz/rC8JfQHADU1NSgvL8fOnTvx+uuvY+zYsRARi+nvvffew5tvvoni4mK8+eabmDx5MoDW9/0DgOzsbOzZswfp6el45513sG3btgaXtbT+WuP+xdosn2ohsrOzkZqais2bN6OqqgoVFRWYMGEC1qxZg1WrVmHTpk3IyMjQvrleXl4oLi7W1i8pKYGnpye8vLy0w+Irp5tbff099NBD+O9//6sd3ZSUlCA4OBg5OTkW0d+ECRPg5eWFBx54ADqdDn379oWVlRXKysospr+NGzfi7bffBgCMGTNGO93W2voDoNXRvn17jBo1Cjk5OXB3d8fJkyfh4eGBkydPaqdELaW/gQMHtt79i1muHFmgzMxM7aJsenq69OzZU06fPl1nmQMHDtS5qNe5c2ftop7BYJAdO3ZoF/XS0tKU93A1V/Z3pSsvalpKf++99568/PLLIiJSUFAgXl5eYjKZLKa/Hj16SGZmpoiIfPnllxIcHCwire/7V1lZKRUVFdrXYWFhkp6eLs8//3ydmwZmzJghIpbTX2vevzBwmsiVP9C+vr7i5eUlvXv3lt69e8uUKVO05ebOnStdunSR7t2717lTJDc3V/z9/aVLly7yj3/8Q0wmk/IeruZaAkfEMvq7fPmyREdHi7+/vwQFBUlGRoa2nCX0980330hwcLDo9Xrp27ev5OXlacu1pv6OHTsmer1e9Hq99OrVS+bOnSsiImVlZXLnnXdK165d5c4775SzZ89q61hCf615/8JfbUNERErwpgEiIlKCgUNEREowcIiISAkGDhERKcHAISIiJRg4RESkBAOHiIiU+D+wGdQk21EpzQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "fig, ax = plt.subplots()\n", + "ax.boxplot([baseline['cycles'], updated['cycles']], vert=False, labels=[\"baseline\", \"updated\"])\n", + "ax.set_title('Time to tokenize VS Code')\n", + "ax.set_ylabel('Time / ms')\n", + "fig.patch.set_facecolor('white')\n", + "ax.set_facecolor('white')\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEICAYAAABYoZ8gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAi60lEQVR4nO3deVxU5f4H8A+biguLCgqCgJqKKCCMiVwVlEB8uaSZNxESMUONNOtSWpq7RaXXxBYjveAtw25WuKIpaW64jAvXJc0UUEBhEAERQZbn94evzo0fTqPAOYrn8369euk858zzfGcaP3PmOZuREEKAiIhUw/hRF0BERMpi8BMRqQyDn4hIZRj8REQqw+AnIlIZBj8Rkcow+OmxsXfvXjg4ODR4v1OnTsXixYsbvN+hQ4di3bp1Dd7v42LBggUICwt71GWQDBj8JIuWLVtK/xkbG8Pc3Fx6vH79ekVrWb16Nd5999169XG/EExOTkZ4eHi9+r2frKwsjBkzBm3btoWlpSV69eqFhISEBh+H1Mv0URdAT6aSkhLp787OzlizZg2eeeaZR1hR4/Hiiy/Cw8MDmZmZaNq0KU6fPo3r168/6rLoCcItflJUeXk5Zs6cCXt7e9jb22PmzJkoLy+/77qxsbHo0aMHsrKyUF5ejujoaHTs2BHt2rXD1KlTcefOHQD/myJavnw5bG1tYWdnh/j4eKmfiRMnYu7cuQCAESNG1Po18sfW9GuvvQZHR0dYWFjA29sb+/fvBwDs2LED7733Hr799lu0bNkSHh4eAAB/f3+sWbMGAFBdXY0lS5bAyckJtra2mDBhAoqKigAAGRkZMDIywrp169CxY0e0bdsWS5cu1fseHTt2DBMnTkSLFi1gamqK3r17Y+jQoTX6iouLg729Pezs7LB8+XLpudXV1YiJiUHnzp3Rpk0b/P3vf0dBQYG0/PDhw/D19YWVlRU8PDywd+9eaVl6ejr8/PzQqlUrBAYGIj8/3/D/UGqUGPykqKVLl+Lw4cM4deoU0tLScPToUSxZsqTWeosXL0ZCQgJ++eUXODg4YNasWfjtt99w6tQp/P7778jOzsaiRYuk9a9fv46ioiJkZ2dj7dq1iIqKws2bN2v1u2XLFpSUlKCkpAQbN25E+/btERAQAADo06cPTp06hYKCAowfPx5jx45FWVkZgoOD8c477+CFF15ASUkJ0tLSavWbkJCAhIQE7NmzB5cvX0ZJSQleffXVGuscOHAAFy5cQEpKChYtWoRff/31vu+Rj48PoqKisGHDBly5cuW+6+zZswcXL17ETz/9hJiYGOzevRvAvS/LpKQk/PLLL8jJyYG1tTWioqIAANnZ2Rg2bBjmzp2LgoICLFu2DGPGjIFOpwMAjB8/Ht7e3sjPz8e77777RO+/UD1BJDMnJyexa9cuIYQQnTp1Etu2bZOW7dixQzg5OQkhhNizZ4+wt7cXr7/+uvjb3/4mCgsLhRBCVFdXi+bNm4vff/9det6hQ4eEs7Oz9LxmzZqJiooKabmNjY1ITU0VQggRHh4u5syZU6OmCxcuCBsbG7Fv3z69dVtZWYlTp04JIYSYP3++CA0NrbHcz89PfPnll0IIIQYPHiw+/fRTadn58+eFqampqKioEOnp6QKAuHr1qrS8T58+IjEx8b7jFhQUiFmzZokePXoIY2Nj4eHhIY4ePSqEEFJfv/76q7T+m2++KSZNmiSEEKJ79+5i9+7d0rKcnBypjpiYGBEWFlZjrKCgIJGQkCAyMzOFiYmJKCkpkZaFhITUes30ZOAWPykqJycHTk5O0mMnJyfk5ORIjwsLCxEXF4e3334blpaWAACdTofS0lJ4e3vDysoKVlZWCA4OlrZUAaBNmzYwNf3fLqvmzZvX2M/wZ0VFRXj22WexePFiDBgwQGpfvnw5XF1dYWlpCSsrKxQVFT3wdMf9XldlZSVyc3Oltvbt2z9QfdbW1oiJicHZs2eRm5sLT09PjBo1CuJP11N0dHSsMdYf72FmZiZGjx4tvU+urq4wMTFBbm4uMjMz8d1330nLrKyscODAAVy7dk36ddCiRYsa/dKTicFPirK3t0dmZqb0+MqVK7C3t5ceW1tbY+vWrYiIiMDBgwcBAG3btoW5uTnOnj2LwsJCFBYWoqioSG9w/pXq6mqMHz8egwYNwpQpU6T2/fv344MPPsB//vMf3Lx5E4WFhbC0tJTC1sjI6KFfl6mpKdq1a/fQNf5Z27ZtER0djZycnBpz9VevXq0x1h/voaOjI5KTk6X3qbCwEGVlZejQoQMcHR3x4osv1lh2+/ZtzJ49G3Z2drh58yZu375do196MjH4SVEhISFYsmQJdDod8vPzsWjRolqHSfr7+2P9+vUYPXo0jhw5AmNjY7z88st4/fXXkZeXB+DefPXOnTsfevw5c+bg9u3bWLlyZY32W7duwdTUFDY2NqisrMSiRYtQXFwsLW/Xrh0yMjJQXV2t93WtWLEC6enpKCkpkfYJ/PlXyIOaNWsWzpw5g8rKSty6dQuff/45unTpgjZt2kjrLF68GKWlpTh79izi4+PxwgsvALh3zsKcOXOkLyGdTodNmzYBAMLCwrBlyxbs3LkTVVVVKCsrw969e5GVlQUnJydoNBrMnz8fd+/exYEDB7Bly5aHrp0aBwY/KWru3LnQaDRwd3dHr1694OXlJR1x82eBgYGIj4/HyJEjcfz4cXzwwQfo0qULfHx8YGFhgWeeeQYXLlx46PETExNx+PBhWFtb1zivYMiQIRg6dCi6du0KJycnNGvWrMZ0ytixYwHcm1Ly8vKq1e+kSZPw4osvYuDAgXBxcUGzZs2watWqh64PAEpLS6Xpmk6dOiEzMxObN2+usY6fnx+6dOmCgIAAREdHIygoCMC9I5NGjhyJoKAgtGrVCj4+Pjhy5AiAe78GNm3ahPfeew82NjZwdHTERx99JH2ZffPNNzhy5Ahat26NhQsXYsKECXWqnx5/RkLwRixEjUVGRgZcXFxQUVFRp18TRAC3+ImIVIfBT0SkMpzqISJSGW7xExGpTKPYO9S2bVs4Ozs/6jKIiBqVjIyM+56E2CiC39nZGVqt9lGXQUTUqGg0mvu2c6qHiEhlGPxERCrD4CciUhkGPxGRyjD4iYhUhsFPRKQyDH4iIpVh8BMRqQyDn4hIZRrFmbtETzLn2dsedQn0GMuIGdbgfXKLn4hIZRj8REQqw+AnIlIZBj8Rkcow+ImIVIbBT0SkMgx+IiKVYfATEakMg5+ISGUY/EREKsPgJyJSGdmCf9KkSbC1tUXPnj2ltoKCAgQGBuKpp55CYGAgbt68KdfwRESkh2zBP3HiROzYsaNGW0xMDAICAnDx4kUEBAQgJiZGruGJiEgP2YJ/4MCBaN26dY22TZs2ITw8HAAQHh6OpKQkuYYnIiI9FL0sc25uLuzs7AAAdnZ2yMvL07tuXFwc4uLiAAA6nU6R+oiI1OCx3bkbGRkJrVYLrVYLGxubR10OEdETQ9Hgb9euHa5duwYAuHbtGmxtbZUcnoiIoHDwjxw5EuvWrQMArFu3Ds8++6ySwxMREWQM/pCQEPTr1w8XLlyAg4MD1q5di9mzZ2PXrl146qmnsGvXLsyePVuu4YmISA/Zdu4mJibetz0lJUWuIYmI6AE8tjt3iYhIHgx+IiKVYfATEakMg5+ISGUY/EREKsPgJyJSGQY/EZHK1Cn44+PjG7oOIiJSSJ2Cf/78+Q1dBxERKUTvmbvu7u73bRdCIDc3V7aCiIhIXnqDPzc3Fzt37oS1tXWNdiEEfH19ZS+MiIjkoTf4hw8fjpKSEnh6etZa5u/vL2NJDct59rZHXQI9pjJihj3qEogeCb3Bv3btWr1P+uabb2QphoiI5PdQO3f/uBUiERE1Xg8V/KtXr5arDiIiUshDBb8QQq46iIhIIQ8V/Fu2bJGrDiIiUojB4L9165b0dwcHB1mLISIi+f1l8GdnZ2P48OFK1UJERArQezjn2bNnMW7cOHz55ZdK1kNERDLTG/yDBg3Cpk2b4OPjo2Q9REQkM71TPX369MH333+vZC1ERKQAvcG/efNmFBcX46233lKyHiIikpne4DcxMUFcXBxatmypZD1ERCQzg4dzzps3T4k6iIhIIQaD/9KlSygvLwcA7N27F7GxsSgsLJS7LiIikonB4B8zZgxMTEzw+++/46WXXkJ6ejrGjx+vRG1ERCQDg8FvbGwMU1NT/Pjjj5g5cyZWrFiBa9euKVEbERHJwGDwm5mZITExEevWrZPO4q2oqJC9MCIikofB4I+Pj0dqairmzJkDFxcXpKenIywsrF6DrlixAm5ubujZsydCQkJQVlZWr/6IiOjBGQz+Hj16IDY2FiEhIQAAFxcXzJ49u84DZmdnIzY2FlqtFmfOnEFVVRU2bNhQ5/6IiOjh6A3+ixcvYuLEiXjjjTeQlZWFoUOHokWLFvDw8IBWq63XoJWVlbhz5w4qKytRWloKe3v7evVHREQPTm/wR0REwNfXF/b29ujbty8mTZqEGzduYNmyZYiKiqrzgB06dEB0dDQ6duwIOzs7WFpaIigoqNZ6cXFx0Gg00Gg00Ol0dR6PiIhq0hv8JSUliIyMRHR0NMzNzTF27Fg0a9YMgYGB0nH9dXHz5k1s2rQJ6enpyMnJwe3bt/H111/XWi8yMhJarRZarRY2NjZ1Ho+IiGrSG/zGxv9bZGFhoXfZw9q9ezdcXFxgY2MDMzMzPPfcczh06FCd+yMiooej97LM58+fh7u7O4QQuHTpEtzd3QHcu+/u5cuX6zxgx44dcfjwYZSWlsLc3BwpKSnQaDR17o+IiB6O3uD/9ddfZRmwb9++eP755+Hl5QVTU1P07t0bkZGRsoxFRES16Q1+Jycn2QZduHAhFi5cKFv/RESkn97gb9WqFYyMjKTHQggYGRlJfxYXFytSIBERNSy9wR8QEIDr16/jueeew7hx49CxY0cl6yIiIpnoPTwnKSkJO3fuhI2NDV5++WX4+fnhs88+Q0FBgZL1ERFRA/vL4zItLS0RERGB5ORkTJ06FfPmzUNCQoJCpRERkRz0TvUAwKFDh5CYmIj9+/ejf//++PHHHzFgwAClaiMiIhnoDX5nZ2dYWVlh3LhxiIuLg6npvVVPnDgBAPDy8lKmQiIialB/GfxGRkbYuXMnfvrpJwghpGVGRkb4+eefFSmQiIgalt7g37t3r4JlEBGRUup+0R0iImqUGPxERCqjN/grKyuVrIOIiBSid47fx8cHDg4OCA4ORnBwMJydnRUsi4iI5KI3+LVaLTIzM5GcnIyZM2ciOzsb/fv3x9ChQ+Hn54emTZsqWScRETWQv5zjd3JywtSpU5GUlIRDhw5hxIgR2L17NwYMGIBhw4YpVSMRETWgvzxz98/MzMwwePBgDB48GACQnZ0tW1FERCSfOh/V06FDh4asg4iIFMLDOYmIVIbBT0SkMnrn+EeMGFHjDlz/3+bNm2UpiIiI5KU3+KOjowEAP/zwA65fv46wsDAAQGJiIo/pJyJqxPQGv5+fHwDg3Xffxb59+6T2ESNGYODAgfJXRkREsjA4x6/T6XD58mXpcXp6OnQ6naxFERGRfAwex79ixQr4+/ujU6dOAICMjAx88cUXshdGRETyMBj8wcHBuHjxIs6fPw8A6N69Oy/XQETUiBmc6iktLcVHH32ETz75BB4eHrhy5Qq2bt2qRG1ERCQDg8EfERGBJk2aIDU1FQDg4OCAuXPnyl4YERHJw2DwX7p0CW+99RbMzMwAAObm5jXuv0tERI2LweBv0qQJ7ty5I53MdenSJc7xExE1YgZ37i5cuBDBwcG4evUqQkNDcfDgQSQkJChQGhERycFg8AcGBsLLywuHDx+GEAIrV65E27ZtlaiNiIhkYHCqRwiB5ORkHD9+HMOHD0dpaSmOHj1ar0ELCwvx/PPPo3v37nB1dZV2HBMRkfwMBv8rr7yC1NRUJCYmAgBatWqFqKioeg362muvITg4GOfPn0daWhpcXV3r1R8RET04g1M9R44cwYkTJ9C7d28AgLW1Ne7evVvnAYuLi7Fv3z5pP0GTJk3QpEmTOvdHREQPx+AWv5mZGaqqqqSjenQ6HYyN634Z/8uXL8PGxgYRERHo3bs3Jk+ejNu3b9daLy4uDhqNBhqNhtcGIiJqQAYTfMaMGRg9ejRyc3MxZ84c9O/fH++8806dB6ysrMSJEycwbdo0nDx5Ei1atEBMTEyt9SIjI6HVaqHVamFjY1Pn8YiIqCaDUz2hoaHw9vZGSkoKACApKalec/IODg5wcHBA3759AQDPP//8fYOfiIjk8UBzNqWlpaiqqkJ1dTXu3LlTrwHbt28PR0dHXLhwAQCQkpKCHj161KtPIiJ6cAaDf9GiRQgPD0dBQQHy8/MRERGBJUuW1GvQVatWITQ0FO7u7jh16lS9po6IiOjhGJzqSUxMxMmTJ9GsWTMAwOzZs+Hl5VWvC7V5enpCq9XW+flERFR3Brf4nZ2dUVZWJj0uLy9H586dZS2KiIjkY3CLv2nTpnBzc0NgYCCMjIywa9cu9O/fHzNmzAAAxMbGyl4kERE1HIPBP3r0aIwePVp67O/vL2c9REQkM4PBHx4eDgCoqKjAmTNn0KFDB9ja2speGBERyUPvHP/UqVNx9uxZAEBRURE8PDwwYcIE9O7dW7puDxERNT56g3///v1wc3MDAMTHx6Nr1644ffo0jh8/jg8//FCxAomIqGHpDf4/Xzht165dGDVqFIB7J2AREVHjpTf4rayssHXrVpw8eRIHDx5EcHAwgHvX2qnv2btERPTo6N25+8UXX2DGjBm4fv06Pv74Y2lLPyUlBcOGDVOsQCIialh6g79r167YsWNHrfYhQ4ZgyJAhshZFRETyqfuF9YmIqFFi8BMRqQyDn4hIZQyeuVteXo7vv/8eGRkZqKyslNrnzZsna2FERCQPg8H/7LPPwtLSEt7e3mjatKkSNRERkYwMBn9WVtZ9j+4hIqLGyeAcv6+vL06fPq1ELUREpACDW/wHDhxAQkICXFxc0LRpUwghYGRkhP/+979K1EdERA3MYPAnJycrUQcRESnEYPA7OTkhLS0N+/fvBwAMGDAAHh4eshdGRETyMDjHv3LlSoSGhiIvLw95eXkICwvDqlWrlKiNiIhkYHCLf+3atThy5AhatGgBAJg1axb69euH6dOny14cERE1PINb/EIImJiYSI9NTEwghJC1KCIiko/BLf6IiAj07dtXuuF6UlISXnrpJdkLIyIieRgM/jfeeAP+/v44cOAAhBCIj49H7969laiNiIhkoDf4i4uLYWFhgYKCAjg7O8PZ2VlaVlBQgNatWytRHxERNTC9wT9+/Hhs3boV3t7eMDIyktr/OIHr8uXLihRIREQNS2/wb926FQCQnp6uWDFERCQ/g0f1BAQEPFAbERE1Dnq3+MvKylBaWor8/HzcvHlTOoSzuLgYOTk59R64qqoKGo0GHTp0kH5dEBGR/PQG/xdffIGPP/4YOTk58Pb2loLfwsICUVFR9R545cqVcHV1RXFxcb37IiKiB6d3que1115Deno6li1bhsuXLyM9PR3p6elIS0vDq6++Wq9Bs7KysG3bNkyePLle/RAR0cMzeBz/9OnTcebMGZw7dw5lZWVS+4QJE+o86MyZM/Hhhx/i1q1beteJi4tDXFwcAECn09V5LCIiqsngzt2FCxdi+vTpmD59Ovbs2YO33noLmzdvrvOAW7duha2tLby9vf9yvcjISGi1Wmi1WtjY2NR5PCIiqslg8G/cuBEpKSlo37494uPjkZaWhvLy8joPePDgQWzevBnOzs4YN24cfv75Z4SFhdW5PyIiejgGg9/c3BzGxsYwNTVFcXExbG1t63Xy1vvvv4+srCxkZGRgw4YNGDx4ML7++us690dERA/H4By/RqNBYWEhXn75ZXh7e6Nly5Z4+umnlaiNiIhkYDD4P/vsMwDA1KlTERwcjOLiYri7uzfI4P7+/vD392+QvoiI6ME80Jm727dvBwA4OzvD3d0dkZGRshdGRETyMBj86enp+OCDD7Bw4UKpTavVyloUERHJx2DwW1lZISUlBbm5uRgxYgSKioqUqIuIiGTyQLdeNDU1xWeffYYxY8agf//+yMvLU6I2IiKSgcGdu1OnTpX+PnHiRPTq1QuffvqprEUREZF8DN6Ba+zYsSgoKJDaXVxcsGzZMkWKIyKihvfAd+D64+qcAHgHLiKiRox34CIiUhmDc/wAkJ2djczMTFRWVkptAwcOlK0oIiKSj8HgnzVrFr799lv06NEDJiYmAO5N9TD4iYgaJ4PBn5SUhAsXLqBp06ZK1ENERDIzeBx/p06dUFFRoUQtRESkAINb/M2bN4enpycCAgJqbPXHxsbKWhgREcnDYPCPHDkSI0eOVKIWIiJSgMHgDw8Px507d3DlyhV069ZNiZqIiEhGBuf4t2zZAk9PTwQHBwMATp06xV8ARESNmMHgX7BgAY4ePQorKysAgKenJ0/qIiJqxAwGv6mpKSwtLWu0GRkZyVYQERHJy2Dw9+zZE9988w2qqqpw8eJFTJ8+Hb6+vkrURkREMjAY/KtWrcLZs2fRtGlTjB8/HhYWFli5cqUStRERkQwMBn9iYiKWLl2KY8eO4dixY1i6dCnmz5+vRG1ERCQDg4dzbty4Ec2aNUNoaCgAICoqCmVlZbIXRkRE8jAY/D/88ANGjhwJY2NjJCcno3Xr1rwDFxFRI6Y3+P981601a9Zg1KhR+Nvf/oZ58+ahoKAArVu3VqRAIiJqWHqD/8933vrjz23btmHbtm28AxcRUSOmN/h5khYR0ZPJ4Bx/RUUFPv/8c+zbtw8A4O/vjylTpsDMzEz24oiIqOEZDP5p06ahoqICr7zyCgDgq6++wrRp07BmzRrZiyMiooanN/grKythamqKY8eOIS0tTWofPHgwPDw8FCmOiIgant4TuJ5++mkAgImJCS5duiS1X758Wbr3bl1cvXoVgwYNgqurK9zc3HgWMBGRwvRu8QshAADLli3DoEGD0KlTJwBARkYG4uPj6z6gqSmWL18OLy8v3Lp1C97e3ggMDESPHj3q3CcRET04vcGv0+nwz3/+EwAwZcoUVFVVoUWLFigrK8PJkycxaNCgOg1oZ2cHOzs7AECrVq3g6uqK7OxsBj8RkUL0Bn9VVRVKSkqkLX8AKCkpAQDcunWrQQbPyMjAyZMn0bdv31rL4uLiEBcXB+DelxARETUMvcFvZ2eHefPmyTZwSUkJxowZg48//hgWFha1lkdGRiIyMhIAoNFoZKuDiEht9O7c/fOWfkOrqKjAmDFjEBoaiueee062cYiIqDa9wZ+SkiLLgEIIvPTSS3B1dcUbb7whyxhERKSf3uCX6yJsBw8exFdffYWff/4Znp6e8PT0xPbt22UZi4iIajN45m5D69+/v6zTSERE9NcM3oGLiIieLAx+IiKVYfATEakMg5+ISGUY/EREKsPgJyJSGQY/EZHKMPiJiFSGwU9EpDIMfiIilWHwExGpDIOfiEhlGPxERCrD4CciUhkGPxGRyjD4iYhUhsFPRKQyDH4iIpVh8BMRqQyDn4hIZRj8REQqw+AnIlIZBj8Rkcow+ImIVIbBT0SkMgx+IiKVYfATEakMg5+ISGUY/EREKsPgJyJSmUcS/Dt27EC3bt3QpUsXxMTEPIoSiIhUS/Hgr6qqQlRUFJKTk3Hu3DkkJibi3LlzSpdBRKRaigf/0aNH0aVLF3Tq1AlNmjTBuHHjsGnTJqXLICJSLVOlB8zOzoajo6P02MHBAUeOHKm1XlxcHOLi4gAA58+fh0ajqdN4betW5hNLp9PBxsbmUZfxWNBo5j/qEgDwM/r/8TNaU30+pxkZGfdtVzz4hRC12oyMjGq1RUZGIjIyUomSVEWj0UCr1T7qMoj04mdUfopP9Tg4OODq1avS46ysLNjb2ytdBhGRaike/H369MHFixeRnp6Ou3fvYsOGDRg5cqTSZRARqZbiUz2mpqb45JNPMGTIEFRVVWHSpElwc3NTugzV4vQZPe74GZWfkbjfpDsRET2xeOYuEZHKMPiJiFSGwf8YMzExgaenJzw8PODl5YVDhw41aP8TJ07Exo0bAQCTJ0/mGdRUJxkZGejZs2eNtgULFmDZsmUP3IezszPy8/P/cp333nvvoWtLSEjAq6+++tDPe9Ix+B9j5ubmOHXqFNLS0vD+++/j7bfflm2sNWvWoEePHrL1T1RfdQl+uj8GfyNRXFwMa2trAEBJSQkCAgLg5eWFXr16SZe8uH37NoYNGwYPDw/07NkT3377LQDg+PHj8PPzg7e3N4YMGYJr167V6t/f3186aaZly5aYM2cOPDw84OPjg9zcXAD3zqgcM2YM+vTpgz59+uDgwYNKvHRqxPz9/TFz5kz4+vqiZ8+eOHr0KADgxo0bCAoKQu/evTFlypQaJ3aOGjUK3t7ecHNzk87enz17Nu7cuQNPT0+EhoYCAL7++ms8/fTT8PT0xJQpU1BVVQUAiI+PR9euXeHn58fPqD6CHlvGxsbCw8NDdOvWTVhYWAitViuEEKKiokIUFRUJIYTQ6XSic+fOorq6WmzcuFFMnjxZen5hYaG4e/eu6Nevn8jLyxNCCLFhwwYREREhhBAiPDxcfPfdd0IIIfz8/MSxY8eEEEIAEJs3bxZCCPHmm2+KxYsXCyGECAkJEfv37xdCCJGZmSm6d+8u91tAjUB6erpwc3Or0TZ//nzx0UcfCT8/P+kz+csvv0jrTZ8+XSxcuFAIIcTWrVsFAKHT6YQQQty4cUMIIURpaalwc3MT+fn5QgghWrRoIfV/7tw5MXz4cHH37l0hhBDTpk0T69atEzk5OcLR0VHk5eWJ8vJy4evrK6KiomR89Y2T4sfx04P7Y6oHAFJTUzFhwgScOXMGQgi888472LdvH4yNjZGdnY3c3Fz06tUL0dHRmDVrFoYPH44BAwbgzJkzOHPmDAIDAwHcuzqqnZ3dX47bpEkTDB8+HADg7e2NXbt2AQB2795dYz9AcXExbt26hVatWsnw6qmxuN8lV/7cHhISAgAYOHAgiouLUVhYiH379uGHH34AAAwbNkz6NQsAsbGx+PHHHwEAV69excWLF9GmTZsafaekpOD48ePo06cPAODOnTuwtbXFkSNH4O/vL13r54UXXsBvv/3WgK/2ycDgbyT69euH/Px86HQ6bN++HTqdDsePH4eZmRmcnZ1RVlaGrl274vjx49i+fTvefvttBAUFYfTo0XBzc0NqauoDj2VmZib9ozUxMUFlZSUAoLq6GqmpqTA3N5flNVLj1KZNG9y8ebNGW0FBAVxcXADU/mL44/H9vjD27t2L3bt3IzU1Fc2bN4e/vz/KyspqrSeEQHh4ON5///0a7UlJSXq/iOh/OMffSJw/fx5VVVVo06YNioqKYGtrCzMzM+zZsweZmZkAgJycHDRv3hxhYWGIjo7GiRMn0K1bN+h0Oin4KyoqcPbs2TrVEBQUhE8++UR6/MevEVK3li1bws7ODikpKQDuhf6OHTvQv39/AJD2NR04cACWlpawtLTEwIEDsX79egBAcnKy9MVRVFQEa2trNG/eHOfPn8fhw4elcczMzFBRUQEACAgIwMaNG5GXlyeNmZmZib59+2Lv3r24ceMGKioq8N133ynzJjQy3OJ/jP2xMwu4t4Wzbt06mJiYIDQ0FCNGjIBGo4Gnpye6d+8OADh9+jTefPNNGBsbw8zMDJ9//jmaNGmCjRs3YsaMGSgqKkJlZSVmzpxZp8tkxMbGIioqCu7u7qisrMTAgQOxevXqhnzJ1Ej9+9//RlRUFP7xj38AAObPn4/OnTsDAKytreHr64vi4mL861//kpaHhITAy8sLfn5+6NixIwAgODgYq1evhru7O7p16wYfHx9pjMjISLi7u8PLywvr16/HkiVLEBQUhOrqapiZmeHTTz+Fj48PFixYgH79+sHOzg5eXl7STl/6H16ygYhk4+/vj2XLltX5fhokD071EBGpDLf4iYhUhlv8REQqw+AnIlIZBj8Rkcow+ImIVIbBT0SkMv8Hj7s1rDIYVJ8AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "# Calculate average time in seconds\n", + "baseline_avg_time = sum(baseline['cycles']) / len(baseline['cycles']) / 1000\n", + "updated_avg_time = sum(updated['cycles']) / len(updated['cycles']) / 1000\n", + "\n", + "# Calculate total size in MB\n", + "total_size_MB = baseline['totalSize'] / (1024 * 1024)\n", + "\n", + "# Calculate average speed in MB/s\n", + "baseline_speed = total_size_MB / baseline_avg_time\n", + "updated_speed = total_size_MB / updated_avg_time\n", + "\n", + "# Plot the bar chart\n", + "ax.bar(['Baseline', 'Updated'], [baseline_speed, updated_speed])\n", + "ax.set_ylabel('Tokenization Speed / MBs^-1')\n", + "fig.patch.set_facecolor('white')\n", + "ax.set_facecolor('white')\n", + "plt.title('Tokenization Speed')\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tokenizer_ts/src/bytePairEncode.ts b/tokenizer_ts/src/bytePairEncode.ts index d74acd6..fc0f353 100644 --- a/tokenizer_ts/src/bytePairEncode.ts +++ b/tokenizer_ts/src/bytePairEncode.ts @@ -1,17 +1,84 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -/** - * Convert a Uint8Array to a string - * @param uint8Array - * @returns string - */ -export function uint8ArrayToString(uint8Array: Uint8Array): string { - return Array.from(uint8Array) - .map(num => num.toString()) - .join("_"); +const enum Constant { + // we have 48 bits per level, we can safely bitwise encode 32 bits at a time, + // so this works in two passes + BytesPerLevel = 6, +} + +// exported for testing +export const binaryMapKey = (k: Uint8Array, start: number, end: number): number => { + const length = end - start; + + // 'lower' and 'upper' are both 24-bit integers, like + // 0xFF FF FF + // ^3 ^2 ^1 + // If we say have a length of 2, we should disregard the last "3" byte, so we + // create a mask like + // 0x00 FF FF (started at 0xFF FF FF and shifted over by 8 bits) + // ^3 ^2 ^1 + // so that we discard the data outside our range + const lowerMask = 0xFFFFFF >>> Math.max(0, (3 - length) * 8); + const lower = (k[start + 0] | (k[start + 1] << 8) | (k[start + 2] << 16)) & lowerMask; + + const upperMask = 0xFFFFFF >>> Math.min(31, Math.max(0, (6 - length) * 8)); + const upper = (k[start + 3] | (k[start + 4] << 8) | (k[start + 5] << 16)) & upperMask; + return lower + (0x1000000 * upper); +}; + +export class BinaryMap { + private readonly map: Map | V> = new Map(); + private thisValue?: V; + + public get(key: Uint8Array, start: number = 0, end: number = key.length): V | undefined { + const value = this.map.get(binaryMapKey(key, start, end)); + const isFinal = end < Constant.BytesPerLevel + start; + + if (isFinal) { + return value instanceof BinaryMap ? value.thisValue : value; + } else if (value instanceof BinaryMap) { + return value.get(key, Constant.BytesPerLevel + start, end); + } else { + return undefined; + } + } + + public set(key: Uint8Array, value: V): void { + const k = binaryMapKey(key, 0, key.length); + const existing = this.map.get(k); + const isFinal = key.length < Constant.BytesPerLevel; + + if (existing === undefined) { + if (isFinal) { + this.map.set(k, value); + } else { + const newMap = new BinaryMap(); + newMap.set(key.subarray(Constant.BytesPerLevel), value); + this.map.set(k, newMap); + } + } else if (isFinal) { + if (existing instanceof BinaryMap) { + existing.thisValue = value; + } else { + this.map.set(k, value); + } + } else { + if (existing instanceof BinaryMap) { + existing.set(key.subarray(Constant.BytesPerLevel), value); + } else { + const newMap = new BinaryMap(); + newMap.set(key.subarray(Constant.BytesPerLevel), value); + newMap.thisValue = existing; + this.map.set(k, newMap); + } + + } + } } +const maxRank = 0x7FFFFFFF; // max int32, try and keep things in integer space + /** * This function implements the byte pair encoding algorithm. * @param mergingBytes: bytes to be merged @@ -20,67 +87,68 @@ export function uint8ArrayToString(uint8Array: Uint8Array): string { */ export function bytePairEncode( mergingBytes: Uint8Array, - ranks: ReadonlyMap + ranks: BinaryMap, + length: number, ): number[] { - if (mergingBytes.length === 1) { - return [ranks.get(mergingBytes[0].toString())!]; + if (length === 1) { + return [ranks.get(mergingBytes)!]; } + let minRank = maxRank; + let minIndex = -1; + const byteIndicesAndRanks: [number, number][] = []; - for (let i = 0; i < mergingBytes.length + 1; i++) { - byteIndicesAndRanks.push([i, Number.MAX_SAFE_INTEGER]); + for (let i = 0; i < length - 1; i++) { + const rank = ranks.get(mergingBytes, i, i + 2) ?? maxRank; + if (rank < minRank) { + minRank = rank; + minIndex = i; + } + + byteIndicesAndRanks.push([i, rank]); } + byteIndicesAndRanks.push([length - 1, maxRank]); + byteIndicesAndRanks.push([length, maxRank]); function getRank(startIndex: number, skip = 0): number { if (startIndex + skip + 2 < byteIndicesAndRanks.length) { - const slice = mergingBytes.slice( + const rank = ranks.get( + mergingBytes, byteIndicesAndRanks[startIndex][0], byteIndicesAndRanks[startIndex + skip + 2][0] ); - const rank = ranks.get(uint8ArrayToString(slice)); if (rank !== undefined) { return rank; } } - return Number.MAX_SAFE_INTEGER; + return maxRank; } - for (let i = 0; i < byteIndicesAndRanks.length - 2; i++) { - const rank = getRank(i); - if (rank !== Number.MAX_SAFE_INTEGER) { - byteIndicesAndRanks[i][1] = rank; + while (minRank !== maxRank) { + byteIndicesAndRanks[minIndex][1] = getRank(minIndex, 1); + if (minIndex > 0) { + byteIndicesAndRanks[minIndex - 1][1] = getRank(minIndex - 1, 1); } - } + byteIndicesAndRanks.splice(minIndex + 1, 1); + - while (byteIndicesAndRanks.length > 1) { - let minRank: [number, number] = [0, Number.MAX_SAFE_INTEGER]; + minIndex = -1; + minRank = maxRank; for (let i = 0; i < byteIndicesAndRanks.length - 1; i++) { - if (byteIndicesAndRanks[i][1] < minRank[1]) { - minRank = [i, byteIndicesAndRanks[i][1]]; + if (byteIndicesAndRanks[i][1] < minRank) { + minRank = byteIndicesAndRanks[i][1]; + minIndex = i; } } - if (minRank[1] !== Number.MAX_SAFE_INTEGER) { - const j = minRank[0]; - byteIndicesAndRanks[j][1] = getRank(j, 1); - if (j > 0) { - byteIndicesAndRanks[j - 1][1] = getRank(j - 1, 1); - } - byteIndicesAndRanks.splice(j + 1, 1); - } else { - break; - } } const outList: number[] = []; for (let i = 0; i < byteIndicesAndRanks.length - 1; i++) { outList.push( ranks.get( - uint8ArrayToString( - mergingBytes.slice( - byteIndicesAndRanks[i][0], - byteIndicesAndRanks[i + 1][0] - ) - ) + mergingBytes, + byteIndicesAndRanks[i][0], + byteIndicesAndRanks[i + 1][0] )! ); } diff --git a/tokenizer_ts/src/textEncoder.ts b/tokenizer_ts/src/textEncoder.ts new file mode 100644 index 0000000..a5393cd --- /dev/null +++ b/tokenizer_ts/src/textEncoder.ts @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * A text encoder interface. + */ +export interface ITextEncoder { + /** + * Number of bytes written in the last call to {@link encode} + */ + length: number; + + /** + * Encodes the text and returns the Uint8Array it was written to. The length + * of data written to the array can be found in {@link length}. + * + * The data returned in the array is only valid until the next call to encode. + */ + encode(text: string): Uint8Array; +} + +class UniversalTextEncoder implements ITextEncoder { + public length = 0; + private encoder = new TextEncoder(); + + public encode(text: string): Uint8Array { + const arr = this.encoder.encode(text); + this.length = arr.length; + return arr; + } +} + +class NodeTextEncoder implements ITextEncoder { + private buffer = Buffer.alloc(256); + public length = 0; + + public encode(text: string): Uint8Array { + while (true) { + this.length = this.buffer.write(text, 'utf8'); + + // buffer.write returns the number of bytes written and can write less + // than the length of the string if the buffer is too small. If this + // might have happened (4 bytes is the longest utf8 codepoint), make + // the buffer bigger and try again. + if (this.length < this.buffer.length - 4) { + return this.buffer; + } + + this.buffer = Buffer.alloc(this.length * 2); + this.length = this.buffer.write(text); + } + } +} + +export const makeTextEncoder = (): ITextEncoder => + typeof Buffer !== 'undefined' ? new NodeTextEncoder() : new UniversalTextEncoder(); diff --git a/tokenizer_ts/src/tikTokenizer.ts b/tokenizer_ts/src/tikTokenizer.ts index 88e6183..15eea14 100644 --- a/tokenizer_ts/src/tikTokenizer.ts +++ b/tokenizer_ts/src/tikTokenizer.ts @@ -3,8 +3,9 @@ import * as fs from "fs"; import { LRUCache } from "lru-cache"; -import { TextDecoder, TextEncoder } from "util"; -import { bytePairEncode, uint8ArrayToString } from "./bytePairEncode"; +import { TextDecoder } from "util"; +import { BinaryMap, bytePairEncode } from "./bytePairEncode"; +import { makeTextEncoder } from './textEncoder'; /** * Load BPE ranks from a file @@ -59,12 +60,12 @@ function escapeRegExp(regex: string) { */ export class TikTokenizer { private regex?: RegExp; - private encoder?: Map; + private encoder?: BinaryMap; private decoder?: Map; private specialTokensRegex?: RegExp; private specialTokensEncoder?: ReadonlyMap; private specialTokensDecoder?: Map; - private textEncoder = new TextEncoder(); + private textEncoder = makeTextEncoder(); private textDecoder = new TextDecoder("utf-8"); public readonly cache: LRUCache; @@ -94,9 +95,9 @@ export class TikTokenizer { specialTokensEncoder: ReadonlyMap, regexPattern: string ): void { - this.encoder = new Map(); + this.encoder = new BinaryMap(); for (const [key, value] of bpeDict) { - this.encoder.set(uint8ArrayToString(key), value); + this.encoder.set(key, value); } this.regex = new RegExp(regexPattern, "gu"); this.specialTokensRegex = new RegExp( @@ -111,7 +112,7 @@ export class TikTokenizer { this.decoder.set(value, key); } - if (this.encoder.size !== this.decoder.size) { + if (bpeDict.size !== this.decoder.size) { throw new Error("Encoder and decoder sizes do not match"); } @@ -200,17 +201,18 @@ export class TikTokenizer { const substring = text.substring(start, end); this.regex!.lastIndex = 0; while ((match = this.regex!.exec(substring))) { - if (this.cache.has(match[0])) { - tokenIds.push(...this.cache.get(match[0])!); + const cached = this.cache.get(match[0]); + if (cached) { + tokenIds.push(...cached); } else { // cache miss const bytes = this.textEncoder.encode(match[0]); - const token = this.encoder?.get(uint8ArrayToString(bytes)); + const token = this.encoder?.get(bytes, 0, this.textEncoder.length); if (token !== undefined) { tokenIds.push(token); this.cache.set(match[0], [token]); } else { - const encodedTokens = bytePairEncode(bytes, this.encoder!); + const encodedTokens = bytePairEncode(bytes, this.encoder!, this.textEncoder.length); tokenIds.push(...encodedTokens); this.cache.set(match[0], encodedTokens); } @@ -248,7 +250,7 @@ export class TikTokenizer { } else { // cache miss const bytes = this.textEncoder.encode(piece); - const token = this.encoder!.get(uint8ArrayToString(bytes)); + const token = this.encoder!.get(bytes, 0, bytes.length); if (token !== undefined) { this.cache.set(piece, [token]); if (tokenCount + 1 <= maxTokenCount) { @@ -259,7 +261,7 @@ export class TikTokenizer { break; } } else { - const encodedTokens = bytePairEncode(bytes, this.encoder!); + const encodedTokens = bytePairEncode(bytes, this.encoder!, this.textEncoder.length); this.cache.set(piece, encodedTokens); if (tokenCount + encodedTokens.length <= maxTokenCount) { tokenCount += encodedTokens.length; @@ -394,8 +396,8 @@ export class TikTokenizer { tokenIds.push(...tokens!); tokenCountMap.set(tokenCount, encodeLength); } else { - const bytes = new TextEncoder().encode(piece); - const token = this.encoder!.get(uint8ArrayToString(bytes)); + const bytes = this.textEncoder.encode(piece); + const token = this.encoder!.get(bytes); if (token !== undefined) { this.cache.set(piece, [token]); tokenCount++; @@ -403,7 +405,7 @@ export class TikTokenizer { tokenIds.push(token); tokenCountMap.set(tokenCount, encodeLength); } else { - const encodedTokens = bytePairEncode(bytes, this.encoder!); + const encodedTokens = bytePairEncode(bytes, this.encoder!, this.textEncoder.length); this.cache.set(piece, encodedTokens); tokenCount += encodedTokens.length; encodeLength += piece.length; @@ -473,7 +475,8 @@ export class TikTokenizer { } else { const specialTokenValue = this.specialTokensDecoder?.get(token); if (specialTokenValue !== undefined) { - tokenBytes = Array.from(this.textEncoder.encode(specialTokenValue)); + const bytes = this.textEncoder.encode(specialTokenValue); + tokenBytes = Array.from(bytes.subarray(0, this.textEncoder.length)); } } decoded.push(...tokenBytes); diff --git a/tokenizer_ts/test/binaryMap.test.ts b/tokenizer_ts/test/binaryMap.test.ts new file mode 100644 index 0000000..6d6d3e7 --- /dev/null +++ b/tokenizer_ts/test/binaryMap.test.ts @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import * as assert from "assert"; +import { BinaryMap, binaryMapKey } from "../src/bytePairEncode"; +suite("BinaryMap Test Suite", function () { + test("Test basic input to map - one level", done => { + const binMap: BinaryMap = new BinaryMap(); + binMap.set(new Uint8Array([1, 50, 24]), 1); + assert(binMap.get(new Uint8Array([1, 50, 24])) === 1); + assert(binMap.get(new Uint8Array([1, 50])) === undefined); + assert(binMap.get(new Uint8Array([1, 50, 24, 100])) === undefined); + + binMap.set(new Uint8Array([1, 50, 24, 100]), 100); + assert(binMap.get(new Uint8Array([1, 50, 24, 100])) === 100); + done(); + }); + test("Test basic input to map - one or two levels", done => { + const binMap: BinaryMap = new BinaryMap(); + binMap.set(new Uint8Array([1, 50, 24, 34, 64, 23]), 1); + binMap.set(new Uint8Array([1, 50, 24, 34, 64, 23, 60, 120, 40]), 2); + binMap.set(new Uint8Array([1, 50, 24, 34, 64, 23, 60, 120, 40, 21, 54, 232]), 3); + assert(binMap.get(new Uint8Array([1, 50, 24, 34, 64, 23])) === 1); + assert(binMap.get(new Uint8Array([1, 50, 24, 34, 64, 23, 60, 120, 40])) === 2); + assert(binMap.get(new Uint8Array([1, 50, 24, 34, 64, 23, 60, 120, 40, 21, 54, 232])) === 3); + done(); + }); + test("Test `get` with start and end specified", done => { + const binMap: BinaryMap = new BinaryMap(); + binMap.set(new Uint8Array([64, 23]), 100); + binMap.set(new Uint8Array([1, 50, 24]), 1); + binMap.set(new Uint8Array([24, 34, 64]), 2); + binMap.set(new Uint8Array([23, 60, 120, 1, 50, 24]), 255); + const mainArray = new Uint8Array([64, 23, 60, 120, 1, 50, 24, 34, 64]); + assert(binMap.get(mainArray, 4, 7) === 1); + assert(binMap.get(mainArray, 6, 9) === 2); + assert(binMap.get(mainArray, 1, 7) === 255); + assert(binMap.get(mainArray, 7, 7) === undefined); + assert(binMap.get(mainArray, 6, 10) === 2); + assert(binMap.get(mainArray, 0, 2) === 100); + done(); + }); +}); +suite("Binary Map Key Function Test", function () { + test("First 3 Max Bytes", done => { + const arr = new Uint8Array([0xFF, 0xFF, 0xFF, 0xAB, 0xCD, 0xEF]); + const result = binaryMapKey(arr, 0, arr.length); + assert.strictEqual(result, 0xEFCDABFFFFFF); + done(); + }); + + test("All 6 Max Bytes", done => { + const arr = new Uint8Array([0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + const result = binaryMapKey(arr, 0, arr.length); + assert.strictEqual(result, 0xFFFFFFFFFFFF); + done(); + }); + + test("First 3 Min Bytes", done => { + const arr = new Uint8Array([0x00, 0x00, 0x00, 0xAB, 0xCD, 0xEF]); + const result = binaryMapKey(arr, 0, arr.length); + assert.strictEqual(result, 0xEFCDAB000000); + done(); + }); + + test("Last 3 Min Bytes", done => { + const arr = new Uint8Array([0xAB, 0xCD, 0xEF, 0x00, 0x00, 0x00]); + const result = binaryMapKey(arr, 0, arr.length); + assert.strictEqual(result, 0x000000EFCDAB); + done(); + }); + + test("Assorted Bytes", done => { + const arr = new Uint8Array([0xBA, 0xDC, 0xFE, 0xEF, 0xCD, 0xAB]); + const result = binaryMapKey(arr, 0, arr.length); + assert.strictEqual(result, 0xABCDEFFEDCBA); + done(); + }); + + test("Assorted Bytes with start/end defined in lower bits", done => { + const arr = new Uint8Array([0xBA, 0xDC, 0xFE, 0xEF, 0xCD, 0xAB]); + const result = binaryMapKey(arr, 1, 3); + assert.strictEqual(result, 0x00000000FEDC); + done(); + }); + + test("Assorted Bytes with start/end defined in upper bits", done => { + const arr = new Uint8Array([0xBA, 0xDC, 0xFE, 0xEF, 0xCD, 0xAB]); + const result = binaryMapKey(arr, 3, 6); + assert.strictEqual(result, 0x000000ABCDEF); + done(); + }); + + test("Assorted Bytes with start/end defined across upper and lower bits", done => { + const arr = new Uint8Array([0xBA, 0xDC, 0xFE, 0xEF, 0xCD, 0xAB]); + const result = binaryMapKey(arr, 2, 5); + assert.strictEqual(result, 0x000000CDEFFE); + done(); + }); +});