From 9cad244097660a1e376f0bf97f959db861044a11 Mon Sep 17 00:00:00 2001 From: Shengyu Fu Date: Tue, 21 May 2024 09:05:53 -0700 Subject: [PATCH] Add support for gpt-4o (#46) * Add support for gpt-4o * Make case insensitive matching because javascript doesn't support (?i: ...) inline modifier for case insensitivity --- tokenizer_ts/package.json | 2 +- tokenizer_ts/src/tokenizerBuilder.ts | 33 +++++++++++++++- tokenizer_ts/test/testdata/tokens_gpt_4o.json | 1 + tokenizer_ts/test/tikTokenizerGpt4o.test.ts | 39 +++++++++++++++++++ 4 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 tokenizer_ts/test/testdata/tokens_gpt_4o.json create mode 100644 tokenizer_ts/test/tikTokenizerGpt4o.test.ts diff --git a/tokenizer_ts/package.json b/tokenizer_ts/package.json index 76e248b..310d6fe 100644 --- a/tokenizer_ts/package.json +++ b/tokenizer_ts/package.json @@ -2,7 +2,7 @@ "name": "@microsoft/tiktokenizer", "displayName": "tiktokenizer", "description": "Tokenizer for OpenAI large language models.", - "version": "1.0.6", + "version": "1.0.7", "author": { "name": "Microsoft Corporation" }, diff --git a/tokenizer_ts/src/tokenizerBuilder.ts b/tokenizer_ts/src/tokenizerBuilder.ts index 41719ae..42cb4be 100644 --- a/tokenizer_ts/src/tokenizerBuilder.ts +++ b/tokenizer_ts/src/tokenizerBuilder.ts @@ -7,12 +7,15 @@ import { TikTokenizer } from "./tikTokenizer"; const MODEL_PREFIX_TO_ENCODING: ReadonlyMap = new Map([ // chat + ["gpt-4o-", "o200k_base"], // e.g., gpt-4o-2024-05-13 ["gpt-4-", "cl100k_base"], // e.g., gpt-4-0314, etc., plus gpt-4-32k - ["gpt-3.5-turbo-", "cl100k_base"] // e.g, gpt-3.5-turbo-0301, -0401, etc. + ["gpt-3.5-turbo-", "cl100k_base"], // e.g, gpt-3.5-turbo-0301, -0401, etc. + ["gpt-35-turbo-", "cl100k_base"] // Azure deployment name ]); export const MODEL_TO_ENCODING: ReadonlyMap = new Map([ // chat + ["gpt-4o", "o200k_base"], ["gpt-4", "cl100k_base"], ["gpt-3.5-turbo", "cl100k_base"], // text @@ -71,6 +74,22 @@ const REGEX_PATTERN_1: string = const REGEX_PATTERN_2: string = "(?:'s|'S|'t|'T|'re|'RE|'Re|'eR|'ve|'VE|'vE|'Ve|'m|'M|'ll|'lL|'Ll|'LL|'d|'D)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"; + +/* + * regex pattern used for gpt-4o + */ +const patterns: string[] = [ + `[^\r\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:'s|'S|'t|'T|'re|'RE|'Re|'eR|'ve|'VE|'vE|'Ve|'m|'M|'ll|'lL|'Ll|'LL|'d|'D)?`, + `[^\r\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:'s|'S|'t|'T|'re|'RE|'Re|'eR|'ve|'VE|'vE|'Ve|'m|'M|'ll|'lL|'Ll|'LL|'d|'D)?`, + `\\p{N}{1,3}`, + ` ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*`, + `\\s*[\\r\\n]+`, + `\\s+(?!\\S)`, + `\\s+`, +]; + +const REGEX_PATTERN_3: string = patterns.join("|"); + function getEncoderFromModelName(modelName: string): string { let encoder = ""; if (!MODEL_TO_ENCODING.has(modelName)) { @@ -112,6 +131,12 @@ export function getSpecialTokensByEncoder( ): Map { let specialTokens: Map = new Map([[ENDOFTEXT, 50256]]); switch (encoder) { + case "o200k_base": + specialTokens = new Map([ + [ENDOFTEXT, 199999], + [ENDOFPROMPT, 200018] + ]); + break; case "cl100k_base": specialTokens = new Map([ [ENDOFTEXT, 100257], @@ -157,6 +182,8 @@ export function getSpecialTokensByModel( */ export function getRegexByEncoder(encoder: string): string { switch (encoder) { + case "o200k_base": + return REGEX_PATTERN_3; case "cl100k_base": return REGEX_PATTERN_2; default: @@ -208,6 +235,10 @@ export async function createByEncoderName( ); switch (encoderName) { + case "o200k_base": + regexPattern = REGEX_PATTERN_3; + mergeableRanksFileUrl = `https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken`; + break; case "cl100k_base": regexPattern = REGEX_PATTERN_2; mergeableRanksFileUrl = `https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken`; diff --git a/tokenizer_ts/test/testdata/tokens_gpt_4o.json b/tokenizer_ts/test/testdata/tokens_gpt_4o.json new file mode 100644 index 0000000..821761f --- /dev/null +++ b/tokenizer_ts/test/testdata/tokens_gpt_4o.json @@ -0,0 +1 @@ +[393, 1328, 2371, 382, 620, 326, 7890, 140855, 350, 143660, 483, 15993, 46, 18, 22391, 446, 18666, 58, 10180, 16811, 69229, 742, 136606, 1565, 88744, 17663, 56882, 1821, 3081, 742, 83306, 742, 8579, 1984, 307, 1821, 3081, 742, 9817, 502, 1821, 39357, 109210, 742, 37057, 307, 1821, 275, 4925, 18, 742, 196013, 307, 1821, 275, 4925, 18, 742, 2235, 52786, 128282, 1821, 275, 4925, 18, 742, 10415, 45255, 37863, 10629, 11, 15993, 950, 11, 15993, 40048, 4516, 1821, 275, 4925, 18, 742, 37863, 2769, 307, 1821, 20294, 66, 20844, 742, 91612, 8579, 2307, 472, 10242, 2307, 502, 13682, 1175, 7216, 34819, 50662, 5833, 34969, 271, 9047, 25, 96391, 84, 23, 2274, 271, 40642, 25, 827, 8579, 2307, 115716, 76960, 23, 17084, 46737, 23844, 271, 285, 25, 15443, 105032, 14847, 742, 4645, 742, 8614, 27, 112511, 17991, 2747, 353, 412, 8, 2747, 22915, 5833, 29, 405, 271, 602, 1328, 382, 261, 9727, 328, 350, 5236, 11, 10302, 6294, 271, 602, 623, 10302, 382, 328, 290, 9239, 10610, 8601, 540, 4010, 1604, 558, 271, 602, 623, 10302, 328, 290, 2174, 2169, 306, 290, 9727, 382, 625, 261, 4529, 1432, 558, 271, 1632, 6505, 7881, 25, 22915, 66307, 112511, 11, 46737, 37292, 314, 350, 15, 485, 30871, 41284, 416, 659, 220, 16, 741, 3726, 44441, 72, 91, 350, 72, 11, 46737, 742, 21911, 8784, 25670, 2457, 271, 1632, 717, 52820, 314, 405, 309, 22063, 10718, 80293, 11587, 309, 1022, 34761, 25, 827, 18136, 66307, 112511, 11, 46737, 8, 17084, 1604, 15257, 25, 46737, 11, 16677, 25, 46737, 91, 405, 352, 538, 350, 5236, 15257, 659, 16677, 659, 220, 17, 8, 464, 7881, 41284, 416, 405, 506, 40642, 198, 699, 887, 522, 4366, 30871, 58, 34761, 62179, 15257, 1651, 15, 485, 34761, 62179, 15257, 659, 16677, 659, 220, 17, 1651, 15, 5169, 699, 887, 47750, 1165, 1234, 352, 388, 1203, 405, 506, 4662, 198, 352, 606, 309, 606, 271, 6755, 271, 602, 1416, 1631, 869, 290, 40642, 4730, 306, 290, 10526, 326, 10621, 11594, 4027, 198, 271, 602, 1373, 3354, 2454, 20345, 11, 1118, 38566, 290, 2086, 328, 10302, 1631, 14409, 558, 271, 395, 575, 306, 220, 15, 485, 34761, 41284, 416, 533, 220, 17, 405, 309, 3981, 717, 52820, 4366, 34761, 11, 575, 11, 220, 15, 8, 405, 352, 6083, 157799, 8, 871, 405, 506, 602, 46737, 742, 21911, 382, 261, 180334, 1432, 326, 6284, 413, 261, 4529, 10302, 198, 506, 15199, 40989, 20836, 30162, 1666, 46737, 742, 21911, 362, 506, 7881, 1768, 1651, 16, 314, 10302, 307, 352, 606, 352, 4662, 871, 405, 506, 4901, 307, 352, 606, 309, 4753, 271, 758, 271, 602, 1843, 481, 679, 297, 7881, 326, 284, 176901, 11, 495, 2226, 532, 2145, 77, 8, 1101, 558, 271, 602, 1416, 2023, 621, 3543, 483, 261, 37237, 326, 621, 532, 2145, 2142, 297, 8, 1101, 558, 271, 602, 1225, 382, 3378, 316, 3331, 484, 297, 382, 4783, 3291, 46465, 1353, 936, 326, 472, 2238, 198, 271, 602, 290, 11956, 89928, 536, 8808, 142335, 290, 22184, 291, 37588, 62461, 4105, 198, 271, 602, 328, 290, 2700, 34761, 63, 9727, 1238, 8866, 5151, 364, 271, 602, 12225, 484, 581, 10691, 11643, 11, 625, 6602, 24702, 13, 1877, 1701, 472, 581, 8513, 418, 3111, 290, 2006, 581, 198, 271, 602, 6821, 621, 11, 495, 382, 23458, 13, 1689, 4224, 2006, 316, 2338, 495, 1481, 413, 316, 1885, 166219, 198, 271, 602, 20345, 17621, 591, 6602, 3472, 503, 316, 7332, 4857, 6602, 176901, 558, 271, 11453, 405, 309, 538, 7881, 41284, 416, 951, 220, 16, 405, 352, 2338, 307, 309, 758, 309, 602, 46737, 742, 21911, 382, 261, 180334, 10302, 1432, 16246, 765, 316, 198, 309, 602, 2304, 290, 1349, 945, 8065, 198, 309, 1632, 6505, 1349, 52820, 25, 350, 112511, 11, 46737, 8, 314, 350, 112511, 742, 21911, 11, 220, 15, 362, 309, 395, 350, 72, 11, 827, 96396, 10302, 915, 306, 7881, 165124, 34761, 41284, 416, 533, 220, 16, 1651, 2340, 1454, 90985, 379, 416, 405, 352, 538, 10302, 464, 1349, 52820, 13, 15, 405, 506, 1349, 52820, 314, 350, 30162, 11, 575, 362, 352, 606, 309, 758, 309, 538, 1349, 52820, 13, 15, 1666, 46737, 742, 21911, 405, 352, 1632, 575, 314, 1349, 52820, 13, 16, 502, 352, 602, 32269, 25, 1416, 553, 1078, 316, 6234, 7881, 1768, 659, 220, 16, 1651, 1416, 621, 625, 621, 480, 198, 352, 602, 5073, 2236, 1354, 553, 11956, 89928, 536, 8808, 316, 34543, 198, 352, 602, 7881, 1768, 60, 326, 7881, 1768, 12, 16, 60, 2254, 28559, 11, 1118, 2023, 11622, 1229, 198, 352, 602, 290, 11956, 13, 23586, 11, 581, 4027, 290, 10302, 40722, 656, 89472, 1072, 198, 352, 602, 7881, 1768, 659, 220, 16, 2155, 656, 106967, 2700, 522, 52820, 0, 63, 483, 2700, 38258, 314, 220, 16, 36060, 352, 7881, 1768, 1651, 16, 314, 717, 52820, 4366, 34761, 11, 575, 11, 220, 16, 741, 29969, 15400, 7, 112511, 742, 21911, 362, 352, 538, 575, 1424, 220, 15, 405, 506, 7881, 1768, 533, 220, 16, 1651, 16, 314, 717, 52820, 4366, 34761, 11, 575, 533, 220, 16, 11, 220, 16, 741, 29969, 15400, 7, 112511, 742, 21911, 362, 352, 758, 352, 7881, 7986, 3649, 659, 220, 16, 362, 309, 388, 1203, 405, 352, 2338, 307, 309, 606, 271, 606, 271, 1632, 6505, 842, 25, 22915, 5833, 29, 314, 22915, 742, 6332, 85527, 102573, 41284, 416, 533, 220, 16, 362, 271, 395, 575, 306, 220, 15, 485, 34761, 41284, 416, 533, 220, 16, 405, 309, 842, 4885, 1526, 102573, 1768, 1651, 15, 485, 34761, 1768, 659, 220, 16, 1651, 15, 1820, 271, 606, 271, 842, 198, 943, 15987, 9784, 9239, 34819, 27276, 172522, 25, 96391, 84, 23, 2155, 40642, 25, 827, 8579, 2307, 115716, 76960, 23, 17084, 46737, 17991, 2747, 22915, 27, 112511, 29, 405, 271, 538, 9047, 41284, 416, 951, 220, 16, 405, 309, 622, 10563, 46759, 81, 4096, 23234, 12778, 32098, 271, 606, 271, 1175, 7216, 34819, 50662, 172522, 11, 40642, 11, 1022, 79, 91, 40642, 58, 5, 30871, 23234, 10181, 485, 79, 10289, 62818, 943, 15987, 9784, 9239, 34819, 39921, 36104, 64, 4513, 30871, 25, 69980, 64, 723, 84, 23, 2155, 40642, 25, 827, 8579, 2307, 115716, 76960, 23, 17084, 46737, 17991, 2747, 22915, 114986, 6, 64, 723, 84, 23, 54651, 405, 271, 538, 9047, 41284, 416, 951, 220, 16, 405, 309, 622, 10563, 46759, 30871, 1735, 271, 606, 271, 1175, 7216, 34819, 50662, 172522, 11, 40642, 11, 1022, 79, 91, 827, 30871, 23234, 10181, 485, 79, 10289, 5169, 8123, 62309, 6198, 12870, 76113, 59319, 198, 393, 71502, 3914, 8887, 328, 290, 1058, 382, 11228, 306, 42588, 13, 623, 39356, 2006, 316, 7733, 495, 869, 382, 656, 2360, 3760, 39357, 198, 393, 42588, 4877, 13, 2214, 4934, 11, 2360, 261, 42588, 8420, 199854, 656, 2700, 63920, 63, 34715, 382, 220, 18, 87, 16107, 1572, 198, 393, 290, 19261, 42588, 581, 1199, 27464, 5551, 11, 4335, 484, 15929, 2360, 261, 42588, 8420, 199854, 656, 2700, 63920, 15007, 1354, 12471, 2009, 9809, 198, 393, 2870, 2360, 290, 2700, 63920, 63, 34715, 326, 2360, 290, 2700, 69, 8830, 109210, 63, 34715, 27464, 3274, 382, 448, 3378, 22391, 2870, 72047, 11, 2700, 63920, 63, 326, 2700, 69, 8830, 109210, 63, 7621, 4296, 2360, 2700, 69, 8830, 109210, 15007, 581, 7103, 2700, 63920, 4231, 8254, 62102, 1225, 18304, 842, 484, 495, 16431, 115778, 402, 198, 393, 1236, 54987, 29133, 4918, 6772, 328, 2700, 63920, 62102, 1328, 13907, 62013, 6198, 13, 4296, 2360, 21402, 198, 393, 2890, 2700, 63920, 15007, 581, 4128, 7103, 495, 11, 2236, 2700, 6555, 27043, 63, 853, 261, 2647, 3490, 3104, 7621, 43352, 25, 5918, 1684, 10312, 1136, 17447, 570, 189718, 14, 63920, 77323, 53530, 27120, 866, 125409, 44190, 198, 393, 54669, 11, 290, 2006, 581, 717, 2846, 495, 382, 483, 4566, 261, 350, 123451, 8, 8591, 2698, 28806, 328, 290, 42588, 395, 198, 393, 2454, 8591, 27464, 14614, 289, 198, 393, 182420, 3914, 357, 10471, 2360, 2700, 25430, 263, 62102, 1225, 18101, 2715, 16107, 1572, 2360, 26534, 29249, 326, 52057, 290, 499, 2654, 7621, 2632, 72185, 2700, 25430, 263, 63, 0, 9024, 8591, 3605, 5178, 413, 306, 3580, 328, 1039, 26534, 5385, 27464, 363, 15893, 198, 393, 71502, 560, 3914, 623, 9682, 6602, 7466, 853, 448, 305, 1942, 11956, 1072, 290, 23458, 328, 2700, 7216, 34819, 27276, 63, 7621, 78369, 11, 581, 1458, 1001, 3101, 0, 25767, 480, 11, 581, 1504, 1606, 163205, 16107, 1572, 26534, 7621, 357, 2061, 448, 92511, 12940, 316, 3319, 290, 11956, 13, 1328, 9289, 4265, 316, 20206, 4590, 124797, 6198, 198, 393, 160334, 11, 889, 480, 2242, 7902, 12151, 156104, 295, 6198, 13, 153616, 423, 11, 480, 16592, 316, 7902, 198, 393, 12151, 156104, 295, 6198, 1952, 1261, 357, 1606, 1458, 15116, 350, 21065, 7594, 357, 116060, 3543, 869, 30, 6294, 393, 54669, 11, 357, 68972, 484, 581, 2023, 717, 12566, 328, 290, 11956, 11, 538, 581, 4665, 290, 920, 328, 20290, 472, 261, 11956, 127868, 5006, 553, 9707, 290, 920, 503, 176901, 484, 553, 6960, 316, 413, 3648, 13, 1958, 1954, 581, 4128, 679, 316, 2411, 198, 393, 1078, 13878, 6505, 3563, 11, 8197, 1199, 11, 503, 158581, 27464, 10242, 289, 198, 393, 71502, 560, 3914, 1416, 1199, 149276, 8579, 2307, 7665, 328, 290, 7024, 10242, 2307, 13, 1328, 382, 10112, 1299, 261, 220, 20, 12, 702, 4, 4449, 3901, 393, 623, 2208, 13847, 17095, 869, 5306, 261, 3261, 328, 185245, 328, 11643, 13, 730, 17346, 11, 495, 2023, 413, 2452, 198, 393, 316, 413, 185245, 328, 1920, 96827, 4524, 328, 121629, 11, 1118, 8224, 1299, 480, 1340, 1217, 413, 261, 7167, 6368, 16107, 364, 1821, 3081, 742, 4571, 742, 9953, 20870, 52, 2220, 307, 15987, 3539, 52190, 6482, 906, 7, 9953, 20870, 52, 2220, 861, 13682, 10691, 23981, 25733, 416, 2747, 46737, 405, 271, 602, 7744, 11087, 316, 1199, 37750, 1572, 316, 1199, 97656, 13, 56665, 853, 495, 7403, 337, 2220, 8591, 1211, 9446, 198, 271, 602, 484, 5882, 2212, 395, 1039, 1199, 1890, 328, 49354, 100572, 306, 1039, 2644, 13, 27864, 412, 271, 602, 4275, 1249, 13, 5551, 11, 1354, 553, 1606, 813, 1991, 6984, 481, 665, 10557, 261, 337, 2220, 11, 813, 1327, 1643, 82723, 198, 271, 602, 5918, 1684, 10312, 1136, 17447, 570, 189718, 17447, 570, 83178, 14, 47339, 3255, 198, 271, 1026, 133568, 723, 84, 23, 26, 220, 23, 60, 314, 723, 15, 26, 3081, 742, 17112, 742, 4461, 8023, 51996, 4946, 742, 9817, 742, 6482, 906, 23177, 1735, 271, 1026, 133568, 723, 84, 23, 26, 220, 23, 60, 314, 723, 15, 26, 3081, 742, 17112, 742, 4461, 8023, 51996, 57947, 6482, 906, 23177, 1735, 271, 1632, 1215, 314, 37750, 405, 309, 3081, 742, 17112, 742, 2623, 82723, 51996, 4946, 742, 9817, 742, 6482, 906, 11, 52190, 6482, 906, 4513, 9817, 742, 5804, 1454, 315, 9503, 15, 198, 271, 4753, 271, 337, 2220, 742, 2845, 4061, 8, 472, 46737, 198, 943, 1671, 16960, 20468, 142185, 25, 46737, 314, 220, 8076, 307, 24654, 5823, 1444, 1592, 2204, 16309, 33, 3111, 405, 271, 49416, 25, 10242, 2307, 115716, 76960, 23, 17084, 46737, 23844, 271, 3582, 65830, 91440, 25, 10242, 2307, 4920, 11, 46737, 23844, 271, 53790, 25, 10242, 2307, 27, 112511, 11, 22915, 76960, 23, 129393, 271, 3582, 65830, 127170, 25, 10242, 2307, 27, 112511, 11, 22915, 76960, 23, 129393, 271, 42588, 183618, 25, 22915, 27, 37057, 23844, 271, 3582, 109210, 183618, 25, 22915, 27, 37057, 23844, 271, 20099, 13535, 28036, 25, 22915, 115716, 76960, 23, 129393, 943, 10766, 16309, 33, 3111, 405, 271, 9784, 1175, 522, 838, 75, 109210, 4366, 1156, 8, 2747, 827, 37057, 405, 309, 602, 5601, 6198, 12870, 5151, 395, 1412, 495, 382, 1078, 198, 309, 602, 7744, 1217, 261, 3389, 441, 171500, 11, 4843, 1520, 261, 3432, 3926, 328, 480, 4175, 309, 602, 5551, 11, 4275, 7403, 484, 495, 8740, 38654, 8197, 316, 4022, 126877, 29249, 198, 309, 827, 1156, 73970, 183618, 58, 16909, 23981, 25733, 416, 1851, 16960, 20468, 142185, 1592, 271, 758, 271, 9784, 1175, 522, 838, 75, 100315, 109210, 4366, 1156, 8, 2747, 827, 37057, 405, 309, 827, 1156, 118760, 109210, 183618, 58, 16909, 23981, 25733, 416, 1851, 16960, 20468, 142185, 1592, 271, 758, 271, 9784, 1175, 44196, 106196, 4366, 1156, 11, 20290, 25, 96391, 112511, 4636, 2747, 22915, 76960, 23, 29, 405, 309, 1632, 6505, 2881, 314, 22915, 742, 6332, 85527, 74272, 41284, 416, 425, 220, 17, 362, 309, 395, 6602, 306, 20290, 405, 352, 1632, 6602, 28036, 314, 1051, 198, 506, 887, 116598, 198, 506, 887, 522, 23460, 446, 506, 887, 29969, 15400, 148349, 173315, 827, 1156, 118760, 65830, 127170, 138013, 4640, 352, 2881, 34151, 23460, 28036, 362, 309, 606, 309, 2881, 198, 271, 758, 271, 9784, 1175, 12949, 62, 110193, 106196, 4366, 1156, 11, 2201, 25, 827, 689, 8, 2747, 22915, 27, 112511, 29, 405, 309, 602, 1328, 382, 290, 10089, 328, 290, 24072, 18485, 26, 290, 1273, 9964, 306, 2105, 198, 309, 602, 1327, 1520, 3283, 25334, 181750, 309, 1632, 42588, 314, 1051, 2697, 522, 838, 75, 109210, 740, 309, 1632, 6505, 2881, 314, 10563, 0, 31374, 309, 395, 2450, 306, 42588, 4231, 27043, 13414, 8, 405, 352, 1632, 9047, 314, 2450, 107220, 1454, 288, 6182, 1454, 288, 28036, 740, 352, 538, 1632, 6083, 23460, 8, 314, 1051, 151105, 775, 172522, 8, 405, 506, 2881, 4885, 9000, 10346, 362, 506, 4901, 307, 352, 606, 352, 2881, 34151, 4366, 7216, 34819, 27276, 172522, 11, 827, 1156, 151105, 1820, 309, 606, 309, 2881, 198, 271, 758, 271, 9784, 1175, 12949, 106196, 4366, 1156, 11, 2201, 25, 827, 689, 11, 9279, 100315, 25, 827, 8579, 1984, 114986, 689, 17991, 2747, 350, 18136, 27, 112511, 17084, 46737, 8, 405, 309, 1632, 3582, 109210, 314, 1051, 2697, 522, 838, 75, 100315, 109210, 740, 309, 1632, 42588, 314, 1051, 2697, 522, 838, 75, 109210, 740, 309, 1632, 6505, 2881, 314, 10563, 0, 99758, 309, 1632, 6505, 1604, 314, 220, 15, 307, 309, 1632, 6505, 2174, 126000, 13535, 13683, 314, 220, 15, 307, 309, 11453, 405, 352, 1632, 6505, 2613, 100315, 307, 352, 1632, 6505, 1604, 52203, 314, 1604, 307, 352, 11453, 405, 506, 602, 9764, 290, 2613, 9279, 3582, 6602, 11, 538, 1062, 198, 506, 2613, 100315, 314, 3582, 109210, 4231, 12526, 14692, 13414, 11, 1604, 52203, 741, 29969, 740, 506, 3981, 2613, 100315, 405, 699, 6083, 2145, 8, 871, 405, 968, 538, 9279, 100315, 13678, 4366, 919, 29271, 10181, 416, 485, 76, 10289, 416, 4636, 405, 1686, 2338, 307, 968, 606, 968, 1604, 52203, 314, 284, 10181, 416, 659, 220, 16, 307, 699, 606, 699, 4662, 871, 2338, 412, 506, 606, 352, 606, 352, 1632, 1268, 314, 2613, 100315, 7975, 15400, 13414, 41284, 2547, 1022, 76, 91, 284, 10181, 9172, 352, 602, 58168, 11, 2105, 581, 810, 11, 12221, 495, 18485, 316, 1175, 12949, 62, 110193, 106196, 198, 352, 395, 2450, 306, 42588, 4231, 27043, 4366, 919, 62179, 485, 419, 4636, 405, 506, 1632, 9047, 314, 2450, 107220, 1454, 288, 6182, 1454, 288, 28036, 740, 506, 538, 1632, 6083, 23460, 8, 314, 1051, 151105, 775, 172522, 8, 405, 699, 2174, 126000, 13535, 13683, 314, 220, 16, 307, 699, 2881, 4885, 9000, 10346, 362, 699, 4901, 307, 506, 606, 506, 1632, 20290, 314, 9239, 34819, 27276, 172522, 11, 827, 1156, 151105, 362, 506, 2174, 126000, 13535, 13683, 314, 20290, 41284, 740, 506, 2881, 34151, 4366, 64329, 362, 352, 758, 352, 3981, 2613, 100315, 405, 506, 602, 1958, 2105, 581, 9349, 290, 3582, 6602, 198, 506, 6083, 2145, 8, 871, 405, 699, 1632, 9047, 314, 284, 9416, 6182, 740, 699, 1632, 6602, 314, 1051, 118760, 65830, 91440, 23234, 12778, 1735, 699, 2881, 4885, 23460, 362, 699, 1604, 314, 284, 10289, 740, 699, 2174, 126000, 13535, 13683, 314, 220, 15, 307, 506, 606, 506, 4662, 871, 2338, 412, 352, 606, 309, 758, 309, 602, 2174, 126000, 13535, 13683, 382, 1495, 1991, 20290, 5831, 591, 290, 2174, 42588, 12648, 13, 1328, 382, 2061, 198, 309, 602, 395, 38801, 85238, 20290, 11, 3630, 481, 8535, 20345, 5251, 350, 55927, 8, 42588, 76848, 198, 309, 350, 3431, 11, 2174, 126000, 13535, 13683, 446, 271, 758, 271, 9784, 1175, 148939, 26359, 126000, 13535, 13683, 1607, 309, 827, 1156, 412, 309, 20290, 25, 22915, 27, 112511, 23844, 309, 6505, 2174, 126000, 13535, 13683, 25, 46737, 412, 271, 1546, 2747, 350, 18136, 27, 112511, 17084, 46737, 8, 405, 309, 602, 27864, 11, 290, 14245, 1919, 1039, 42588, 76848, 665, 413, 85238, 558, 309, 602, 2214, 290, 14189, 328, 38801, 85238, 20290, 11, 85238, 42588, 87130, 198, 309, 602, 382, 1606, 261, 4792, 538, 261, 12648, 484, 673, 3333, 117824, 11, 3630, 495, 665, 198, 309, 602, 4124, 316, 107009, 328, 20290, 10275, 4525, 316, 413, 22105, 558, 309, 602, 567, 1353, 74, 17096, 4787, 1039, 2615, 3479, 656, 3463, 290, 2381, 82, 9, 58612, 81, 3392, 15344, 198, 309, 602, 8302, 13, 1328, 665, 319, 1940, 13, 7075, 6512, 77, 1, 659, 392, 392, 316, 5025, 6512, 77, 2381, 77, 26880, 309, 602, 7306, 382, 261, 4853, 326, 32042, 9295, 734, 309, 405, 352, 1632, 6602, 15224, 13305, 36700, 314, 1022, 10346, 91, 405, 506, 1051, 166487, 198, 699, 887, 522, 23460, 446, 699, 887, 3726, 44441, 10346, 28036, 91, 405, 968, 6602, 28036, 198, 1686, 887, 2340, 1234, 1686, 887, 25656, 1234, 1686, 887, 586, 44441, 5, 65, 91, 723, 65, 6, 14089, 287, 36853, 77, 787, 287, 36853, 83, 15316, 21841, 4366, 65, 2210, 699, 4759, 699, 887, 29969, 15400, 6216, 446, 352, 4753, 352, 538, 2174, 126000, 13535, 13683, 1424, 220, 15, 198, 506, 1725, 6602, 15224, 13305, 36700, 4366, 64329, 32184, 15515, 41284, 416, 533, 2174, 126000, 13535, 13683, 5169, 352, 405, 506, 2049, 350, 6684, 126000, 13535, 13683, 464, 20290, 41284, 4574, 699, 1725, 6602, 15224, 13305, 36700, 4366, 64329, 32184, 15515, 41284, 416, 533, 2174, 126000, 13535, 13683, 533, 220, 16, 5169, 506, 405, 699, 2174, 126000, 13535, 13683, 2757, 220, 16, 307, 506, 606, 352, 606, 309, 606, 309, 15199, 40989, 20836, 6684, 126000, 13535, 13683, 5017, 20290, 41284, 9172, 309, 350, 64329, 11, 2174, 126000, 13535, 13683, 446, 271, 758, 271, 9784, 1175, 12949, 11119, 55927, 106196, 1607, 309, 827, 1156, 412, 309, 2201, 25, 827, 689, 412, 309, 9279, 100315, 25, 827, 8579, 1984, 114986, 689, 23844, 271, 1546, 2747, 350, 18136, 27, 112511, 17084, 10242, 1984, 115716, 27, 112511, 193859, 405, 309, 1632, 350, 64329, 11, 2174, 126000, 13535, 13683, 8, 314, 1051, 2697, 12949, 106196, 13414, 11, 9279, 100315, 362, 309, 538, 2174, 126000, 13535, 13683, 951, 220, 15, 405, 352, 602, 1843, 2174, 126000, 13535, 13683, 382, 12421, 11, 290, 2174, 6602, 673, 261, 3582, 6602, 326, 581, 679, 198, 352, 602, 860, 85238, 11643, 198, 352, 622, 350, 64329, 11, 10242, 1984, 742, 1389, 2147, 309, 606, 309, 1632, 350, 10522, 20290, 11, 2174, 126000, 13535, 13683, 8, 7732, 352, 1051, 2697, 148939, 26359, 126000, 13535, 13683, 74272, 11, 2174, 126000, 13535, 13683, 861, 309, 1632, 85238, 28036, 314, 1051, 2697, 44196, 106196, 4366, 64329, 32184, 15515, 41284, 416, 533, 2174, 126000, 13535, 13683, 485, 4640, 309, 20290, 8992, 54368, 74272, 41284, 416, 533, 2174, 126000, 13535, 13683, 861, 309, 602, 9691, 25, 581, 1757, 2075, 27008, 316, 1646, 6623, 22105, 20290, 198, 309, 602, 1328, 1481, 10389, 290, 4527, 328, 2881, 2488, 5066, 1261, 38801, 5127, 756, 198, 309, 602, 41199, 316, 290, 18485, 306, 448, 12787, 3926, 328, 495, 1974, 279, 309, 1632, 6505, 5127, 756, 314, 10242, 1984, 742, 1389, 740, 309, 538, 85238, 28036, 3109, 35051, 416, 405, 352, 622, 350, 64329, 11, 5127, 756, 362, 309, 758, 309, 602, 1328, 382, 290, 4224, 3546, 13, 6214, 1646, 722, 4590, 20290, 484, 1604, 483, 85238, 28036, 198, 309, 602, 350, 26167, 20290, 484, 9707, 3981, 85238, 28036, 446, 309, 602, 146791, 1365, 495, 591, 290, 11453, 4895, 9335, 483, 6198, 306, 261, 5355, 1890, 558, 309, 1632, 6505, 2438, 314, 1051, 198, 352, 887, 64288, 13535, 28036, 198, 352, 887, 67655, 15466, 44441, 87, 91, 1215, 9416, 62789, 416, 464, 85238, 28036, 9416, 62789, 2147, 309, 2049, 2438, 464, 1051, 165009, 13535, 28036, 41284, 1234, 352, 1725, 1051, 165009, 13535, 28036, 58, 4859, 1651, 61429, 14161, 4366, 23823, 562, 28036, 446, 309, 405, 352, 5127, 756, 13222, 57650, 0, 20070, 506, 1051, 151105, 33734, 165009, 13535, 28036, 58, 4859, 1651, 288, 62789, 169856, 352, 28211, 352, 2438, 2757, 220, 16, 307, 309, 758, 309, 602, 6549, 6096, 1952, 945, 128602, 9578, 13, 3604, 1753, 350, 2141, 8, 4149, 4010, 395, 290, 989, 964, 3321, 198, 309, 602, 6602, 11, 192808, 6623, 11643, 591, 484, 6602, 350, 366, 1062, 8, 316, 85238, 28036, 412, 309, 602, 326, 2881, 2488, 1096, 290, 6062, 4435, 326, 1921, 1412, 581, 717, 558, 309, 395, 575, 306, 220, 16, 485, 23823, 562, 28036, 41284, 416, 405, 352, 1632, 18633, 314, 827, 23823, 562, 28036, 165124, 72, 1735, 352, 1632, 43412, 314, 827, 23823, 562, 28036, 1768, 485, 1735, 352, 1632, 6505, 2438, 314, 1051, 198, 506, 887, 64288, 13535, 28036, 198, 506, 887, 67655, 15466, 44441, 87, 91, 1215, 9416, 62789, 416, 464, 43412, 362, 352, 602, 9691, 25, 81164, 139802, 538, 43412, 13217, 483, 392, 392, 3901, 352, 2049, 2438, 464, 1051, 165009, 13535, 28036, 41284, 1234, 506, 1725, 1051, 165009, 13535, 28036, 58, 4859, 1651, 61429, 14161, 1858, 24677, 446, 352, 405, 506, 1632, 21568, 314, 723, 26155, 11, 1051, 165009, 13535, 28036, 58, 4859, 1651, 288, 62789, 134663, 41859, 740, 506, 1632, 41096, 314, 3981, 3081, 742, 689, 742, 2845, 94167, 23, 4366, 67395, 4295, 8, 405, 699, 602, 12328, 2146, 11, 495, 382, 9239, 34819, 27276, 4366, 67395, 4295, 11, 827, 1156, 151105, 446, 699, 602, 3072, 581, 3572, 679, 17882, 261, 42588, 12648, 1118, 1481, 7332, 176901, 558, 699, 602, 350, 148588, 4149, 306, 290, 14324, 328, 85238, 42588, 76848, 446, 699, 602, 2632, 8452, 316, 52888, 12, 23, 326, 621, 42588, 87130, 558, 699, 602, 457, 1940, 13, 483, 567, 1353, 74, 17096, 392, 220, 116932, 8625, 12648, 316, 392, 392, 659, 392, 1073, 1150, 699, 602, 889, 9239, 34819, 27276, 568, 220, 1073, 1405, 1666, 9239, 34819, 27276, 568, 31232, 699, 13927, 1858, 8, 871, 1051, 2697, 12949, 62, 110193, 106196, 1858, 38448, 699, 602, 8529, 2905, 11, 5588, 503, 625, 495, 10454, 382, 6145, 20102, 402, 5588, 1354, 198, 699, 602, 1481, 413, 261, 42588, 12648, 2254, 290, 52888, 12, 23, 132880, 387, 2438, 558, 699, 602, 63659, 38273, 4951, 484, 860, 1001, 738, 4862, 8611, 350, 13181, 722, 11, 1665, 9289, 198, 699, 602, 8611, 722, 290, 3464, 30416, 306, 290, 5880, 85238, 6602, 13847, 446, 699, 27100, 108255, 871, 9239, 34819, 27276, 4366, 67395, 4295, 11, 827, 1156, 151105, 2313, 699, 602, 42566, 1299, 290, 3992, 382, 61631, 889, 25570, 734, 699, 602, 27100, 3454, 8, 871, 1051, 2697, 12949, 62, 110193, 106196, 7, 66907, 405, 699, 602, 257, 3081, 742, 689, 742, 2845, 94167, 23, 11119, 14772, 4366, 67395, 4295, 165124, 68, 41805, 18721, 5089, 141457, 699, 602, 21968, 506, 4753, 506, 1632, 6505, 26775, 314, 22915, 742, 1389, 740, 506, 1632, 6505, 26775, 13683, 314, 220, 15, 307, 506, 395, 6602, 306, 41096, 405, 699, 26775, 4885, 23460, 362, 699, 26775, 13683, 2757, 1051, 166487, 58, 5, 10346, 1651, 3303, 740, 699, 538, 26775, 13683, 5064, 85238, 28036, 41284, 416, 405, 968, 2338, 307, 699, 606, 506, 606, 506, 5127, 756, 13222, 78669, 362, 506, 2438, 2757, 220, 16, 307, 352, 606, 309, 758, 309, 602, 1328, 382, 1217, 625, 35737, 13, 8406, 581, 12190, 19429, 484, 42588, 76848, 553, 22105, 412, 309, 602, 40817, 11, 1023, 553, 625, 13, 4569, 382, 11, 538, 12673, 11643, 1504, 316, 1520, 261, 12648, 7680, 306, 198, 309, 602, 85238, 28036, 11, 495, 2023, 1520, 20290, 4149, 1118, 1039, 18485, 1481, 10275, 2411, 198, 309, 602, 1481, 413, 49536, 558, 309, 602, 2214, 4994, 11, 483, 329, 555, 17, 11, 290, 1199, 328, 2381, 82, 31717, 25309, 59, 50, 8, 4748, 484, 6512, 77, 3392, 1, 2023, 198, 309, 602, 2575, 261, 12648, 11, 319, 1940, 13, 6512, 77, 3392, 15, 1, 76848, 1511, 6512, 77, 1, 69105, 77, 149331, 15, 672, 4137, 6512, 77, 1, 261, 4149, 6602, 558, 309, 602, 7306, 382, 261, 4853, 326, 32042, 9295, 734, 309, 602, 1328, 12471, 1849, 538, 581, 4862, 6234, 2381, 82, 31717, 25309, 59, 50, 446, 309, 538, 85238, 28036, 41284, 416, 1424, 220, 16, 405, 352, 1632, 2174, 33390, 11456, 314, 287, 689, 742, 44196, 26359, 94167, 23, 7, 23823, 562, 28036, 9416, 62789, 2147, 352, 538, 85238, 28036, 41284, 416, 533, 2174, 33390, 11456, 13, 16, 1424, 220, 15, 198, 506, 1725, 2174, 33390, 11456, 13, 15, 7975, 15400, 6216, 11, 1022, 66, 91, 274, 3109, 86186, 67973, 4574, 352, 405, 506, 1632, 6505, 322, 35217, 314, 9239, 34819, 27276, 1607, 699, 827, 23823, 562, 28036, 165124, 23823, 562, 28036, 41284, 416, 533, 2174, 33390, 11456, 13, 16, 2274, 699, 827, 1156, 151105, 412, 506, 2596, 506, 322, 35217, 34151, 36692, 34819, 27276, 1607, 699, 827, 23823, 562, 28036, 58, 23823, 562, 28036, 41284, 416, 533, 2174, 33390, 11456, 13, 16, 485, 2274, 699, 827, 1156, 151105, 412, 506, 37882, 506, 5127, 756, 13222, 8178, 35217, 362, 352, 606, 309, 758, 309, 350, 64329, 11, 5127, 756, 446, 271, 606, 943, 24654, 79, 1818, 1545, 82, 1592, 10766, 16309, 33, 3111, 405, 271, 22063, 1389, 1592, 271, 9784, 620, 1607, 309, 49416, 25, 10242, 2307, 115716, 76960, 23, 17084, 46737, 23844, 309, 3582, 65830, 91440, 25, 10242, 2307, 4920, 11, 46737, 23844, 309, 8302, 25, 827, 689, 412, 271, 1546, 2747, 15993, 2769, 128891, 29, 405, 309, 1632, 42588, 314, 59319, 742, 1389, 62660, 446, 352, 887, 3726, 20493, 44441, 68, 91, 15993, 11173, 742, 1389, 51996, 196013, 742, 37863, 1638, 2255, 11, 1175, 4513, 68, 3552, 8609, 16829, 138400, 309, 1632, 3582, 109210, 314, 405, 352, 1632, 1175, 34761, 314, 3582, 65830, 91440, 198, 506, 887, 20817, 1234, 506, 887, 3726, 44441, 82, 91, 39357, 109210, 742, 27731, 1858, 2210, 506, 887, 25670, 51996, 18136, 77452, 79023, 352, 59319, 742, 1389, 120881, 34761, 11004, 83692, 10990, 506, 887, 3726, 20493, 44441, 68, 91, 15993, 11173, 742, 1389, 51996, 196013, 742, 37863, 1638, 2255, 11, 1175, 4513, 68, 3552, 8609, 16829, 3901, 309, 6755, 309, 1632, 53790, 25, 10242, 2307, 27, 112511, 11, 22915, 76960, 23, 3920, 7732, 352, 49416, 42608, 1454, 3726, 44441, 7, 74, 11, 323, 47976, 10393, 85, 11, 372, 29419, 60498, 25670, 2457, 309, 3696, 20836, 42863, 41284, 416, 951, 53790, 41284, 9172, 309, 1632, 3582, 65830, 127170, 25, 10242, 2307, 27, 112511, 11, 22915, 76960, 23, 3920, 314, 3582, 65830, 91440, 198, 352, 887, 2340, 1234, 352, 887, 3726, 44441, 7, 74, 11, 323, 47976, 10393, 85, 11, 372, 9416, 28036, 1454, 935, 29582, 24521, 352, 887, 25670, 2457, 309, 602, 53663, 2236, 357, 4128, 1761, 1495, 316, 5485, 56665, 5477, 625, 2966, 316, 3343, 290, 4014, 198, 309, 1632, 6505, 20099, 13535, 28036, 25, 22915, 115716, 76960, 23, 3920, 314, 49416, 18392, 1454, 565, 26705, 1454, 25670, 740, 309, 20099, 13535, 28036, 19551, 2457, 309, 13927, 148985, 33, 3111, 405, 352, 49416, 412, 352, 3582, 65830, 91440, 412, 352, 53790, 412, 352, 3582, 65830, 127170, 412, 352, 42588, 183618, 25, 350, 15, 485, 21911, 20468, 142185, 741, 3726, 44441, 75590, 42588, 29419, 9503, 25670, 5757, 352, 3582, 109210, 183618, 25, 350, 15, 485, 21911, 20468, 142185, 446, 506, 887, 3726, 44441, 75590, 3582, 109210, 29419, 4574, 506, 887, 25670, 5757, 352, 20099, 13535, 28036, 412, 309, 4759, 271, 758, 271, 602, 54367, 46988, 271, 602, 70820, 198, 271, 602, 54367, 17380, 279, 271, 9784, 31288, 62, 110193, 4366, 1156, 11, 13104, 25, 26534, 11, 2201, 25, 827, 689, 8, 2747, 22915, 27, 112511, 29, 405, 309, 13104, 91717, 73844, 173315, 1051, 2697, 12949, 62, 110193, 106196, 13414, 2210, 271, 758, 271, 9784, 31288, 4366, 1156, 11, 13104, 25, 26534, 11, 2201, 25, 827, 689, 11, 9279, 100315, 25, 10242, 1984, 114986, 689, 17991, 2747, 22915, 27, 112511, 29, 405, 309, 13104, 91717, 73844, 173315, 1051, 2697, 12949, 106196, 13414, 11, 827, 32510, 100315, 741, 15, 446, 271, 758, 271, 9784, 1175, 12949, 28036, 4366, 1156, 11, 13104, 25, 26534, 11, 11643, 25, 96391, 84, 23, 4636, 2747, 22915, 27, 112511, 29, 405, 309, 13104, 91717, 73844, 173315, 405, 352, 3981, 3081, 742, 689, 742, 2845, 94167, 23, 41557, 8, 405, 506, 13927, 13414, 8, 871, 1051, 2697, 12949, 62, 110193, 106196, 13414, 2313, 506, 27100, 3454, 8, 871, 405, 699, 1632, 2201, 314, 37750, 354, 3081, 742, 689, 742, 2845, 94167, 23, 11119, 14772, 4366, 17941, 165124, 68, 41805, 18721, 5089, 416, 4636, 4753, 699, 1632, 350, 64329, 11, 2174, 126000, 13535, 13683, 8, 314, 1051, 2697, 12949, 106196, 13414, 11, 827, 8579, 1984, 742, 1389, 2147, 699, 1632, 350, 10522, 20290, 11, 2174, 126000, 13535, 13683, 8, 7732, 968, 1051, 2697, 148939, 26359, 126000, 13535, 13683, 74272, 11, 2174, 126000, 13535, 13683, 362, 699, 538, 1073, 64329, 3109, 35051, 416, 1725, 2174, 126000, 13535, 13683, 1424, 220, 15, 405, 968, 602, 154580, 1277, 290, 20290, 591, 290, 2174, 9047, 326, 2461, 418, 3111, 402, 290, 15448, 11643, 198, 968, 602, 6083, 13347, 38273, 11, 889, 495, 1340, 625, 413, 6145, 538, 68530, 679, 1458, 261, 42588, 198, 968, 602, 12648, 2870, 290, 4529, 52888, 12, 23, 326, 290, 15796, 11643, 11, 1118, 382, 4436, 495, 198, 968, 602, 2595, 382, 1249, 198, 968, 1632, 6505, 85238, 28036, 7732, 1686, 1051, 2697, 44196, 106196, 4366, 64329, 32184, 15515, 41284, 416, 533, 2174, 126000, 13535, 13683, 485, 4640, 968, 85238, 28036, 34151, 12526, 62789, 4366, 17941, 51414, 41805, 18721, 5089, 416, 485, 20114, 968, 20290, 8992, 54368, 74272, 41284, 416, 533, 2174, 126000, 13535, 13683, 362, 968, 20290, 34151, 36692, 34819, 27276, 4366, 23823, 562, 28036, 11, 827, 1156, 151105, 1820, 699, 606, 699, 20290, 198, 506, 606, 352, 606, 309, 4759, 271, 758, 271, 9784, 31288, 14161, 11119, 55927, 1607, 309, 827, 1156, 412, 309, 13104, 25, 26534, 412, 309, 2201, 25, 827, 689, 412, 309, 9279, 100315, 25, 10242, 1984, 114986, 689, 23844, 271, 1546, 2747, 15993, 27, 37863, 40048, 29, 405, 309, 1632, 350, 64329, 11, 5127, 756, 8, 7732, 352, 13104, 91717, 73844, 173315, 1051, 2697, 12949, 11119, 55927, 106196, 13414, 11, 827, 32510, 100315, 1820, 309, 1632, 13104, 10828, 24884, 756, 7732, 352, 15993, 950, 742, 1389, 139986, 11, 5127, 756, 42608, 1454, 3726, 44441, 27794, 91, 15993, 950, 742, 1389, 139986, 11, 827, 27794, 165124, 199793, 309, 350, 64329, 11, 13104, 10828, 24884, 756, 741, 14020, 118507, 139986, 446, 271, 758, 271, 9784, 31288, 46053, 13535, 4366, 1156, 11, 9047, 25, 96391, 84, 23, 4636, 2747, 15993, 2769, 27, 112511, 29, 405, 309, 538, 1632, 6083, 23460, 8, 314, 1051, 151105, 775, 172522, 741, 47750, 1165, 416, 405, 352, 622, 13927, 23460, 362, 309, 606, 309, 538, 1632, 13927, 172522, 6182, 8, 314, 3081, 742, 689, 742, 2845, 94167, 23, 172522, 8, 405, 352, 538, 1632, 6083, 23460, 8, 314, 1051, 118760, 65830, 91440, 775, 172522, 6182, 741, 47750, 1165, 416, 405, 506, 622, 13927, 23460, 362, 352, 606, 309, 606, 309, 27100, 104059, 11173, 742, 1389, 51996, 196013, 742, 37863, 2035, 2255, 11, 1175, 4513, 30871, 3552, 117029, 24521, 271, 758, 271, 9784, 31288, 46053, 126000, 4366, 1156, 11, 9047, 25, 96391, 84, 23, 4636, 2747, 22915, 27, 112511, 29, 405, 309, 538, 1632, 6083, 23460, 8, 314, 1051, 151105, 775, 172522, 8, 405, 352, 622, 10563, 46759, 9, 10346, 1735, 309, 606, 309, 9239, 34819, 27276, 172522, 11, 827, 1156, 151105, 446, 271, 758, 271, 602, 54367, 46988, 271, 602, 5244, 7148, 198, 271, 602, 54367, 17380, 279, 271, 9784, 32043, 28036, 4366, 1156, 11, 13104, 25, 26534, 11, 20290, 25, 22915, 27, 112511, 17991, 2747, 15993, 27, 37863, 10629, 29, 405, 309, 1632, 11643, 314, 13104, 91717, 73844, 173315, 1051, 2697, 44196, 106196, 4366, 64329, 1820, 309, 15993, 10629, 742, 1389, 139986, 11, 827, 17941, 741, 14020, 1234, 271, 758, 271, 9784, 32043, 46053, 13535, 28036, 4366, 1156, 11, 13104, 25, 26534, 11, 6602, 25, 46737, 8, 2747, 15993, 2769, 27, 37863, 27, 37863, 10629, 3920, 405, 309, 538, 1632, 6083, 41557, 8, 314, 1051, 166487, 775, 4366, 10346, 8, 405, 352, 622, 13927, 104059, 10629, 742, 1389, 139986, 11, 11643, 741, 14020, 2147, 309, 606, 309, 538, 1632, 6083, 41557, 8, 314, 1051, 118760, 65830, 127170, 775, 4366, 10346, 8, 405, 352, 622, 13927, 104059, 10629, 742, 1389, 139986, 11, 11643, 741, 14020, 2147, 309, 606, 309, 27100, 104059, 11173, 742, 1389, 51996, 196013, 742, 37863, 2035, 2255, 11, 1175, 4513, 10346, 3552, 8609, 24521, 271, 758, 271, 602, 54367, 46988, 271, 602, 94557, 108338, 198, 271, 602, 54367, 17380, 279, 271, 9784, 6602, 48405, 20393, 4366, 1156, 11, 13104, 25, 26534, 8, 2747, 22915, 27, 37863, 27, 37863, 10629, 3920, 405, 309, 1051, 165009, 13535, 28036, 198, 352, 887, 2340, 1234, 352, 887, 3726, 44441, 87, 91, 15993, 10629, 742, 1389, 139986, 11, 1215, 741, 14020, 4574, 352, 887, 25670, 1234, 271, 606, 943, 24654, 79, 1818, 2917, 1592, 13682, 1175, 83, 8251, 2488, 4963, 5823, 25, 26534, 11, 284, 25, 827, 37863, 5300, 8, 2747, 15993, 2769, 149289, 405, 271, 284, 1950, 10602, 51996, 9243, 33, 3111, 23177, 71020, 271, 13927, 106732, 943, 24654, 29312, 16066, 11587, 4847, 10742, 405, 271, 1199, 20294, 66, 20844, 742, 91612, 8579, 2307, 472, 10242, 2307, 502, 271, 1199, 34715, 742, 7216, 34819, 39921, 502, 271, 22063, 3190, 1592, 271, 9784, 1869, 73215, 9969, 416, 405, 309, 1632, 6505, 40642, 314, 10242, 2307, 742, 4211, 740, 309, 40642, 13222, 3229, 1, 378, 4050, 935, 29582, 2547, 220, 16, 362, 309, 40642, 13222, 3229, 1, 8301, 4050, 935, 29582, 2547, 220, 17, 861, 309, 1632, 694, 314, 9239, 34819, 39921, 3229, 1, 152936, 672, 827, 81, 4096, 362, 309, 3696, 28373, 20836, 430, 11, 10563, 46759, 65, 1, 378, 672, 287, 1, 8301, 34923, 271, 606, 739] \ No newline at end of file diff --git a/tokenizer_ts/test/tikTokenizerGpt4o.test.ts b/tokenizer_ts/test/tikTokenizerGpt4o.test.ts new file mode 100644 index 0000000..0d302c2 --- /dev/null +++ b/tokenizer_ts/test/tikTokenizerGpt4o.test.ts @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import * as assert from "assert"; +import * as fs from "fs"; +import { suite, before } from "mocha"; +import { createByModelName } from "../src/tokenizerBuilder"; +import { TikTokenizer } from "../src/tikTokenizer"; +const ENDOFTEXT: string = "<|endoftext|>"; +const ENDOFPROMPT: string = "<|endofprompt|>"; +const specialTokens: ReadonlyMap = new Map([ + [ENDOFTEXT, 199999], + [ENDOFPROMPT, 200018] +]); + +suite("TikTokenizer gpt-4o Test Suite", function() { + let tokenizer_gpt4o: TikTokenizer; + before(async () => { + tokenizer_gpt4o = await createByModelName("gpt-4o", specialTokens); + }); + + test("tokenize source code - gpt-4o", done => { + const source = fs.readFileSync("test/testdata/lib.rs.txt", "utf8"); + const filePath = "test/testdata/tokens_gpt_4o.json"; + + fs.readFile(filePath, "utf8", (err, data) => { + assert.strictEqual(err, null); + const jsonArray = JSON.parse(data) as Array; + let encoded = tokenizer_gpt4o.encode( + source, + Array.from(specialTokens.keys()) + ); + assert.deepStrictEqual(encoded.length, 5609); + assert.deepStrictEqual(encoded, jsonArray); + assert.strictEqual(tokenizer_gpt4o.decode(encoded), source); + done(); + }); + }); +});