-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Move the Tokenizer's data into separate packages. (#7248)
* Move the Tokenizer's data into separate packages. * Address the feedback * More feedback addressing * More feedback addressing * Trimming/AoT support * Make data types internal
- Loading branch information
Showing
29 changed files
with
729 additions
and
123 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| <Project> | ||
| <UsingTask TaskName="CompressFile" | ||
| TaskFactory="RoslynCodeTaskFactory" | ||
| AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.Core.dll" > | ||
| <ParameterGroup> | ||
| <Files ParameterType="Microsoft.Build.Framework.ITaskItem[]" Required="true" /> | ||
| </ParameterGroup> | ||
| <Task> | ||
| <Using Namespace="System.Globalization" /> | ||
| <Using Namespace="System.IO" /> | ||
| <Using Namespace="System.IO.Compression" /> | ||
| <Code Type="Fragment" Language="cs"> | ||
| <![CDATA[ | ||
| foreach (var file in Files) | ||
| { | ||
| string fileName = file.GetMetadata("FullPath"); | ||
| string fileContent = File.ReadAllText(fileName); | ||
| int capacity = 1; | ||
| int eolIndex = 0; | ||
| do | ||
| { | ||
| if ((eolIndex = fileContent.IndexOf('\n', eolIndex)) >= 0) | ||
| { | ||
| eolIndex++; | ||
| capacity++; | ||
| } | ||
| else | ||
| { | ||
| break; | ||
| } | ||
| } while (eolIndex < fileContent.Length); | ||
| using var sourceStream = File.OpenRead(fileName); | ||
| using var reader = new StreamReader(sourceStream); | ||
| using var destStream = new DeflateStream(File.Create(file.GetMetadata("Destination")), CompressionLevel.Optimal); | ||
| using var streamWriter = new StreamWriter(destStream); | ||
| streamWriter.WriteLine($"Capacity: {capacity.ToString(CultureInfo.InvariantCulture)}"); | ||
| string line; | ||
| int destLineNumber = 0; | ||
| while ((line = reader.ReadLine()) != null) | ||
| { | ||
| if (line.Length == 0) { continue; } | ||
| int index = line.IndexOf(' '); | ||
| if (index <= 0 || index == line.Length - 1 || !int.TryParse(line.Substring(index + 1), out int id) || id < destLineNumber) | ||
| { | ||
| Log.LogError($"Invalid format in the file {file.GetMetadata("FullPath")} line {line}"); | ||
| break; | ||
| } | ||
| while (destLineNumber < id) | ||
| { | ||
| // ensure id always aligns with the line number | ||
| streamWriter.WriteLine(); | ||
| destLineNumber++; | ||
| } | ||
| streamWriter.WriteLine(line.Substring(0, index)); | ||
| destLineNumber++; | ||
| } | ||
| } | ||
| ]]> | ||
| </Code> | ||
| </Task> | ||
| </UsingTask> | ||
|
|
||
| <Target Name="CompressTiktokenData" | ||
| BeforeTargets="AssignTargetPaths" | ||
| DependsOnTargets="_EnsureTokenizerDataEmbeddedResourceDestination" | ||
| Inputs="@(TokenizerDataEmbeddedResource)" | ||
| Outputs="@(TokenizerDataEmbeddedResource->'%(Destination)')"> | ||
|
|
||
| <CompressFile Files="@(TokenizerDataEmbeddedResource)" /> | ||
|
|
||
| <ItemGroup> | ||
| <EmbeddedResource Include="@(TokenizerDataEmbeddedResource->'%(Destination)')" LogicalName="%(FileName)%(Extension).deflate" /> | ||
| </ItemGroup> | ||
| </Target> | ||
|
|
||
| <Target Name="_EnsureTokenizerDataEmbeddedResourceDestination" > | ||
| <ItemGroup> | ||
| <TokenizerDataEmbeddedResource Condition="'%(TokenizerDataEmbeddedResource.Destination)' == ''" Destination="$(IntermediateOutputPath)%(FileName).deflate" /> | ||
| </ItemGroup> | ||
| </Target> | ||
| </Project> |
13 changes: 13 additions & 0 deletions
13
src/Microsoft.ML.Tokenizers.Data.Cl100kBase/Cl100kBaseTokenizerData.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
|
|
||
| namespace Microsoft.ML.Tokenizers | ||
| { | ||
| /// <summary> | ||
| /// Cl100kBaseTokenizerData is internally used by Microsoft.ML.Tokenizers library to bind to the cl100k_base.tiktoken data file. | ||
| /// </summary> | ||
| internal sealed class Cl100kBaseTokenizerData | ||
| { | ||
| } | ||
| } |
File renamed without changes.
31 changes: 31 additions & 0 deletions
31
src/Microsoft.ML.Tokenizers.Data.Cl100kBase/Microsoft.ML.Tokenizers.Data.Cl100kBase.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>netstandard2.0</TargetFramework> | ||
| <Nullable>enable</Nullable> | ||
| <IsPackable>true</IsPackable> | ||
| <PackageDescription>The Microsoft.ML.Tokenizers.Data.Cl100kBase class includes the Tiktoken tokenizer data file cl100k_base.tiktoken, which is utilized by models such as GPT-4.</PackageDescription> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <!-- | ||
| The following file are compressed using the DeflateStream and embedded as resources in the assembly. | ||
| The files are downloaded from the following sources and compressed to the Destination. | ||
| - cl100k_base.tiktoken: https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken | ||
| The file under MIT copyright license https://github.com/openai/tiktoken/blob/main/LICENSE | ||
| In the CompressFile task above we modify the file's content to elimenate the ranks, thus reducing the file size, | ||
| since the rank corresponds to the line number in the file. For the file p50k_base.tiktoken, | ||
| we introduce empty lines to replace any missing ranks, ensuring that the rank consistently aligns with the line number. | ||
| After we eleminate the ranks from the file, we compress the file using the DeflateStream and embed it as a resource in the assembly. | ||
| --> | ||
| <TokenizerDataEmbeddedResource Include="Data\cl100k_base.tiktoken" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj"/> | ||
| </ItemGroup> | ||
|
|
||
| <Import Project="$(RepositoryEngineeringDir)TokenizerData.targets" /> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| ## About | ||
|
|
||
| The `Microsoft.ML.Tokenizers.Data.Cl100kBase` includes the Tiktoken tokenizer data file `cl100k_base.tiktoken`, which is utilized by models such as GPT-4. | ||
|
|
||
| ## Key Features | ||
|
|
||
| * This package mainly contains the cl100k_base.tiktoken file, which is used by the Tiktoken tokenizer. This data file is used by the following models: | ||
| 1. gpt-4 | ||
| 2. gpt-3.5-turbo | ||
| 3. gpt-3.5-turbo-16k | ||
| 4. gpt-35 | ||
| 5. gpt-35-turbo | ||
| 6. gpt-35-turbo-16k | ||
| 7. text-embedding-ada-002 | ||
| 8. text-embedding-3-small | ||
| 9. text-embedding-3-large | ||
|
|
||
| ## How to Use | ||
|
|
||
| Reference this package in your project to use the Tiktoken tokenizer with the specified models. | ||
|
|
||
| ```csharp | ||
|
|
||
| // Create a tokenizer for the specified model or any other listed model name | ||
| Tokenizer tokenizer = TiktokenTokenizer.CreateForModel("gpt-4"); | ||
|
|
||
| // Create a tokenizer for the specified encoding | ||
| Tokenizer tokenizer = TiktokenTokenizer.CreateForEncoding("cl100k_base"); | ||
|
|
||
| ``` | ||
|
|
||
| ## Main Types | ||
|
|
||
| Users shouldn't use any types exposed by this package directly. This package is intended to provide tokenizer data files. | ||
|
|
||
| ## Additional Documentation | ||
|
|
||
| * [API documentation](https://learn.microsoft.com/en-us/dotnet/api/microsoft.ml.tokenizers) | ||
|
|
||
| ## Related Packages | ||
|
|
||
| <!-- The related packages associated with this package --> | ||
| Microsoft.ML.Tokenizers | ||
|
|
||
| ## Feedback & Contributing | ||
|
|
||
| Microsoft.ML.Tokenizers.Data.Cl100kBase is released as open source under the [MIT license](https://licenses.nuget.org/MIT). Bug reports and contributions are welcome at [the GitHub repository](https://github.com/dotnet/machinelearning). |
File renamed without changes.
13 changes: 13 additions & 0 deletions
13
src/Microsoft.ML.Tokenizers.Data.Gpt2/Gpt2TokenizerData.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
|
|
||
| namespace Microsoft.ML.Tokenizers | ||
| { | ||
| /// <summary> | ||
| /// Gpt2TokenizerData is internally used by Microsoft.ML.Tokenizers library to bind to the gpt2.tiktoken data file. | ||
| /// </summary> | ||
| internal sealed class Gpt2TokenizerData | ||
| { | ||
| } | ||
| } |
31 changes: 31 additions & 0 deletions
31
src/Microsoft.ML.Tokenizers.Data.Gpt2/Microsoft.ML.Tokenizers.Data.Gpt2.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>netstandard2.0</TargetFramework> | ||
| <Nullable>enable</Nullable> | ||
| <IsPackable>true</IsPackable> | ||
| <PackageDescription>The Microsoft.ML.Tokenizers.Data.Gpt2 includes the Tiktoken tokenizer data file gpt2.tiktoken, which is utilized by models such as Gpt-2.</PackageDescription> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <!-- | ||
| The following file are compressed using the DeflateStream and embedded as resources in the assembly. | ||
| The files are downloaded from the following sources and compressed to the Destination. | ||
| - gpt2.tiktoken: https://fossies.org/linux/misc/whisper-20231117.tar.gz/whisper-20231117/whisper/assets/gpt2.tiktoken?m=b | ||
| The file under MIT copyright license https://github.com/openai/tiktoken/blob/main/LICENSE | ||
| In the CompressFile task above we modify the file's content to elimenate the ranks, thus reducing the file size, | ||
| since the rank corresponds to the line number in the file. For the file p50k_base.tiktoken, | ||
| we introduce empty lines to replace any missing ranks, ensuring that the rank consistently aligns with the line number. | ||
| After we eleminate the ranks from the file, we compress the file using the DeflateStream and embed it as a resource in the assembly. | ||
| --> | ||
| <TokenizerDataEmbeddedResource Include="Data\gpt2.tiktoken" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj"/> | ||
| </ItemGroup> | ||
|
|
||
| <Import Project="$(RepositoryEngineeringDir)TokenizerData.targets" /> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| ## About | ||
|
|
||
| The `Microsoft.ML.Tokenizers.Data.Gpt2` includes the Tiktoken tokenizer data file gpt2.tiktoken, which is utilized by models such as `Gpt-2`. | ||
|
|
||
| ## Key Features | ||
|
|
||
| * This package mainly contains the gpt2.tiktoken file, which is used by the Tiktoken tokenizer. This data file is used by the Gpt-2 model. | ||
|
|
||
| ## How to Use | ||
|
|
||
| Reference this package in your project to use the Tiktoken tokenizer with the specified model. | ||
|
|
||
| ```csharp | ||
|
|
||
| // Create a tokenizer for the specified model | ||
| Tokenizer tokenizer = TiktokenTokenizer.CreateForModel("Gpt-2"); | ||
|
|
||
| ``` | ||
|
|
||
| ## Main Types | ||
|
|
||
| Users shouldn't use any types exposed by this package directly. This package is intended to provide tokenizer data files. | ||
|
|
||
| ## Additional Documentation | ||
|
|
||
| * [API documentation](https://learn.microsoft.com/en-us/dotnet/api/microsoft.ml.tokenizers) | ||
|
|
||
| ## Related Packages | ||
|
|
||
| <!-- The related packages associated with this package --> | ||
| Microsoft.ML.Tokenizers | ||
|
|
||
| ## Feedback & Contributing | ||
|
|
||
| Microsoft.ML.Tokenizers.Data.Gpt2 is released as open source under the [MIT license](https://licenses.nuget.org/MIT). Bug reports and contributions are welcome at [the GitHub repository](https://github.com/dotnet/machinelearning). |
File renamed without changes.
31 changes: 31 additions & 0 deletions
31
src/Microsoft.ML.Tokenizers.Data.O200kBase/Microsoft.ML.Tokenizers.Data.O200kBase.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>netstandard2.0</TargetFramework> | ||
| <Nullable>enable</Nullable> | ||
| <IsPackable>true</IsPackable> | ||
| <PackageDescription>The Microsoft.ML.Tokenizers.Data.O200kBase includes the Tiktoken tokenizer data file o200k_base.tiktoken, which is utilized by models such as gpt-4o.</PackageDescription> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <!-- | ||
| The following file are compressed using the DeflateStream and embedded as resources in the assembly. | ||
| The files are downloaded from the following sources and compressed to the Destination. | ||
| - o200k_base.tiktoken https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken | ||
| The file under MIT copyright license https://github.com/openai/tiktoken/blob/main/LICENSE | ||
| In the CompressFile task above we modify the file's content to elimenate the ranks, thus reducing the file size, | ||
| since the rank corresponds to the line number in the file. For the file p50k_base.tiktoken, | ||
| we introduce empty lines to replace any missing ranks, ensuring that the rank consistently aligns with the line number. | ||
| After we eleminate the ranks from the file, we compress the file using the DeflateStream and embed it as a resource in the assembly. | ||
| --> | ||
| <TokenizerDataEmbeddedResource Include="Data\o200k_base.tiktoken" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\Microsoft.ML.Tokenizers\Microsoft.ML.Tokenizers.csproj"/> | ||
| </ItemGroup> | ||
|
|
||
| <Import Project="$(RepositoryEngineeringDir)TokenizerData.targets" /> | ||
| </Project> |
13 changes: 13 additions & 0 deletions
13
src/Microsoft.ML.Tokenizers.Data.O200kBase/O200kBaseTokenizerData.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
|
|
||
| namespace Microsoft.ML.Tokenizers | ||
| { | ||
| /// <summary> | ||
| /// O200kBaseTokenizerData is internally used by Microsoft.ML.Tokenizers library to bind to the o200k_base.tiktoken data file. | ||
| /// </summary> | ||
| internal sealed class O200kBaseTokenizerData | ||
| { | ||
| } | ||
| } |
Oops, something went wrong.