[tokenizer] Updates tokenizer to 0.20.0 (#3452)

deepjavalibrary · Sep 4, 2024 · 581f1dd · 581f1dd
1 parent f204abc
commit 581f1dd
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 2 deletions.
diff --git a/extensions/tokenizers/rust/Cargo.toml b/extensions/tokenizers/rust/Cargo.toml
@@ -27,6 +27,8 @@ candle = { git = "https://github.com/huggingface/candle", rev = "f76bb7794aa8659
 candle-nn = { git = "https://github.com/huggingface/candle", rev = "f76bb7794aa8659c5023797979a3392cdfc01f32", package = "candle-nn" }
 candle-transformers = { git = "https://github.com/huggingface/candle", rev = "f76bb7794aa8659c5023797979a3392cdfc01f32", package = "candle-transformers" }
 candle-flash-attn = { git = "https://github.com/huggingface/candle", rev = "f76bb7794aa8659c5023797979a3392cdfc01f32", package = "candle-flash-attn" }
+# openssl-src 300.3.2 is broken for AL2
+openssl-src = { git = "https://github.com/alexcrichton/openssl-src-rs", rev = "c591de676017446c9f1d719e77eb162159ce8886", package = "openssl-src" }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 openssl = { version = "0.10", features = ["vendored"] }

diff --git a/...ions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java b/...ions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
@@ -38,7 +38,7 @@ public class HuggingFaceTokenizerTest {
     public void testVersion() {
         try (HuggingFaceTokenizer tokenizer = HuggingFaceTokenizer.newInstance("bert-base-cased")) {
             String djlVersion = Engine.getDjlVersion();
-            Assert.assertEquals(tokenizer.getVersion(), "0.19.1-" + djlVersion);
+            Assert.assertEquals(tokenizer.getVersion(), "0.20.0-" + djlVersion);
         }
     }
 

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -14,7 +14,7 @@ lightgbm = "3.2.110"
 tensorrt = "8.4.1"
 fasttext = "0.9.2"
 sentencepiece = "0.2.0"
-tokenizers = "0.19.1"
+tokenizers = "0.20.0"
 
 slf4j = "2.0.13"
 log4j_slf4j = "2.23.1"