test(tokenizer): byte-level is not plain byte-pair-encoding (#23)

hyperonym · Mar 9, 2023 · 902f3bd · 902f3bd
1 parent b3cdb09
commit 902f3bd
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
@@ -9,8 +9,8 @@
 class TestTokenizer:
     """Test stateful tokenizer for stream decoding."""
 
-    def test_byte_pair_encoding(self):
-        """Test with Byte-Pair-Encoding."""
+    def test_byte_level_byte_pair_encoding(self):
+        """Test with byte-level Byte-Pair-Encoding."""
         tokenizer = AutoTokenizer.from_pretrained(
             pretrained_model_name_or_path="./tests/data/tiny-random-bloom",
             local_files_only=True,