py : improve BPE tokenizer support (#5189)

ggerganov · Jan 29, 2024 · e76627b · e76627b
1 parent fbe7dfa
commit e76627b
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/convert.py b/convert.py
@@ -334,7 +334,10 @@ def load(model_plus: ModelPlus) -> Params:
 class BpeVocab:
     def __init__(self, fname_tokenizer: Path, fname_added_tokens: Path | None) -> None:
         self.bpe_tokenizer = json.loads(open(str(fname_tokenizer), encoding="utf-8").read())
-        self.vocab = self.bpe_tokenizer["model"]["vocab"]
+        try:
+            self.vocab = self.bpe_tokenizer["model"]["vocab"]
+        except:
+            self.vocab = self.bpe_tokenizer
         added_tokens: dict[str, int]
         if fname_added_tokens is not None:
             # FIXME: Verify that added tokens here _cannot_ overlap with the main vocab.