From 72e9bd23250811083e8b2a37fd6143779d85cc51 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 27 Oct 2023 09:35:43 +0200 Subject: [PATCH] nits --- src/transformers/tokenization_utils_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 3d4bae494990..5fd83fe20f4a 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -2230,7 +2230,7 @@ def _from_pretrained( # allows converting a fast -> slow: add the `tokenizer.json`'s `"added_tokens"` to the slow tokenizer # if `tokenizer_config.json` is `None` - if "Fast" not in cls.__name__ and tokenizer_file is not None: + if tokenizer_file is not None: # This is for slow so can be done before with open(tokenizer_file, encoding="utf-8") as tokenizer_file_handle: tokenizer_file_handle = json.load(tokenizer_file_handle)