separated init of tokenizer

CHANGELOG.md

@@ -1,3 +1,7 @@
+v1.0.2rc
+-------
+
+- uses separated tokenizer_path to init tokenizer in T5Embedder
 
 v1.0.1
 ------

README.md

@@ -30,7 +30,7 @@ We introduce DeepFloyd IF, a novel state-of-the-art open-source text-to-image mo
 [![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/DeepFloyd/IF)
 
 ```shell
-pip install deepfloyd_if==1.0.1
+pip install deepfloyd_if==1.0.2rc0
 pip install xformers==0.0.16
 pip install git+https://github.com/openai/CLIP.git --no-deps
 ```

deepfloyd_if/__init__.py

@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
 
 
-__version__ = '1.0.1'
+__version__ = '1.0.2rc0'

deepfloyd_if/modules/t5.py

@@ -62,7 +62,7 @@ def __init__(self, device, dir_or_name='t5-v1_1-xxl', *, cache_dir=None, hf_toke
         self.cache_dir = cache_dir or os.path.expanduser('~/.cache/IF_')
         self.dir_or_name = dir_or_name
 
-        path = dir_or_name
+        tokenizer_path, path = dir_or_name, dir_or_name
         if dir_or_name in self.available_models:
             cache_dir = os.path.join(self.cache_dir, dir_or_name)
             for filename in [
@@ -71,9 +71,17 @@ def __init__(self, device, dir_or_name='t5-v1_1-xxl', *, cache_dir=None, hf_toke
             ]:
                 hf_hub_download(repo_id=f'DeepFloyd/{dir_or_name}', filename=filename, cache_dir=cache_dir,
                                 force_filename=filename, token=self.hf_token)
-            path = cache_dir
+            tokenizer_path, path = cache_dir, cache_dir
+        else:
+            cache_dir = os.path.join(self.cache_dir, 't5-v1_1-xxl')
+            for filename in [
+                'config.json', 'special_tokens_map.json', 'spiece.model', 'tokenizer_config.json',
+            ]:
+                hf_hub_download(repo_id='DeepFloyd/t5-v1_1-xxl', filename=filename, cache_dir=cache_dir,
+                                force_filename=filename, token=self.hf_token)
+            tokenizer_path = cache_dir
 
-        self.tokenizer = AutoTokenizer.from_pretrained(path)
+        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
         self.model = T5EncoderModel.from_pretrained(path, **t5_model_kwargs).eval()
 
     def get_text_embeddings(self, texts):