fix path (NVIDIA#11121)

Signed-off-by: Chen Cui <[email protected]>
paarthneekhara · Jan 18, 2025 · 05cb0d8 · 05cb0d8
1 parent 26bf509
commit 05cb0d8
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/nemo/collections/llm/gpt/data/fine_tuning.py b/nemo/collections/llm/gpt/data/fine_tuning.py
@@ -212,7 +212,10 @@ def _extract_tokenizer_model_name(self) -> str:
             tokenizer_model_name = self.packed_sequence_specs.tokenizer_model_name
         elif isinstance(self.tokenizer, AutoTokenizer):
             name = self.tokenizer.tokenizer.name_or_path
-            if name.endswith("nemo_tokenizer"):
+            if name.endswith("context/nemo_tokenizer"):
+                # NEMO_HOME/hf_org/hf_model/context/nemo_tokenizer => hf_org--hf_model
+                tokenizer_model_name = '--'.join(name.split("/")[-4:-2])
+            elif name.endswith("nemo_tokenizer"):
                 # NEMO_HOME/hf_org/hf_model/nemo_tokenizer => hf_org--hf_model
                 tokenizer_model_name = '--'.join(name.split("/")[-3:-1])
             else: