Fix for getting tokenizer in character-based ASR models when using ta…

…rred dataset (NVIDIA#5442) Signed-off-by: Jonghwan Hyeon <[email protected]> Signed-off-by: Jonghwan Hyeon <[email protected]> Signed-off-by: Hainan Xu <[email protected]>
hainan-xv · Nov 29, 2022 · 6f7792a · 6f7792a
1 parent 92fa8cb
commit 6f7792a
Show file tree

Hide file tree

Showing 2 changed files with 0 additions and 4 deletions.
diff --git a/nemo/collections/asr/models/ctc_models.py b/nemo/collections/asr/models/ctc_models.py
@@ -385,7 +385,6 @@ def _setup_dataloader_from_config(self, config: Optional[Dict]):
             if is_concat:
                 dataset = audio_to_text_dataset.get_concat_tarred_dataset(
                     config=config,
-                    tokenizer=self.tokenizer,
                     shuffle_n=shuffle_n,
                     global_rank=self.global_rank,
                     world_size=self.world_size,
@@ -394,7 +393,6 @@ def _setup_dataloader_from_config(self, config: Optional[Dict]):
             else:
                 dataset = audio_to_text_dataset.get_tarred_dataset(
                     config=config,
-                    tokenizer=self.tokenizer,
                     shuffle_n=shuffle_n,
                     global_rank=self.global_rank,
                     world_size=self.world_size,

diff --git a/nemo/collections/asr/models/rnnt_models.py b/nemo/collections/asr/models/rnnt_models.py
@@ -497,7 +497,6 @@ def _setup_dataloader_from_config(self, config: Optional[Dict]):
             if is_concat:
                 dataset = audio_to_text_dataset.get_concat_tarred_dataset(
                     config=config,
-                    tokenizer=self.tokenizer,
                     shuffle_n=shuffle_n,
                     global_rank=self.global_rank,
                     world_size=self.world_size,
@@ -506,7 +505,6 @@ def _setup_dataloader_from_config(self, config: Optional[Dict]):
             else:
                 dataset = audio_to_text_dataset.get_tarred_dataset(
                     config=config,
-                    tokenizer=self.tokenizer,
                     shuffle_n=shuffle_n,
                     global_rank=self.global_rank,
                     world_size=self.world_size,