Updated StreamingTextDataloader to pass take in shuffle_block_size (#503

)
mosaicml · Aug 3, 2023 · 358e064 · 358e064
1 parent b4e27c0
commit 358e064
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/llmfoundry/data/text_data.py b/llmfoundry/data/text_data.py
@@ -129,6 +129,7 @@ def __init__(self,
             shuffle=shuffle,
             shuffle_algo=shuffle_algo,
             shuffle_seed=shuffle_seed,
+            shuffle_block_size=shuffle_block_size,
         )
         self.tokenizer = tokenizer
         self.max_seq_len = max_seq_len