increase batch size for 13B

allenai · Sep 25, 2024 · 7e3ddd4 · 7e3ddd4
1 parent 84b72c9
commit 7e3ddd4
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/scripts/train/OLMo-13B.py b/src/scripts/train/OLMo-13B.py
@@ -42,7 +42,7 @@ def build_trainer_config(common: CommonComponents) -> TrainerConfig:
     return (
         TrainerConfig(
             save_folder=common.save_folder,
-            global_batch_size=1024 * 4096,
+            global_batch_size=2048 * 4096,
             rank_microbatch_size=1 * 4096,
             autocast_precision=DType.bfloat16,
             save_overwrite=True,