update default LN eps to match latest OLMo model (#58)

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Changed
+
+- Updated default layer norm epsilon for OLMo models from `1e-5` to `1e-6` to match latest model.
+
 ## [v1.3.2](https://github.com/allenai/OLMo-core/releases/tag/v1.3.2) - 2024-09-27
 
 ### Added

src/olmo_core/nn/transformer/model.py

@@ -294,6 +294,7 @@ def olmo_1B(cls, vocab_size: int, **kwargs) -> "TransformerConfig":
             block_name=kwargs.pop("block_name", TransformerBlockType.reordered_norm),
             qk_norm=kwargs.pop("qk_norm", True),
             rope_theta=kwargs.pop("rope_theta", 500_000),
+            layer_norm_eps=1e-6,
             **kwargs,
         )
 
@@ -307,6 +308,7 @@ def olmo_7B(cls, vocab_size: int, **kwargs) -> "TransformerConfig":
             block_name=kwargs.pop("block_name", TransformerBlockType.reordered_norm),
             qk_norm=kwargs.pop("qk_norm", True),
             rope_theta=kwargs.pop("rope_theta", 500_000),
+            layer_norm_eps=1e-6,
             **kwargs,
         )
 
@@ -320,6 +322,7 @@ def olmo_13B(cls, vocab_size: int, **kwargs) -> "TransformerConfig":
             block_name=kwargs.pop("block_name", TransformerBlockType.reordered_norm),
             qk_norm=kwargs.pop("qk_norm", True),
             rope_theta=kwargs.pop("rope_theta", 500_000),
+            layer_norm_eps=1e-6,
             **kwargs,
         )
 
@@ -475,6 +478,7 @@ def llama_like(
         n_heads: int,
         n_kv_heads: Optional[int] = None,
         qk_norm: bool = False,
+        layer_norm_eps: float = 1e-5,
         rope_theta: int = 500_000,
         rope_type: Optional[RoPEType] = None,
         hidden_size_multiple_of: int = 256,
@@ -513,7 +517,7 @@ def llama_like(
         # Configure global layer norm.
         layer_norm = LayerNormConfig(
             name=LayerNormType.fused_rms if fused_ops else LayerNormType.rms,
-            eps=1e-5,
+            eps=layer_norm_eps,
             bias=False,
             dtype=dtype,
         )