undo merge error (#1325)

EleutherAI · Nov 27, 2024 · f532580 · f532580
1 parent a8f7913
commit f532580
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 4 deletions.
diff --git a/megatron/model/transformer.py b/megatron/model/transformer.py
@@ -1248,9 +1248,8 @@ def forward(self, x, attention_mask, layer_past=None):
                     raise KeyError(self.moe_type)
 
             with torch.enable_grad() if not self.eval else nullcontext():
-                if (
-                    mlp_bias == None,
-                    self.num_experts > 1 and self.moe_type == "deepspeed",
+                if mlp_bias == None or (
+                    self.num_experts > 1 and self.moe_type == "deepspeed"
                 ):
                     # No dropout either
                     assert mlp_bias is None

diff --git a/megatron/neox_arguments/arguments.py b/megatron/neox_arguments/arguments.py
@@ -962,7 +962,7 @@ def calculate_derived(self):
             else:
                 fp16_conflict = "DeepSpeed fp16 field was set but precision conflicts"
                 assert self.precision == "fp16", fp16_conflict
-        
+
         if self.bf16 and self.bf16.get("enabled", False):
             if self.precision is None:
                 self.update_value("precision", "bfloat16")