disable bprop reduce for cpl when SP is enabled

Signed-off-by: Anmol Gupta <[email protected]>
anmolgupt · Apr 11, 2024 · 23c8ced · 23c8ced
1 parent 27d8b20
commit 23c8ced
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/nemo/collections/nlp/modules/common/megatron/adapters/parallel_adapters.py b/nemo/collections/nlp/modules/common/megatron/adapters/parallel_adapters.py
@@ -187,6 +187,7 @@ def __init__(
                 bias=False,
                 gather_output=True,
                 init_method=self._get_init_fn(column_init_method),
+                disable_grad_reduce=self._sequence_parallel,
             )
         if gather_output:
             self.linear_out = RowParallelLinear(