[pipe] prevent deadlock with multiple evals sequence (#1944)

* [pipe] prevent deadlock with multiple evals sequence * style * style * style * align DSE commit w. latest master Co-authored-by: Jeff Rasley <[email protected]> Co-authored-by: Olatunji Ruwase <[email protected]>
microsoft · May 10, 2022 · dbeadf1 · dbeadf1
1 parent de88718
commit dbeadf1
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/deepspeed/runtime/pipe/engine.py b/deepspeed/runtime/pipe/engine.py
@@ -445,6 +445,10 @@ def eval_batch(self,
         sched = schedule.InferenceSchedule(micro_batches=self.micro_batches,
                                            stages=self.num_stages,
                                            stage_id=self.stage_id)
+
+        # prevent dead-lock with multiple evals sequence
+        dist.barrier()
+
         with torch.no_grad():
             self._exec_schedule(sched)