cancels entire series if there's a scheduler error (#399)

* cancels entire series if there's a scheduler error * includes test id in output error * replaced by_sigterm parameter with message Co-authored-by: Paul Ferrell <[email protected]>
hpc · Apr 6, 2021 · 07e9ebe · 07e9ebe
1 parent dd6ec47
commit 07e9ebe
Showing 1 changed file with 5 additions and 5 deletions.
diff --git a/lib/pavilion/series.py b/lib/pavilion/series.py
@@ -539,12 +539,12 @@ def run_tests(self, wait: Union[None, int] = None,
             try:
                 sched.schedule_tests(self.pav_cfg, [test])
             except schedulers.SchedulerPluginError as err:
-                fprint('Error scheduling test: ', file=self.errfile,
+                fprint('Error scheduling test: ', test.id, file=self.errfile,
                        color=output.RED)
                 fprint(err, bullet='  ', file=self.errfile)
                 fprint('Cancelling already kicked off tests.',
                        file=self.errfile)
-                sched.cancel_job(test)
+                self.cancel_series(message="Killed because of scheduler error.")
                 return errno.EINVAL
 
         # Tests should all be scheduled now, and have the SCHEDULED state
@@ -684,15 +684,15 @@ def create_set_graph(self):
 
         return
 
-    def cancel_series(self):
+    def cancel_series(self, message=None):
         """Goes through all test objects assigned to series and cancels tests
         that haven't been completed. """
 
         for test_id, test_obj in self.tests.items():
             if not (test_obj.path/'RUN_COMPLETE').exists():
                 sched = schedulers.get_plugin(test_obj.scheduler)
                 sched.cancel_job(test_obj)
-                test_obj.status.set(STATES.COMPLETE, "Killed by SIGTERM.")
+                test_obj.status.set(STATES.COMPLETE, message)
                 test_obj.set_run_complete()
 
     def run_series(self):
@@ -702,7 +702,7 @@ def run_series(self):
         def sigterm_handler(_signals, _frame_type):
             """Calls cancel_series and exists."""
 
-            self.cancel_series()
+            self.cancel_series(message="Series killed by SIGTERM.")
             sys.exit()
 
         signal.signal(signal.SIGTERM, sigterm_handler)