Skip to content

Commit

Permalink
Not allow shutdown while still jobs running. (NVIDIA#454)
Browse files Browse the repository at this point in the history
* Not allow shutdown while still jobs running.

* Added protection for restart.
  • Loading branch information
yhwen authored Apr 28, 2022
1 parent e0aa543 commit 49bc48c
Showing 1 changed file with 8 additions and 0 deletions.
8 changes: 8 additions & 0 deletions nvflare/private/fed/server/training_cmds.py
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,10 @@ def shutdown(self, conn: Connection, args: List[str]):
if not isinstance(engine, ServerEngineInternalSpec):
raise TypeError("engine must be ServerEngineInternalSpec but got {}".format(type(engine)))

if engine.job_runner.running_jobs:
conn.append_error("There are still jobs running. Please let them finish or abort_job before shutdown.")
return

if target_type == self.TARGET_TYPE_SERVER:
if engine.get_clients():
conn.append_error("There are still active clients. Shutdown all clients first.")
Expand Down Expand Up @@ -516,6 +520,10 @@ def restart(self, conn: Connection, args: List[str]):
if not isinstance(engine, ServerEngineInternalSpec):
raise TypeError("engine must be ServerEngineInternalSpec but got {}".format(type(engine)))

if engine.job_runner.running_jobs:
conn.append_error("There are still jobs running. Please let them finish or abort_job before restart.")
return

target_type = args[1]
if target_type == self.TARGET_TYPE_SERVER or target_type == self.TARGET_TYPE_ALL:

Expand Down

0 comments on commit 49bc48c

Please sign in to comment.