Partial #6914 job_agent separated terminating vs destroying (#7337)

* Partial #6914 job_agent separated terminating vs destroying
- does not cancel the sbatch job when terminating
- job_agent _SBATCH_ID_FILE write
- job_supervisor concept of verify_status but doesn't change semantics yet
- pkcli.elegant-schema better approach to updating schema
- const.DEV_SRC_RADIASOFT_DIR
- _must_verify_status needs to be global
- _ComputeJob.is_destroyed unused

etc/schema-import/elegant/README

@@ -0,0 +1,2 @@
+Run:
+sirepo elegant-schema parse-manual

sirepo/const.py

@@ -1,11 +1,12 @@
-# -*- coding: utf-8 -*-
 """Constant values
 
 :copyright: Copyright (c) 2021 RadiaSoft LLC.  All Rights Reserved.
 :license: http://www.apache.org/licenses/LICENSE-2.0.html
 """
+
 import asyncio
 from pykern.pkcollections import PKDict
+import pykern.pkio
 
 ASYNC_CANCELED_ERROR = asyncio.CancelledError
 
@@ -53,3 +54,6 @@
 SRUNIT_USER_AGENT = "srunit/1.0"
 
 TEST_PORT_RANGE = range(10000, 11000)
+
+#: hardwired root of development src tree
+DEV_SRC_RADIASOFT_DIR = pykern.pkio.py_path("~/src/radiasoft/")

sirepo/job.py

@@ -29,13 +29,13 @@
 OP_RUN = "run"
 OP_SBATCH_AGENT_READY = "sbatch_agent_ready"
 OP_SBATCH_LOGIN = "sbatch_login"
+OP_VERIFY_STATUS = "verify_status"
 OP_BEGIN_SESSION = "begin_session"
 
-
 #: Types of slots required by op types
 CPU_SLOT_OPS = frozenset((OP_ANALYSIS, OP_RUN))
-SLOT_OPS = frozenset().union(*[CPU_SLOT_OPS, (OP_IO,)])
-
+#: All ops that have slots (see job_driver.DriverBase._slots_ready)
+SLOT_OPS = frozenset().union(*[CPU_SLOT_OPS, (OP_IO, OP_VERIFY_STATUS)])
 
 _OK_REPLY = PKDict(state="ok")
 

sirepo/job_driver/__init__.py

@@ -395,11 +395,11 @@ async def _alloc_check(alloc, msg):
         if n == job.OP_SBATCH_LOGIN:
             return res
         await _alloc_check(
-            op.op_slot.alloc, "Waiting for another simulation to complete await=op_slot"
+            op.op_slot.alloc, "Waiting for another sim op to complete await=op_slot"
         )
         await _alloc_check(
             op.run_dir_slot.alloc,
-            "Waiting for access to simulation state await=run_dir_slot",
+            "Waiting for access to sim state await=run_dir_slot",
         )
         if n not in job.CPU_SLOT_OPS:
             return res

sirepo/job_driver/sbatch.py

@@ -1,10 +1,9 @@
-# -*- coding: utf-8 -*-
 """TODO(e-carlin): Doc
 
 :copyright: Copyright (c) 2019 RadiaSoft LLC.  All Rights Reserved.
 :license: http://www.apache.org/licenses/LICENSE-2.0.html
 """
-from __future__ import absolute_import, division, print_function
+
 from pykern import pkconfig
 from pykern import pkio
 from pykern import pkjson
@@ -16,6 +15,7 @@
 import asyncssh
 import datetime
 import errno
+import sirepo.const
 import sirepo.job_supervisor
 import sirepo.simulation_db
 import sirepo.util
@@ -232,10 +232,13 @@ def _agent_start_dev(self):
 scancel -u $USER >& /dev/null || true
 """
         if self.cfg.shifter_image:
-            res += """
-(cd ~/src/radiasoft/sirepo && git pull -q) || true
-(cd ~/src/radiasoft/pykern && git pull -q) || true
-"""
+            res += (
+                "\n".join(
+                    f"(cd {sirepo.const.DEV_SRC_RADIASOFT_DIR.join(p)} && git pull -q || true)"
+                    for p in ("pykern", "sirepo")
+                )
+                + "\n"
+            )
         return res
 
     def _raise_sbatch_login_srexception(self, reason, msg):