Improve Command and Command not run events

kadi/commands/command_sets.py

@@ -1,9 +1,9 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
-import re
 from pathlib import Path
 
 import astropy.units as u
 from cxotime import CxoTime
+from parse_cm.backstop import parse_backstop_params
 from Quaternion import Quat
 from ska_helpers.utils import convert_to_int_float_str
 
@@ -173,22 +173,17 @@ def cmd_set_hrc_not_run(load_name, date=None):
 
 
 def cmd_set_command(*args, date=None):
-    params_str = args[0]
-    cmd_type, args_str = params_str.split("|", 1)
-    cmd = {"type": cmd_type.strip().upper()}
+    """Parse Command or Command not run params string and return a command dict.
 
-    # Strip spaces around equals signs and uppercase args (note that later the
-    # keys are lowercased).
-    args_str = re.sub(r"\s*=\s*", "=", args_str).upper()
+    The format follows Backstop ``"<cmd_type> | PARAM1=VAL1, PARAM2=VAL2, .."``.
+    This code follows the key steps in parse_cm.backstop.read_backstop_as_list().
+    """
+    params_str = args[0].strip().replace(" ", "").upper()
 
-    params = {}
-    for param in args_str.split():
-        key, val = param.split("=")
-        if key == "TLMSID":
-            cmd["tlmsid"] = val
-        else:
-            params[key] = convert_to_int_float_str(val)
-    cmd["params"] = params
+    cmd_type, args_str = params_str.split("|", 1)
+    params = parse_backstop_params(args_str)
+    tlmsid = params.pop("tlmsid", "None")
+    cmd = {"type": cmd_type, "tlmsid": tlmsid, "params": params}
 
     return (cmd,)
 
@@ -204,6 +199,8 @@ def cmd_set_end_scs(*args, date=None):
 
 def cmd_set_command_not_run(*args, date=None):
     (cmd,) = cmd_set_command(*args, date=date)
+    # Save original type which gets used later in CommandTable.remove_not_run_cmds().
+    cmd["params"]["__type__"] = cmd["type"]
     cmd["type"] = "NOT_RUN"
     return (cmd,)
 

kadi/commands/commands_v2.py

@@ -402,7 +402,7 @@ def update_archive_and_get_cmds_recent(
     start = CxoTime(min(loads["cmd_start"]))
     stop = CxoTime(max(loads["cmd_stop"]))
     # Allow for variations in input format of date
-    dates = np.array([CxoTime(date).date for date in cmd_events["Date"]])
+    dates = np.array([CxoTime(date).date for date in cmd_events["Date"]], dtype=str)
     bad = (dates < (start - 14 * u.day).date) | (dates > stop.date)
     cmd_events = cmd_events[~bad]
     cmd_events_ids = [evt["Event"] + " at " + evt["Date"] for evt in cmd_events]

kadi/commands/core.py

@@ -868,15 +868,44 @@ def remove_not_run_cmds(self):
         the "Command not run" event in the Command Events sheet, e.g. the
         LETG retract command in the loads after the LETG insert anomaly.
         """
-        idxs_remove = set()
         idxs_not_run = np.where(self["type"] == "NOT_RUN")[0]
+        if len(idxs_not_run) == 0:
+            return
+
+        idxs_remove = set()
         for idx in idxs_not_run:
-            cmd = self[idx]
-            ok = (self["date"] == cmd["date"]) & (self["tlmsid"] == cmd["tlmsid"])
-            idxs_remove.update(np.where(ok)[0])
-        if idxs_remove:
-            logger.info(f"Removing {len(idxs_remove)} NOT_RUN cmds")
-            self.remove_rows(list(idxs_remove))
+            cmd_not_run = self[idx]
+            ok = (
+                (self["date"] == cmd_not_run["date"])
+                & (self["type"] == cmd_not_run["__type__"])
+                & (self["tlmsid"] == cmd_not_run["tlmsid"])
+            )
+            for key in ("scs", "step"):
+                if cmd_not_run[key] != 0:
+                    ok &= self[key] == cmd_not_run[key]
+
+            # Indexes of self commands that *might* match cmd_not_run.
+            idxs = np.arange(len(self))[ok].tolist()
+
+            # Now check that the params match.
+            idxs_match = []
+            for idx in idxs:
+                # Get the intersection of the keys in cmd_not_run["params"] and self["params"][idx]
+                self_params = self["params"][idx]
+                cmd_not_run_params = cmd_not_run["params"]
+                keys = set(cmd_not_run_params) & set(self_params)
+
+                # Check that the values match for all common keys
+                match = all(cmd_not_run_params[key] == self_params[key] for key in keys)
+                if match:
+                    idxs_match.append(idx)
+
+            idxs_remove.update(idxs_match)
+
+        logger.info(f"Removing {len(idxs_remove)} NOT_RUN cmds")
+        for idx in sorted(idxs_remove, reverse=True):
+            logger.debug(f"  {self[idx]}")
+        self.remove_rows(list(idxs_remove) + list(idxs_not_run))
 
 
 def get_par_idx_update_pars_dict(pars_dict, cmd, params=None, rev_pars_dict=None):

kadi/commands/tests/test_commands.py

@@ -892,9 +892,9 @@ def test_get_starcats_as_table():
 @pytest.mark.parametrize(
     "par_str",
     [
-        "ACISPKT|  TLmSID= aa0000000 par1 = 1    par2=-1.0",
-        "AcisPKT|TLmSID=AA0000000 par1=1 par2=-1.0",
-        "ACISPKT|  TLmSID = aa0000000  par1  =1    par2 =  -1.0",
+        "ACISPKT|  TLmSID= aa0000000, par1 = 1 ,   par2=-1.0",
+        "AcisPKT|TLmSID=AA0000000 ,par1=1, par2=-1.0",
+        "ACISPKT|  TLmSID = aa0000000 , par1  =1,    par2 =  -1.0",
     ],
 )
 def test_get_cmds_from_event_case(par_str):
@@ -914,8 +914,8 @@ def test_get_cmds_from_event_case(par_str):
     Event,Params
     Observing not run,FEB1422A
     Load not run,OCT2521A
-    Command,ACISPKT | TLMSID=AA00000000
-    Command not run,COMMAND_SW | TLMSID=4OHETGIN
+    Command,"ACISPKT | TLMSID= AA00000000, CMDS= 3, WORDS= 3, PACKET(40)= D80000300030603001300"
+    Command not run,"COMMAND_SW | TLMSID=4OHETGIN, HEX= 8050300, MSID= 4OHETGIN"
     RTS,"RTSLOAD,1_4_CTI,NUM_HOURS=39:00:00,SCS_NUM=135"
     Obsid,65527
     Maneuver,0.70546907 0.32988307 0.53440901 0.32847766
@@ -934,10 +934,10 @@ def test_get_cmds_from_event_case(par_str):
         "2020:001:00:00:00.000 | LOAD_EVENT       | None       | CMD_EVT  | event=Load_not_run, event_date=2020:001:00:00:00, event_type=LOAD_NOT_RUN, load=OCT2521A, scs=0"  # noqa
     ],
     [
-        "2020:001:00:00:00.000 | ACISPKT          | AA00000000 | CMD_EVT  | event=Command, event_date=2020:001:00:00:00, scs=0"  # noqa
+        "2020:001:00:00:00.000 | ACISPKT          | AA00000000 | CMD_EVT  | event=Command, event_date=2020:001:00:00:00, cmds=3, words=3, scs=0"  # noqa
     ],
     [
-        "2020:001:00:00:00.000 | NOT_RUN          | 4OHETGIN   | CMD_EVT  | event=Command_not_run, event_date=2020:001:00:00:00, scs=0"  # noqa
+        "2020:001:00:00:00.000 | NOT_RUN          | 4OHETGIN   | CMD_EVT  | event=Command_not_run, event_date=2020:001:00:00:00, hex=8050300, msid=4OHETGIN, __type__=COMMAND_SW, scs=0"  # noqa
     ],
     [
         "2020:001:00:00:00.000 | COMMAND_SW       | OORMPEN    | CMD_EVT  | event=RTS,"
@@ -1417,3 +1417,95 @@ def test_hrc_not_run_scenario(stop_date_2023200):  # noqa: ARG001
     assert states_out == states_exp
 
     commands_v2.clear_caches()
+
+
+test_command_not_run_cases = [
+    {
+        # Matches multiple commands
+        "event": {
+            "date": "2023:351:13:30:33.849",
+            "event": "Command not run",
+            "params_str": "COMMAND_SW | TLMSID= COACTSX",
+        },
+        "removed": [3, 4],
+    },
+    {
+        # Matches one command with multiple criteria
+        "event": {
+            "date": "2023:351:13:30:33.849",
+            "event": "Command not run",
+            "params_str": (
+                "COMMAND_SW | TLMSID= COACTSX, HEX= 840B100, "
+                "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 690"
+            ),
+        },
+        "removed": [3],
+    },
+    {
+        # Wrong TLMSID
+        "event": {
+            "date": "2023:351:13:30:33.849",
+            "event": "Command not run",
+            "params_str": (
+                "COMMAND_SW | TLMSID= XXXXXXX, HEX= 840B100, "
+                "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 690"
+            ),
+        },
+        "removed": [],
+    },
+    {
+        # Wrong SCS
+        "event": {
+            "date": "2023:351:13:30:33.849",
+            "event": "Command not run",
+            "params_str": (
+                "COMMAND_SW | TLMSID= XXXXXXX, HEX= 840B100, "
+                "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 133, STEP= 690"
+            ),
+        },
+        "removed": [],
+    },
+    {
+        # Wrong Step
+        "event": {
+            "date": "2023:351:13:30:33.849",
+            "event": "Command not run",
+            "params_str": (
+                "COMMAND_SW | TLMSID= XXXXXXX, HEX= 840B100, "
+                "MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 111"
+            ),
+        },
+        "removed": [],
+    },
+    {
+        # No TLMSID
+        "event": {
+            "date": "2023:351:19:38:41.550",
+            "event": "Command not run",
+            "params_str": "SIMTRANS   | POS= 92904, SCS= 131, STEP= 1191",
+        },
+        "removed": [6],
+    },
+]
+
+
+@pytest.mark.parametrize("case", test_command_not_run_cases)
+def test_command_not_run(case):
+    backstop_text = """
+2023:351:13:30:32.824 | 0 0 | COMMAND_SW | TLMSID= AOMANUVR, HEX= 8034101, MSID= AOMANUVR, SCS= 128, STEP= 686
+2023:351:13:30:33.849 | 1 0 | COMMAND_SW | TLMSID= AOACRSTE, HEX= 8032001, MSID= AOACRSTE, SCS= 128, STEP= 688
+2023:351:13:30:33.849 | 2 0 | COMMAND_SW | TLMSID= COENASX, HEX= 844B100, MSID= COENASX, COENAS1=177 , SCS= 128, STEP= 689
+2023:351:13:30:33.849 | 3 0 | COMMAND_SW | TLMSID= COACTSX, HEX= 840B100, MSID= COACTSX, COACTS1=177 , COACTS2=0 , SCS= 128, STEP= 690
+2023:351:13:30:33.849 | 4 0 | COMMAND_SW | TLMSID= COACTSX, HEX= 8402600, MSID= COACTSX, COACTS1=38 , COACTS2=0 , SCS= 128, STEP= 691
+2023:351:13:30:55.373 | 5 0 | COMMAND_HW | TLMSID= 4MC5AEN, HEX= 4800012, MSID= 4MC5AEN, SCS= 131, STEP= 892
+2023:351:19:38:41.550 | 6 0 | SIMTRANS   | POS= 92904, SCS= 131, STEP= 1191
+    """  # noqa
+    cmds = commands.read_backstop(backstop_text.strip().splitlines())
+    cmds["source"] = "DEC1123A"
+    cmds_exp = cmds.copy()
+    cmds_exp.remove_rows(case["removed"])
+    cmds_from_event = get_cmds_from_event(**case["event"])
+    cmds_with_event = cmds.add_cmds(cmds_from_event)
+    cmds_with_event.sort_in_backstop_order()
+    cmds_with_event.remove_not_run_cmds()
+    assert cmds_with_event.pformat_like_backstop() == cmds_exp.pformat_like_backstop()

utils/make_hrc_disable_events.py

@@ -0,0 +1,129 @@
+"""Create a scenario to handle HRC commands not run to to HRC being disabled.
+
+This removes HRC state-impacting commands::
+
+    {"tlmsid": "224PCAON"},
+    {"tlmsid": "215PCAON"},
+    {"tlmsid": "COACTSX", "coacts1": 134},
+    {"tlmsid": "COENASX", "coenas1": 89},
+    {"tlmsid": "COENASX", "coenas1": 90},
+
+In practice the hardware commands are not in loads since the HRC return to science.
+
+This script creates a scenario ~/.kadi/<scenario> that can be used to remove these
+commands from the flight kadi commands database. It then tests that by setting
+``KADI_SCENARIO=<scenario>`` and getting kadi states over the time period of interest.
+
+The simplest way to do import to the flight sheet is to import the CSV file into a
+temporary Google Sheet, then copy/paste that table into the flight Chandra Command
+Events Google Sheet. Remember to create empty rows for the copy/paste.
+"""
+
+import argparse
+
+import kadi.paths
+from astropy import table
+from cxotime import CxoTime
+from kadi.commands import get_cmds
+from kadi.commands.core import CommandTable
+from kadi.commands.states import get_states
+from ska_helpers.utils import temp_env_var
+
+
+# When was F_HRC_SAFING script run in late 2023
+hrc_safing_date = "2023:343:02:00:00"
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(
+        description="Print HRC state-impacting commands that were not run due to HRC being disabled."
+    )
+    parser.add_argument(
+        "--start",
+        default=hrc_safing_date,
+        help=f"Start time for searching for commands (default={hrc_safing_date}))",
+    )
+    parser.add_argument(
+        "--stop",
+        help="Stop time for searching for commands (default=NOW)",
+    )
+    parser.add_argument(
+        "--status",
+        default="Definitive",
+        help="Status of command events (Definitive or In-work)",
+    )
+    parser.add_argument(
+        "--scenario",
+        default="hrc_disable",
+        help="Scenario name (default=hrc_disable). This creates ~/.kadi/<scenario>/cmd_events.csv.",
+    )
+    return parser
+
+
+def main():
+    opt = get_parser().parse_args()
+    make_cmd_events(opt)
+    test_cmd_events(opt)
+
+
+def test_cmd_events(opt):
+    def get_states_local():
+        states = get_states(
+            start=opt.start,
+            stop=opt.stop,
+            state_keys=["hrc_15v", "hrc_24v", "hrc_i", "hrc_s"],
+            merge_identical=True,
+        )
+        return states
+
+    print("Current flight states:")
+    get_states_local().pprint_all()
+    print()
+    print(f"States with HRC disable scenario {opt.scenario}:")
+    with temp_env_var("KADI_SCENARIO", opt.scenario):
+        get_states_local().pprint_all()
+
+
+def make_cmd_events(opt):
+    cmd_kwargs_list = [
+        {"tlmsid": "224PCAON"},
+        {"tlmsid": "215PCAON"},
+        {"tlmsid": "COACTSX", "coacts1": 134},
+        {"tlmsid": "COENASX", "coenas1": 89},
+        {"tlmsid": "COENASX", "coenas1": 90},
+    ]
+
+    rows = []
+    start = CxoTime(opt.start)
+    stop = CxoTime(opt.stop)
+
+    for cmd_kwargs in cmd_kwargs_list:
+        cmds: CommandTable = get_cmds(start=start, stop=stop, **cmd_kwargs)
+        print(f"{len(cmds)} cmd(s) found for {cmd_kwargs}")
+        params_str = ", ".join([f"{k.upper()}={v}" for k, v in cmd_kwargs.items()])
+        for cmd in cmds:
+            row = (
+                opt.status,
+                cmd["date"],
+                "Command not run",
+                f"{cmd['type']} | {params_str}",
+                "Tom Aldcroft",
+                "Jean Connelly",
+                f"Not run due to F_HRC_SAFING at {hrc_safing_date}",
+            )
+            rows.append(row)
+
+    names = "State Date Event Params Author Reviewer Comment".split()
+    cmd_events = table.Table(rows=rows, names=names)
+    cmd_events.sort("Date", reverse=True)
+    cmd_events.pprint_all()
+
+    cmd_events_path = kadi.paths.CMD_EVENTS_PATH(opt.scenario)
+    cmd_events_path.parent.mkdir(parents=True, exist_ok=True)
+    print()
+    print(f"Writing {len(cmd_events)} events to {cmd_events_path}")
+    cmd_events.write(cmd_events_path, format="ascii.csv", overwrite=True)
+
+
+if __name__ == "__main__":
+    main()