Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix bug in OBS obs_stop for cmd_evt maneuver and improve default_stop handling #319

Merged
merged 4 commits into from
Feb 15, 2024
Merged

Fix bug in OBS obs_stop for cmd_evt maneuver and improve default_stop handling #319

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
33fb607
Fix a bug in OBS obs_stop for last cmd_evt maneuver
taldcroft Feb 9, 2024
b31e454
Apply KADI_COMMANDS_DEFAULT_STOP to command events
taldcroft Feb 10, 2024
3058819
Script and log output for validation
taldcroft Feb 11, 2024
3301432
Fix test failures
taldcroft Feb 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 51 additions & 18 deletions kadi/commands/commands_v2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -981,6 +981,9 @@ def update_cmd_events(scenario=None) -> Table:
ok = np.isin(cmd_events["State"], allowed_states)
cmd_events = cmd_events[ok]

# If KADI_COMMANDS_DEFAULT_STOP is set, filter out events after that date.
cmd_events = filter_cmd_events_default_stop(cmd_events)

logger.info(f"Writing {len(cmd_events)} cmd_events to {cmd_events_path}")
cmd_events.write(cmd_events_path, format="csv", overwrite=True)
return cmd_events
Expand All @@ -1003,6 +1006,24 @@ def get_cmd_events(scenario=None):
cmd_events_path = paths.CMD_EVENTS_PATH(scenario)
logger.info(f"Reading command events {cmd_events_path}")
cmd_events = Table.read(str(cmd_events_path), format="csv", fill_values=[])

# If KADI_COMMANDS_DEFAULT_STOP is set, filter out events after that date.
cmd_events = filter_cmd_events_default_stop(cmd_events)

return cmd_events


def filter_cmd_events_default_stop(cmd_events):
if (stop := os.environ.get("KADI_COMMANDS_DEFAULT_STOP")) is not None:
stop = CxoTime(stop)
# Filter table based on stop date. Need to use CxoTime on each event separately
# because the date format could be inconsistent.
ok = [CxoTime(cmd_event["Date"]).date <= stop.date for cmd_event in cmd_events]
logger.debug(
f"Filtering cmd_events to stop date {stop.date} "
f"({np.count_nonzero(ok)} vs {len(cmd_events)})"
)
cmd_events = cmd_events[ok]
return cmd_events


Expand Down Expand Up @@ -1315,26 +1336,12 @@ def get_matching_block_idx(cmds_arch, cmds_recent):
idx_arch_recent = cmds_arch.find_date(cmds_recent["date"][0])
logger.info("Selecting commands from cmds_arch[{}:]".format(idx_arch_recent))
cmds_arch_recent = cmds_arch[idx_arch_recent:]
cmds_arch_recent.rev_pars_dict = weakref.ref(REV_PARS_DICT)

# Define the column names that specify a complete and unique row
key_names = ("date", "type", "tlmsid", "scs", "step", "source", "vcdu")
arch_vals = get_list_for_matching(cmds_arch_recent)
recent_vals = get_list_for_matching(cmds_recent)

recent_vals = [
tuple(
row[x].decode("ascii") if isinstance(row[x], bytes) else str(row[x])
for x in key_names
)
for row in cmds_arch_recent
]
arch_vals = [
tuple(
row[x].decode("ascii") if isinstance(row[x], bytes) else str(row[x])
for x in key_names
)
for row in cmds_recent
]

diff = difflib.SequenceMatcher(a=recent_vals, b=arch_vals, autojunk=False)
diff = difflib.SequenceMatcher(a=arch_vals, b=recent_vals, autojunk=False)

matching_blocks = diff.get_matching_blocks()
logger.info("Matching blocks for (a) recent commands and (b) existing HDF5")
Expand Down Expand Up @@ -1362,6 +1369,32 @@ def get_matching_block_idx(cmds_arch, cmds_recent):
return idx0_arch, idx0_recent


def get_list_for_matching(cmds: CommandTable) -> list[tuple]:
# Define the column names that specify a complete and unique row
keys = ("date", "type", "tlmsid", "scs", "step", "source", "vcdu")
rows = []
for cmd in cmds:
row = tuple(
cmd[key].decode("ascii") if isinstance(cmd[key], bytes) else str(cmd[key])
for key in keys
)
# Special case for OBS command. The obs_stop param can change during anomaly
# recovery when there is a CMD_EVT maneuver but no subsequent maneuver to define
# obs_stop. In theory we could include these params in the match tuple for every
# command in the matching, but it turns out that (for reasons I don't fully
# understand) the AOSTRCAT yang and zang params are very slightly different at
# floating point precision level, so the matches fail. Since all the other
# commands are not mutable in this way we just apply this for OBS commands.
if cmd["tlmsid"] == "OBS":
row_params = tuple(
(key, cmd["params"][key]) for key in sorted(cmd["params"])
)
row += row_params
rows.append(row)

return rows


TYPE_MAP = ["ACQ", "GUI", "BOT", "FID", "MON"]
IMGSZ_MAP = ["4x4", "6x6", "8x8"]
PAR_MAPS = [
Expand Down
7 changes: 4 additions & 3 deletions kadi/commands/tests/test_commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,15 +364,16 @@ def stop_date_fixture_factory(stop_date):
def stop_date_fixture(monkeypatch):
commands_v2.clear_caches()
monkeypatch.setenv("KADI_COMMANDS_DEFAULT_STOP", stop_date)
cmds_dir = Path(conf.commands_dir) / stop_date
cmds_dir = Path(conf.commands_dir) / CxoTime(stop_date).iso[:9]
with commands_v2.conf.set_temp("commands_dir", str(cmds_dir)):
yield
commands_v2.clear_caches()

return stop_date_fixture


stop_date_2021_10_24 = stop_date_fixture_factory("2021-10-24")
# 2021:297 0300z just after recovery maneuver following 2021:296 NSM
stop_date_2021_10_24 = stop_date_fixture_factory("2021-10-24T03:00:00")
stop_date_2020_12_03 = stop_date_fixture_factory("2020-12-03")


Expand Down Expand Up @@ -527,7 +528,7 @@ def test_cmds_scenario(stop_date_2020_12_03): # noqa: ARG001
# First make the cmd_events.csv file for the scenario
scenario = "test_acis"
cmds_dir = Path(commands_v2.conf.commands_dir) / scenario
cmds_dir.mkdir(exist_ok=True)
cmds_dir.mkdir(exist_ok=True, parents=True)
# Note variation in format of date, since this comes from humans.
cmd_evts_text = """\
Date,Event,Params,Author,Comment
Expand Down
126 changes: 126 additions & 0 deletions validate/pr319/fix-matching-for-obs-cmds.log
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
building file list ... done
./
cmds2.h5
cmds2.pkl

sent 142252954 bytes received 70 bytes 284506048.00 bytes/sec
total size is 142235385 speedup is 1.00
Running out of branch 'fix-matching-for-obs-cmds'
Using kadi dir: fix-matching-for-obs-cmds
kadi file /Users/aldcroft/git/kadi/kadi/__init__.py
Making kadi dir: fix-matching-for-obs-cmds
Syncing archive to state as of 2024-02-04 (about midnight EST)
Running: rsync -av /Users/aldcroft/ska/data/kadi/cmds-from-2024feb04/ fix-matching-for-obs-cmds/
kadi version: 7.8.2.dev29+gb31e454

********************************************************************************
Last 5 observations as of 2024:035:12:00:00 using scenario='flight'
********************************************************************************
obsid obs_start obs_stop simpos manvr_start npnt_enab
----- --------------------- --------------------- ------ --------------------- ---------
29247 2024:037:09:06:17.311 2024:037:11:57:17.060 75624 2024:037:08:26:51.170 True
29255 2024:037:12:36:53.396 2024:037:15:27:53.145 92904 2024:037:11:57:27.311 True
29248 2024:037:16:07:31.012 2024:037:20:55:10.762 75624 2024:037:15:28:03.396 True
29256 2024:037:21:23:18.513 2024:038:00:27:26.696 75624 2024:037:20:55:21.013 True
43722 2024:038:00:56:28.907 2024:038:01:48:23.442 -99616 2024:038:00:27:36.947 True

Updating commands as of 2024:036:12:00:00
******************************************
Running: /Users/aldcroft/git/kadi/kadi/scripts/update_cmds_v2.py
Version: 7.8.2.dev29+gb31e454
Time: Sun Feb 11 06:36:49 2024
User: root
Machine: saos-MBP.home
Processing args:
{'data_root': 'fix-matching-for-obs-cmds',
'log_level': 50,
'lookback': None,
'scenario': None,
'stop': '2024:036:12:00:00'}
******************************************
********************************************************************************
Last 5 observations as of 2024:036:12:00:00 using scenario='flight'
********************************************************************************
obsid obs_start obs_stop simpos manvr_start npnt_enab
----- --------------------- --------------------- ------ --------------------- ---------
28446 2024:034:23:43:10.567 2024:035:00:31:23.316 75624 2024:034:23:39:10.567 True
27349 2024:035:01:09:52.874 2024:035:03:27:11.000 92904 2024:035:00:31:33.567 True
27349 2024:035:03:49:38.563 2024:036:02:17:50.000 -99616 2024:035:03:27:11.000 False
27349 2024:036:02:46:43.204 2024:036:03:09:52.000 -99616 2024:036:02:18:00.250 False
65522 2024:036:03:09:52.000 2024:036:03:09:52.000 -99616 2024:036:02:18:00.250 False

Updating commands as of 2024:037:04:30:00
******************************************
Running: /Users/aldcroft/git/kadi/kadi/scripts/update_cmds_v2.py
Version: 7.8.2.dev29+gb31e454
Time: Sun Feb 11 06:36:54 2024
User: root
Machine: saos-MBP.home
Processing args:
{'data_root': 'fix-matching-for-obs-cmds',
'log_level': 50,
'lookback': None,
'scenario': None,
'stop': '2024:037:04:30:00'}
******************************************
********************************************************************************
Last 5 observations as of 2024:037:04:30:00 using scenario='flight'
********************************************************************************
obsid obs_start obs_stop simpos manvr_start npnt_enab
----- --------------------- --------------------- ------ --------------------- ---------
27349 2024:035:03:49:38.563 2024:036:02:17:50.000 -99616 2024:035:03:27:11.000 False
27349 2024:036:02:46:43.204 2024:036:03:09:52.000 -99616 2024:036:02:18:00.250 False
65522 2024:036:03:09:52.000 2024:036:20:42:12.000 -99616 2024:036:02:18:00.250 False
62619 2024:036:20:42:12.000 2024:037:03:59:44.000 -99616 2024:036:02:18:00.250 False
62619 2024:037:04:06:42.777 2024:042:06:13:08.510 -99616 2024:037:03:59:54.250 True

Updating commands as of 2024:037:04:50:00
******************************************
Running: /Users/aldcroft/git/kadi/kadi/scripts/update_cmds_v2.py
Version: 7.8.2.dev29+gb31e454
Time: Sun Feb 11 06:36:57 2024
User: root
Machine: saos-MBP.home
Processing args:
{'data_root': 'fix-matching-for-obs-cmds',
'log_level': 50,
'lookback': None,
'scenario': None,
'stop': '2024:037:04:50:00'}
******************************************
********************************************************************************
Last 5 observations as of 2024:037:04:50:00 using scenario='flight'
********************************************************************************
obsid obs_start obs_stop simpos manvr_start npnt_enab
----- --------------------- --------------------- ------ --------------------- ---------
27349 2024:036:02:46:43.204 2024:036:03:09:52.000 -99616 2024:036:02:18:00.250 False
65522 2024:036:03:09:52.000 2024:036:20:42:12.000 -99616 2024:036:02:18:00.250 False
62619 2024:036:20:42:12.000 2024:037:03:59:44.000 -99616 2024:036:02:18:00.250 False
62619 2024:037:04:06:42.777 2024:037:04:36:00.000 -99616 2024:037:03:59:54.250 True
62619 2024:037:04:52:05.911 2024:042:06:13:08.510 -99616 2024:037:04:36:10.250 True

Updating commands as of 2024:038:12:00:00
******************************************
Running: /Users/aldcroft/git/kadi/kadi/scripts/update_cmds_v2.py
Version: 7.8.2.dev29+gb31e454
Time: Sun Feb 11 06:37:00 2024
User: root
Machine: saos-MBP.home
Processing args:
{'data_root': 'fix-matching-for-obs-cmds',
'log_level': 50,
'lookback': None,
'scenario': None,
'stop': '2024:038:12:00:00'}
******************************************
********************************************************************************
Last 5 observations as of 2024:038:12:00:00 using scenario='flight'
********************************************************************************
obsid obs_start obs_stop simpos manvr_start npnt_enab
----- --------------------- --------------------- ------ --------------------- ---------
65522 2024:036:03:09:52.000 2024:036:20:42:12.000 -99616 2024:036:02:18:00.250 False
62619 2024:036:20:42:12.000 2024:037:03:59:44.000 -99616 2024:036:02:18:00.250 False
62619 2024:037:04:06:42.777 2024:037:04:36:00.000 -99616 2024:037:03:59:54.250 True
62619 2024:037:04:52:05.911 2024:038:01:15:00.000 -99616 2024:037:04:36:10.250 True
43729 2024:038:01:20:47.178 2024:038:01:49:18.954 -99616 2024:038:01:15:10.251 True

126 changes: 126 additions & 0 deletions validate/pr319/master.log
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
building file list ... done
./
cmds2.h5
cmds2.pkl

sent 142252954 bytes received 70 bytes 284506048.00 bytes/sec
total size is 142235385 speedup is 1.00
Running out of branch 'master'
Using kadi dir: master
kadi file /Users/aldcroft/git/kadi/kadi/__init__.py
Making kadi dir: master
Syncing archive to state as of 2024-02-04 (about midnight EST)
Running: rsync -av /Users/aldcroft/ska/data/kadi/cmds-from-2024feb04/ master/
kadi version: 7.8.2.dev27+gd1b8982

********************************************************************************
Last 5 observations as of 2024:035:12:00:00 using scenario='flight'
********************************************************************************
obsid obs_start obs_stop simpos manvr_start npnt_enab
----- --------------------- --------------------- ------ --------------------- ---------
29247 2024:037:09:06:17.311 2024:037:11:57:17.060 75624 2024:037:08:26:51.170 True
29255 2024:037:12:36:53.396 2024:037:15:27:53.145 92904 2024:037:11:57:27.311 True
29248 2024:037:16:07:31.012 2024:037:20:55:10.762 75624 2024:037:15:28:03.396 True
29256 2024:037:21:23:18.513 2024:038:00:27:26.696 75624 2024:037:20:55:21.013 True
43722 2024:038:00:56:28.907 2024:038:01:48:23.442 -99616 2024:038:00:27:36.947 True

Updating commands as of 2024:036:12:00:00
******************************************
Running: /Users/aldcroft/git/kadi/kadi/scripts/update_cmds_v2.py
Version: 7.8.2.dev27+gd1b8982
Time: Sun Feb 11 06:37:53 2024
User: root
Machine: saos-MBP.home
Processing args:
{'data_root': 'master',
'log_level': 50,
'lookback': None,
'scenario': None,
'stop': '2024:036:12:00:00'}
******************************************
********************************************************************************
Last 5 observations as of 2024:036:12:00:00 using scenario='flight'
********************************************************************************
obsid obs_start obs_stop simpos manvr_start npnt_enab
----- --------------------- --------------------- ------ --------------------- ---------
27349 2024:036:02:46:43.204 2024:036:03:09:52.000 -99616 2024:036:02:18:00.250 False
65522 2024:036:03:09:52.000 2024:036:20:42:12.000 -99616 2024:036:02:18:00.250 False
62619 2024:036:20:42:12.000 2024:037:03:59:44.000 -99616 2024:036:02:18:00.250 False
62619 2024:037:04:06:42.777 2024:037:04:36:00.000 -99616 2024:037:03:59:54.250 True
62619 2024:037:04:52:05.911 2024:043:01:42:33.505 -99616 2024:037:04:36:10.250 True

Updating commands as of 2024:037:04:30:00
******************************************
Running: /Users/aldcroft/git/kadi/kadi/scripts/update_cmds_v2.py
Version: 7.8.2.dev27+gd1b8982
Time: Sun Feb 11 06:37:58 2024
User: root
Machine: saos-MBP.home
Processing args:
{'data_root': 'master',
'log_level': 50,
'lookback': None,
'scenario': None,
'stop': '2024:037:04:30:00'}
******************************************
********************************************************************************
Last 5 observations as of 2024:037:04:30:00 using scenario='flight'
********************************************************************************
obsid obs_start obs_stop simpos manvr_start npnt_enab
----- --------------------- --------------------- ------ --------------------- ---------
27349 2024:036:02:46:43.204 2024:036:03:09:52.000 -99616 2024:036:02:18:00.250 False
65522 2024:036:03:09:52.000 2024:036:20:42:12.000 -99616 2024:036:02:18:00.250 False
62619 2024:036:20:42:12.000 2024:037:03:59:44.000 -99616 2024:036:02:18:00.250 False
62619 2024:037:04:06:42.777 2024:037:04:36:00.000 -99616 2024:037:03:59:54.250 True
62619 2024:037:04:52:05.911 2024:043:01:42:33.505 -99616 2024:037:04:36:10.250 True

Updating commands as of 2024:037:04:50:00
******************************************
Running: /Users/aldcroft/git/kadi/kadi/scripts/update_cmds_v2.py
Version: 7.8.2.dev27+gd1b8982
Time: Sun Feb 11 06:38:00 2024
User: root
Machine: saos-MBP.home
Processing args:
{'data_root': 'master',
'log_level': 50,
'lookback': None,
'scenario': None,
'stop': '2024:037:04:50:00'}
******************************************
********************************************************************************
Last 5 observations as of 2024:037:04:50:00 using scenario='flight'
********************************************************************************
obsid obs_start obs_stop simpos manvr_start npnt_enab
----- --------------------- --------------------- ------ --------------------- ---------
27349 2024:036:02:46:43.204 2024:036:03:09:52.000 -99616 2024:036:02:18:00.250 False
65522 2024:036:03:09:52.000 2024:036:20:42:12.000 -99616 2024:036:02:18:00.250 False
62619 2024:036:20:42:12.000 2024:037:03:59:44.000 -99616 2024:036:02:18:00.250 False
62619 2024:037:04:06:42.777 2024:037:04:36:00.000 -99616 2024:037:03:59:54.250 True
62619 2024:037:04:52:05.911 2024:043:01:42:33.505 -99616 2024:037:04:36:10.250 True

Updating commands as of 2024:038:12:00:00
******************************************
Running: /Users/aldcroft/git/kadi/kadi/scripts/update_cmds_v2.py
Version: 7.8.2.dev27+gd1b8982
Time: Sun Feb 11 06:38:03 2024
User: root
Machine: saos-MBP.home
Processing args:
{'data_root': 'master',
'log_level': 50,
'lookback': None,
'scenario': None,
'stop': '2024:038:12:00:00'}
******************************************
********************************************************************************
Last 5 observations as of 2024:038:12:00:00 using scenario='flight'
********************************************************************************
obsid obs_start obs_stop simpos manvr_start npnt_enab
----- --------------------- --------------------- ------ --------------------- ---------
65522 2024:036:03:09:52.000 2024:036:20:42:12.000 -99616 2024:036:02:18:00.250 False
62619 2024:036:20:42:12.000 2024:037:03:59:44.000 -99616 2024:036:02:18:00.250 False
62619 2024:037:04:06:42.777 2024:037:04:36:00.000 -99616 2024:037:03:59:54.250 True
62619 2024:037:04:52:05.911 2024:043:01:42:33.505 -99616 2024:037:04:36:10.250 True
43729 2024:038:01:20:47.178 2024:038:01:49:18.954 -99616 2024:038:01:15:10.251 True

Loading