diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dabf94a0..60f157ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.2.1 + rev: v0.3.5 hooks: # Run the linter. - id: ruff diff --git a/docs/api_commands.rst b/docs/api_commands.rst index 5005a913..35e0c408 100644 --- a/docs/api_commands.rst +++ b/docs/api_commands.rst @@ -5,7 +5,7 @@ Commands API Commands -------- -.. automodule:: kadi.commands.commands +.. automodule:: kadi.commands.commands_v2 :members: Observations diff --git a/docs/commands_states.rst b/docs/commands_states.rst index fdb41b99..4e11da11 100644 --- a/docs/commands_states.rst +++ b/docs/commands_states.rst @@ -13,16 +13,11 @@ The Commands archive is a table of every load command that has been run, or is c approved to be run, on the spacecraft since 2002. This archive accounts for load stoppages, replans, and certain non-load commands like ACIS CTI runs or Normal Sun Mode transitions. -As of this release there are two versions of the commands archive: - -- `Commands archive v2`_ (flight): this provides improved timeliness during - anomalies and better team-wide communication of non-load spacecraft - commanding. This relies on the `Chandra Command Events`_ sheet and OCCweb FOT - mission planning approved load products to maintain the commands database. -- `Commands archive v1`_ (legacy): this is the legacy version and - relies on iFOT load segments and the Chandra.cmd_states timelines database to - maintain the commands database. It is currently deprecated and will be - removed in a future release. +The flight `Commands archive v2`_ relies on the `Chandra Command Events`_ sheet and +OCCweb FOT mission planning approved load products to maintain the commands database. +This archive provides an up-to-date view of commands and states even during anomalies. +The `Chandra Command Events`_ sheet is kept current during anomalies by FOT MP and helps +facilitate team-wide communication of non-load spacecraft commanding. **States and continuity** @@ -342,21 +337,6 @@ permanently disable caching you can edit your configuration file (see >>> with conf.set_temp('cache_starcats', False): ... starcats = get_starcats('2022:001', '2022:002') -Commands archive v1 -------------------- - -Version 1 of the commands archive is provided for legacy support but it will be -removed in a future release. - -For details of the commands v1 archive please see: -please see: - -.. toctree:: - :maxdepth: 2 - - commands_v1.rst - - Chandra states and continuity ------------------------------ diff --git a/docs/commands_v1.rst b/docs/commands_v1.rst deleted file mode 100644 index a415a5b5..00000000 --- a/docs/commands_v1.rst +++ /dev/null @@ -1,159 +0,0 @@ -.. |get_cmds| replace:: :func:`~kadi.commands.commands.get_cmds` -.. |CommandTable| replace:: :class:`~kadi.commands.commands.CommandTable` - -Commands archive v1 -------------------- -In order to use the v1 version do the following:: - - >>> from kadi import commands - >>> commands.conf.commands_version = "1" # must be the string "1" not int 1 - -An alternative is to set the ``KADI_COMMANDS_VERSION`` environment variable to -``1``. This will globally apply to all subsequent Python sessions that inherit -this environment. For example from a linux/Mac bash command shell you can -enter:: - - $ export KADI_COMMANDS_VERSION=1 - -The basic way to select commands is with the |get_cmds| method. For example you can find -load commands from early in 2013 with:: - - >>> from kadi import commands - >>> cmds = commands.get_cmds('2013:001:00:00:00', '2013:001:00:56:10') - >>> print(cmds) - date type tlmsid scs step time timeline_id vcdu params - --------------------- ---------- ---------- --- ---- ------------- ----------- ------- ------ - 2013:001:00:37:37.653 ORBPOINT None 0 0 473387924.837 426098988 5533112 N/A - 2013:001:00:53:07.181 COMMAND_SW AOACRSTD 129 1524 473388854.365 426098990 5584176 N/A - 2013:001:00:54:07.181 COMMAND_SW AOFUNCDS 129 1526 473388914.365 426098990 5584410 N/A - 2013:001:00:55:07.181 COMMAND_SW AOFUNCDS 129 1528 473388974.365 426098990 5584644 N/A - 2013:001:00:56:07.181 COMMAND_SW AONMMODE 129 1530 473389034.365 426098990 5584878 N/A - 2013:001:00:56:07.181 ACISPKT AA00000000 132 1620 473389034.365 426098991 5584878 N/A - 2013:001:00:56:07.181 SIMTRANS None 132 1623 473389034.365 426098991 5584878 N/A - 2013:001:00:56:07.438 COMMAND_SW AONM2NPE 129 1532 473389034.622 426098990 5584879 N/A - - -In the |get_cmds| method, commands are selected with ``start <= date < stop``, where each -of these are evaluated as a date string with millisec precision. In order to get commands -at exactly a certain date you need to select with the ``date`` argument:: - - >>> print(commands.get_cmds(date='2013:001:00:56:07.181')) - date type tlmsid scs step timeline_id params - --------------------- ---------- ---------- --- ---- ----------- ------ - 2013:001:00:56:07.181 COMMAND_SW AONMMODE 129 1530 426098990 N/A - 2013:001:00:56:07.181 ACISPKT AA00000000 132 1620 426098991 N/A - 2013:001:00:56:07.181 SIMTRANS None 132 1623 426098991 N/A - -The output ``cmds`` is based on the astropy `Table -`_ object with many powerful and handy -features built in. For instance you could sort by ``type``, ``tlmsid`` and ``date``:: - - >>> cmds_type = cmds.copy() - >>> cmds_type.sort(['type', 'tlmsid', 'date']) - >>> print(cmds_type) - date type tlmsid scs step timeline_id params - --------------------- ---------- ---------- --- ---- ----------- ------ - 2013:001:00:56:07.181 ACISPKT AA00000000 132 1620 426098991 N/A - 2013:001:00:53:07.181 COMMAND_SW AOACRSTD 129 1524 426098990 N/A - 2013:001:00:54:07.181 COMMAND_SW AOFUNCDS 129 1526 426098990 N/A - 2013:001:00:55:07.181 COMMAND_SW AOFUNCDS 129 1528 426098990 N/A - 2013:001:00:56:07.438 COMMAND_SW AONM2NPE 129 1532 426098990 N/A - 2013:001:00:56:07.181 COMMAND_SW AONMMODE 129 1530 426098990 N/A - 2013:001:00:37:37.653 ORBPOINT None 0 0 426098988 N/A - 2013:001:00:56:07.181 SIMTRANS None 132 1623 426098991 N/A - -You can print a single command and get all the information about it:: - - >>> print(cmds[5]) - 2013:001:00:56:07.181 ACISPKT tlmsid=AA00000000 scs=132 step=1620 timeline_id=426098991 cmds=3 packet(40)=D80000300030603001300 words=3 - -This command has a number of attributes like ``date`` or ``tlmsid`` (shown in the original table) as well as command *parameters*: ``cmds``, ``packet(40)``, and ``words``. You can access any of the attributes or parameters like a dictionary:: - - >>> print(cmds[5]['packet(40)']) - D80000300030603001300 - -You probably noticed the first time we printed ``cmds`` that the command parameters -``params`` were all listed as ``N/A`` (Not Available). What happens if we print the -table again: - - >>> print(cmds) - date type tlmsid scs step timeline_id params - --------------------- ---------- ---------- --- ---- ----------- ----------------------------------------------- - 2013:001:00:37:37.653 ORBPOINT None 0 0 426098988 N/A - 2013:001:00:53:07.181 COMMAND_SW AOACRSTD 129 1524 426098990 N/A - 2013:001:00:54:07.181 COMMAND_SW AOFUNCDS 129 1526 426098990 N/A - 2013:001:00:55:07.181 COMMAND_SW AOFUNCDS 129 1528 426098990 N/A - 2013:001:00:56:07.181 COMMAND_SW AONMMODE 129 1530 426098990 N/A - 2013:001:00:56:07.181 ACISPKT AA00000000 132 1620 426098991 cmds=3 packet(40)=D80000300030603001300 words=3 - 2013:001:00:56:07.181 SIMTRANS None 132 1623 426098991 N/A - 2013:001:00:56:07.438 COMMAND_SW AONM2NPE 129 1532 426098990 N/A - -So what happened? The answer is that for performance reasons the |CommandTable| class is -lazy about loading the command parameters, and only does so when you directly request the -parameter value (as we did with ``packet(40)``). If you want to just fetch them all -at once you can do so with the ``fetch_params()`` method:: - - >>> cmds.fetch_params() - >>> print(cmds) - date type tlmsid scs step timeline_id params - --------------------- ---------- ---------- --- ---- ----------- ----------------------------------------------- - 2013:001:00:37:37.653 ORBPOINT None 0 0 426098988 event_type=EQF013M - 2013:001:00:53:07.181 COMMAND_SW AOACRSTD 129 1524 426098990 hex=8032000 msid=AOACRSTD - 2013:001:00:54:07.181 COMMAND_SW AOFUNCDS 129 1526 426098990 aopcadsd=21 hex=8030215 msid=AOFUNCDS - 2013:001:00:55:07.181 COMMAND_SW AOFUNCDS 129 1528 426098990 aopcadsd=32 hex=8030220 msid=AOFUNCDS - 2013:001:00:56:07.181 COMMAND_SW AONMMODE 129 1530 426098990 hex=8030402 msid=AONMMODE - 2013:001:00:56:07.181 ACISPKT AA00000000 132 1620 426098991 cmds=3 packet(40)=D80000300030603001300 words=3 - 2013:001:00:56:07.181 SIMTRANS None 132 1623 426098991 pos=-99616 - 2013:001:00:56:07.438 COMMAND_SW AONM2NPE 129 1532 426098990 hex=8030601 msid=AONM2NPE - -Finally, note that you can request the value of an attribute or parameter for the entire -command table. Note that command rows without that parameter will have a ``None`` object:: - - >>> print(cmds['msid']) - msid - -------- - None - AOACRSTD - AOFUNCDS - AOFUNCDS - AONMMODE - None - None - AONM2NPE - -Notes and caveats -^^^^^^^^^^^^^^^^^^ - -* The exact set of load commands relies on the `Chandra commanded states database - `_ to determine which command - loads ran on-board and for what duration. This information comes from a combination of - the iFOT load segments database and SOT update procedures for load interrupts. It has - been used operationally since 2009 and has frequent validation checking in the course of - thermal load review. Nevertheless there are likely a few missing commands here and - there, particularly associated with load stoppages and replans. - -* The kadi commands archive includes all commands for approved loads. Once loads have - been ingested into the database and iFOT has been updated accordingly, then the kadi - commands will reflect this update (within an hour). - -* Conversely if there is a load interrupt (SCS-107 or anomaly) then this will be reflected - in the commands archive within an hour after an on-call person runs a script to update - the `Chandra commanded states database - `_. - -* Each load command has an identifier that can be used to retrieve the exact set of mission - planning products in which the command was generated. This is valid even in the case - of a re-open replan in which a command load effectively has two source directories. - -* The archive includes a select set of non-load commands which result from either - autonomous on-board commanding (e.g. SCS-107) or real-time ground commanding - (e.g. anomaly recovery). This list is not comprehensive but includes those - commands which typically affect mission planning continuity and thermal modeling. - -* The parameters for the ACA star catalog command ``AOSTRCAT`` are not included since this - alone would dramatically increase the database file size. However, the commands are - included. - -* The command archive is stored in a highly performant HDF5 file backed by a - dictionary-based index file of unique command parameters. As of 2018-Jan, the commands - archive is stored in two files with a total size about 52 Mb. diff --git a/docs/commands_v2.rst b/docs/commands_v2.rst index 87637090..34ad515e 100644 --- a/docs/commands_v2.rst +++ b/docs/commands_v2.rst @@ -27,29 +27,6 @@ The other key web resource is the OCCweb `FOT mission planning approved load pro directory tree. This is used to automatically find all recent approved loads and incorporate them into the load commands archive. -Differences from v1 -------------------- - -Apart from the fundamental change in data sources mentioned above, some key -changes from v1 are as follows: - -- Commands table includes a ``source`` column that defines the source of the - command. Most commonly this is a weekly load name, but it can also indicate - a non-load command event for which further details are provided in the command - parameters. -- Information about each distinct observation is embedded into the command - archive as ``LOAD_EVENT`` pseudo-commands. The - :func:`~kadi.commands.observations.get_observations` provides a fast and - convenient way to find observations, both past and planned. See the - :ref:`getting-observations` section for more details. -- Information about each ACA star catalog is stored in the command - archive. The :func:`~kadi.commands.observations.get_observations` provides a - convenient way to find ACA star catalogs, both past and planned. See the - :ref:`getting-star-catalogs` section for more details. -- There are configuration options which can be set programmatically or in a fixed - configuration file to control behavior of the package. See the - `Configuration options`_ section for more details. - Scenarios --------- @@ -95,7 +72,6 @@ of the 2021:296 NSM recovery:: >>> from kadi import paths >>> from kadi.commands import conf, get_cmds - >>> conf.commands_version = '2' >>> cmds = get_cmds(start='2022:001') # Ensure local cmd_events.csv is up to date @@ -174,10 +150,6 @@ The available options with the default settings are as follows:: ## downloading from Google Sheets and OCCweb. commands_dir = ~/.kadi - ## Default version of kadi commands ("1" or "2"). Overridden by - ## KADI_COMMANDS_VERSION environment variable. - commands_version = 2 - ## Google Sheet ID for command events (flight scenario). cmd_events_flight_id = 19d6XqBhWoFjC-z1lS1nM6wLE_zjr4GYB1lOvrEGCbKQ @@ -200,9 +172,9 @@ Python to change a parameter for all subsequent code:: You can also temporarily change an option within a context manager:: - >>> with conf.set_temp('commands_version', '2'): - ... cmds2 = get_cmds('2022:001', '2022:002') # Use commands v2 - >>> cmds1 = get_cmds('2022:001', '2022:002') # Use commands v1 + >>> with conf.set_temp('include_in_work_command_events', True): + ... cmds_in_work = get_cmds('2022:001', '2022:002') # Use Commands In-work events + >>> cmds_flight = get_cmds('2022:001', '2022:002') # Use only Predictive or Definitive For an even-more permanent solution you can write out the configuration file to disk and then edit it. Be wary of "temporarily" changing an option and then @@ -221,10 +193,6 @@ Environment variables Override the default location of kadi flight data files ``cmds2.h5`` and ``cmds2.pkl``. -``KADI_COMMANDS_VERSION`` - Override the default kadi commands version. In order to use the commands - archive v2 you should set this to ``2``. - ``KADI_COMMANDS_DEFAULT_STOP`` For testing and demonstration purposes, this environment variable can be set to a date which is used as the default stop time for commands. In effect this diff --git a/kadi.cfg b/kadi.cfg index 3d7165f5..c4509646 100644 --- a/kadi.cfg +++ b/kadi.cfg @@ -21,10 +21,5 @@ # commands_dir = "~/.kadi" -# Default version of kadi commands ("1" or "2"). Overridden by -# KADI_COMMANDS_VERSION environment variable. -# commands_version = "1" - - # Google Sheet ID for command events (flight scenario). # cmd_events_flight_id = "19d6XqBhWoFjC-z1lS1nM6wLE_zjr4GYB1lOvrEGCbKQ" diff --git a/kadi/commands/__init__.py b/kadi/commands/__init__.py index 60146c27..2ab981ad 100644 --- a/kadi/commands/__init__.py +++ b/kadi/commands/__init__.py @@ -3,73 +3,8 @@ logger = logging.getLogger(__name__) -from astropy.config import ConfigNamespace -from kadi.config import ConfigItem - - -class Conf(ConfigNamespace): - """ - Configuration parameters for kadi. - """ - - default_lookback = ConfigItem( - 30, "Default lookback for previous approved loads (days)." - ) - cache_loads_in_astropy_cache = ConfigItem( - False, - "Cache backstop downloads in the astropy cache. Should typically be False, " - "but useful during development to avoid re-downloading backstops.", - ) - cache_starcats = ConfigItem( - True, - "Cache star catalogs that are retrieved to a file to avoid repeating the " - "slow process of identifying fid and stars in catalogs. The cache file is " - "conf.commands_dir/starcats.db.", - ) - clean_loads_dir = ConfigItem( - True, - "Clean backstop loads (like APR1421B.pkl.gz) in the loads directory that are " - "older than the default lookback. Most users will want this to be True, but " - "for development or if you always want a copy of the loads set to False.", - ) - commands_dir = ConfigItem( - "~/.kadi", - "Directory where command loads and command events are stored after " - "downloading from Google Sheets and OCCweb.", - ) - commands_version = ConfigItem( - "2", - 'Default version of kadi commands ("1" or "2"). Overridden by ' - "KADI_COMMANDS_VERSION environment variable.", - ) - - cmd_events_flight_id = ConfigItem( - "19d6XqBhWoFjC-z1lS1nM6wLE_zjr4GYB1lOvrEGCbKQ", - "Google Sheet ID for command events (flight scenario).", - ) - - cmd_events_exclude_intervals_gid = ConfigItem( - "1681877928", - "Google Sheet gid for validation exclude intervals in command events", - ) - - star_id_match_halfwidth = ConfigItem( - 1.5, "Half-width box size of star ID match for get_starcats() (arcsec)." - ) - - fid_id_match_halfwidth = ConfigItem( - 40, "Half-width box size of fid ID match for get_starcats() (arcsec)." - ) - - include_in_work_command_events = ConfigItem( - False, "Include In-work command events that are not yet approved." - ) - - -# Create a configuration instance for the user -conf = Conf() - - -from .commands import * -from .core import * +from kadi.commands.commands_v2 import * +from kadi.commands.core import * +from kadi.commands.observations import * +from kadi.config import conf diff --git a/kadi/commands/commands.py b/kadi/commands/commands.py deleted file mode 100644 index 27b19095..00000000 --- a/kadi/commands/commands.py +++ /dev/null @@ -1,77 +0,0 @@ -import functools -import os - -from kadi.commands import conf -from kadi.commands.observations import * # noqa - - -def get_cmds(start=None, stop=None, inclusive_stop=False, scenario=None, **kwargs): - """ - Get commands beteween ``start`` and ``stop``. - - By default the interval is ``start`` <= date < ``stop``, but if - ``inclusive_stop=True`` then the interval is ``start`` <= date <= ``stop``. - - Additional ``key=val`` pairs can be supplied to further filter the results. - Both ``key`` and ``val`` are case insensitive. In addition to the any of - the command parameters such as TLMSID, MSID, SCS, STEP, or POS, the ``key`` - can be: - - type - Command type e.g. COMMAND_SW, COMMAND_HW, ACISPKT, SIMTRANS - date - Exact date of command e.g. '2013:003:22:11:45.530' - - If ``date`` is provided then ``start`` and ``stop`` values are ignored. - - Examples:: - - >>> from kadi import commands cmds = commands.get_cmds('2012:001', - >>> '2012:030') cmds = commands.get_cmds('2012:001', '2012:030', - >>> type='simtrans') cmds = commands.get_cmds(type='acispkt', - >>> tlmsid='wsvidalldn') cmds = commands.get_cmds(msid='aflcrset') - >>> print(cmds) - - Parameters - ---------- - start : DateTime format (optional) Start time, defaults to beginning - of available commands (2002:001) - stop : DateTime format (optional) Stop time, defaults to end of available - commands - inclusive_stop - bool, include commands at exactly ``stop`` if True. - scenario : str, None - Commands scenario (applicable only for V2 commands) - kwargs - key=val keyword argument pairs for filtering - - Returns - ------- - CommandTable - Commands in the interval and matching the filter criteria. - """ - commands_version = os.environ.get("KADI_COMMANDS_VERSION", conf.commands_version) - if commands_version == "2": - from kadi.commands.commands_v2 import get_cmds as get_cmds_ - - get_cmds_ = functools.partial(get_cmds_, scenario=scenario) - else: - from kadi.commands.commands_v1 import get_cmds as get_cmds_ - - cmds = get_cmds_(start=start, stop=stop, inclusive_stop=inclusive_stop, **kwargs) - return cmds - - -def clear_caches(): - """Clear all commands caches. - - This is useful for testing and in case upstream products like the Command - Events sheet have changed during a session. - """ - commands_version = os.environ.get("KADI_COMMANDS_VERSION", conf.commands_version) - if commands_version == "2": - from kadi.commands.commands_v2 import clear_caches as clear_caches_vN - else: - from kadi.commands.commands_v1 import clear_caches as clear_caches_vN - - clear_caches_vN() diff --git a/kadi/commands/commands_v1.py b/kadi/commands/commands_v1.py deleted file mode 100644 index aa055bae..00000000 --- a/kadi/commands/commands_v1.py +++ /dev/null @@ -1,73 +0,0 @@ -import warnings -import weakref - -from astropy.table import Column -from cxotime import CxoTime - -from kadi.commands.core import LazyVal, _find, load_idx_cmds, load_pars_dict - -# Warn about deprecation but use FutureWarning so it actually shows up (since -# DeprecationWarning is ignored by default) -warnings.warn("kadi commands v1 is deprecated, use v2 instead", FutureWarning) - -# Globals that contain the entire commands table and the parameters index -# dictionary. -IDX_CMDS = LazyVal(load_idx_cmds) -PARS_DICT = LazyVal(load_pars_dict) -REV_PARS_DICT = LazyVal(lambda: {v: k for k, v in PARS_DICT.items()}) - - -def get_cmds(start=None, stop=None, inclusive_stop=False, **kwargs): - """ - Get commands beteween ``start`` and ``stop``. - - By default the interval is ``start`` <= date < ``stop``, but if - ``inclusive_stop=True`` then the interval is ``start`` <= date <= ``stop``. - - Additional ``key=val`` pairs can be supplied to further filter the results. - Both ``key`` and ``val`` are case insensitive. In addition to the any of - the command parameters such as TLMSID, MSID, SCS, STEP, or POS, the ``key`` - can be: - - type - Command type e.g. COMMAND_SW, COMMAND_HW, ACISPKT, SIMTRANS - date - Exact date of command e.g. '2013:003:22:11:45.530' - - If ``date`` is provided then ``start`` and ``stop`` values are ignored. - - Examples:: - - >>> from kadi import commands cmds = commands.get_cmds('2012:001', - >>> '2012:030') cmds = commands.get_cmds('2012:001', '2012:030', - >>> type='simtrans') cmds = commands.get_cmds(type='acispkt', - >>> tlmsid='wsvidalldn') cmds = commands.get_cmds(msid='aflcrset') - >>> print(cmds) - - Parameters - ---------- - start : DateTime format (optional) - Start time, defaults to beginning of available commands (2002:001) - stop : DateTime format (optional) - Stop time, defaults to end of available commands - inclusive_stop : bool - Include commands at exactly ``stop`` if True. - **kwargs : dict - key=val keyword argument pairs for filtering - - Returns - ------- - :class:`~kadi.commands.commands.CommandTable` of commands - """ - out = _find(start, stop, inclusive_stop, IDX_CMDS, PARS_DICT, **kwargs) - out.rev_pars_dict = weakref.ref(REV_PARS_DICT) - out["params"] = None if len(out) > 0 else Column([], dtype=object) - - out.add_column(CxoTime(out["date"], format="date").secs, name="time", index=6) - out["time"].info.format = ".3f" - - # Convert 'date' from bytestring to unicode. This is for compatibility with - # the legacy V1 API. - out.convert_bytestring_to_unicode() - - return out diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index 888ad907..ead46efb 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -24,21 +24,23 @@ from testr.test_helper import has_internet from kadi import occweb, paths -from kadi.commands import conf, get_cmds_from_backstop from kadi.commands.command_sets import get_cmds_from_event from kadi.commands.core import ( CommandTable, LazyVal, _find, + get_cmds_from_backstop, get_par_idx_update_pars_dict, load_idx_cmds, load_name_to_cxotime, load_pars_dict, vstack_exact, ) +from kadi.config import conf + +__all__ = ["clear_caches", "get_cmds"] # TODO configuration options, but use DEFAULT_* in the mean time -# - commands_version (v1, v2) MATCHING_BLOCK_SIZE = 500 diff --git a/kadi/commands/core.py b/kadi/commands/core.py index 645edaac..4ce93076 100644 --- a/kadi/commands/core.py +++ b/kadi/commands/core.py @@ -2,7 +2,6 @@ import calendar import functools import logging -import os import pickle import struct import warnings @@ -16,7 +15,6 @@ from cxotime import CxoTime from ska_helpers import retry -from kadi.commands import conf from kadi.paths import IDX_CMDS_PATH, PARS_DICT_PATH __all__ = [ @@ -759,14 +757,6 @@ def sort_in_backstop_order(self): This matches the order in backstop. """ - # Legacy sort for V1 commands archive - if ( - os.environ.get("KADI_COMMANDS_VERSION", conf.commands_version) == "1" - and "timeline_id" in self.colnames - ): - self.sort(["date", "step", "scs"]) - return - # For V2 use stable sort just on date, preserving the existing order. # Copied verbatim from astropy.table.Table.sort except 'stable' sort. # (Astropy sort does not provide `kind` argument for .sort(), hopefully diff --git a/kadi/commands/states.py b/kadi/commands/states.py index d02b37d4..1e302198 100644 --- a/kadi/commands/states.py +++ b/kadi/commands/states.py @@ -3,6 +3,7 @@ This is based entirely on known history of commands. """ + import collections import contextlib import functools diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index eceb6037..3c506348 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -1,5 +1,4 @@ import os -import warnings from pathlib import Path # Use data file from parse_cm.test for get_cmds_from_backstop test. @@ -16,16 +15,9 @@ from Quaternion import Quat from testr.test_helper import has_internet -warnings.filterwarnings( - "ignore", - category=FutureWarning, - message="kadi commands v1 is deprecated, use v2 instead", -) - import kadi.commands.states as kcs from kadi import commands from kadi.commands import ( - commands_v1, commands_v2, conf, core, @@ -35,55 +27,32 @@ read_backstop, ) from kadi.commands.command_sets import get_cmds_from_event -from kadi.scripts import update_cmds_v1, update_cmds_v2 +from kadi.scripts import update_cmds_v2 HAS_MPDIR = Path(os.environ["SKA"], "data", "mpcrit1", "mplogs", "2020").exists() HAS_INTERNET = has_internet() -VERSIONS = ["1", "2"] if HAS_INTERNET else ["1"] - - -@pytest.fixture(scope="module", params=VERSIONS) -def version(request): - return request.param - - -@pytest.fixture() -def version_env(monkeypatch, version): - if version is None: - monkeypatch.delenv("KADI_COMMANDS_VERSION", raising=False) - else: - monkeypatch.setenv("KADI_COMMANDS_VERSION", version) - return version @pytest.fixture(scope="module", autouse=True) def cmds_dir(tmp_path_factory): - with commands_v2.conf.set_temp("cache_loads_in_astropy_cache", True): - with commands_v2.conf.set_temp("clean_loads_dir", False): + with commands.conf.set_temp("cache_loads_in_astropy_cache", True): + with commands.conf.set_temp("clean_loads_dir", False): cmds_dir = tmp_path_factory.mktemp("cmds_dir") - with commands_v2.conf.set_temp("commands_dir", str(cmds_dir)): + with commands.conf.set_temp("commands_dir", str(cmds_dir)): yield -def test_find(version): - if version == "1": - idx_cmds = commands_v1.IDX_CMDS - pars_dict = commands_v1.PARS_DICT - else: - idx_cmds = commands_v2.IDX_CMDS - pars_dict = commands_v2.PARS_DICT +def test_find(): + idx_cmds = commands_v2.IDX_CMDS + pars_dict = commands_v2.PARS_DICT cs = core._find( "2012:029:12:00:00", "2012:030:12:00:00", idx_cmds=idx_cmds, pars_dict=pars_dict ) assert isinstance(cs, Table) - assert len(cs) == 147 if version == "1" else 151 # OBS commands in v2 only - if version == "1": - assert np.all(cs["timeline_id"][:10] == 426098447) - assert np.all(cs["timeline_id"][-10:] == 426098448) - else: - assert np.all(cs["source"][:10] == "JAN2612A") - assert np.all(cs["source"][-10:] == "JAN3012C") + assert len(cs) == 151 + assert np.all(cs["source"][:10] == "JAN2612A") + assert np.all(cs["source"][-10:] == "JAN3012C") assert cs["date"][0] == "2012:029:13:00:00.000" assert cs["date"][-1] == "2012:030:11:00:01.285" assert cs["tlmsid"][-1] == "CTXBON" @@ -122,16 +91,12 @@ def test_find(version): assert len(cs) == 2494 -def test_get_cmds(version_env): +def test_get_cmds(): cs = commands.get_cmds("2012:029:12:00:00", "2012:030:12:00:00") assert isinstance(cs, commands.CommandTable) - assert len(cs) == 147 if version_env == "1" else 151 # OBS commands in v2 only - if version_env == "2": - assert np.all(cs["source"][:10] == "JAN2612A") - assert np.all(cs["source"][-10:] == "JAN3012C") - else: - assert np.all(cs["timeline_id"][:10] == 426098447) - assert np.all(cs["timeline_id"][-10:] == 426098448) + assert len(cs) == 151 # OBS commands in v2 only + assert np.all(cs["source"][:10] == "JAN2612A") + assert np.all(cs["source"][-10:] == "JAN3012C") assert cs["date"][0] == "2012:029:13:00:00.000" assert cs["date"][-1] == "2012:030:11:00:01.285" assert cs["tlmsid"][-1] == "CTXBON" @@ -145,29 +110,19 @@ def test_get_cmds(version_env): assert repr(cmd).startswith("" - ) - assert str(cmd).endswith( - "scs=133 step=161 source=JAN3012C vcdu=15639968 pos=73176" - ) - else: - assert repr(cmd).endswith( - "scs=133 step=161 timeline_id=426098449 vcdu=15639968 pos=73176>" - ) - assert str(cmd).endswith( - "scs=133 step=161 timeline_id=426098449 vcdu=15639968 pos=73176" - ) + assert repr(cmd).endswith( + "scs=133 step=161 source=JAN3012C vcdu=15639968 pos=73176>" + ) + assert str(cmd).endswith("scs=133 step=161 source=JAN3012C vcdu=15639968 pos=73176") assert cmd["pos"] == 73176 assert cmd["step"] == 161 -def test_get_cmds_zero_length_result(version_env): +def test_get_cmds_zero_length_result(): cmds = commands.get_cmds(date="2017:001:12:00:00") assert len(cmds) == 0 - source_name = "source" if version_env == "2" else "timeline_id" + source_name = "source" assert cmds.colnames == [ "idx", "date", @@ -182,7 +137,7 @@ def test_get_cmds_zero_length_result(version_env): ] -def test_get_cmds_inclusive_stop(version_env): # noqa: ARG001 +def test_get_cmds_inclusive_stop(): # get_cmds returns start <= date < stop for inclusive_stop=False (default) # or start <= date <= stop for inclusive_stop=True. # Query over a range that includes two commands at exactly start and stop. @@ -194,7 +149,7 @@ def test_get_cmds_inclusive_stop(version_env): # noqa: ARG001 assert np.all(cmds["date"] == [start, stop]) -def test_cmds_as_list_of_dict(version_env): # noqa: ARG001 +def test_cmds_as_list_of_dict(): cmds = commands.get_cmds("2020:140", "2020:141") cmds_list = cmds.as_list_of_dict() assert isinstance(cmds_list, list) @@ -205,7 +160,7 @@ def test_cmds_as_list_of_dict(version_env): # noqa: ARG001 assert np.all(cmds_rt[name] == cmds[name]) -def test_cmds_as_list_of_dict_ska_parsecm(version_env): +def test_cmds_as_list_of_dict_ska_parsecm(): """Test the ska_parsecm=True compatibility mode for list_of_dict""" cmds = commands.get_cmds("2020:140", "2020:141") cmds_list = cmds.as_list_of_dict(ska_parsecm=True) @@ -222,10 +177,9 @@ def test_cmds_as_list_of_dict_ska_parsecm(version_env): "tlmsid": "CIMODESL", "type": "COMMAND_HW", "vcdu": 12516929, + "source": "MAY1820A", } - exp.update( - {"timeline_id": 426104285} if version_env == "1" else {"source": "MAY1820A"} - ) + assert cmds_list[0] == exp for cmd in cmds_list: @@ -233,14 +187,14 @@ def test_cmds_as_list_of_dict_ska_parsecm(version_env): assert all(param.upper() == param for param in cmd["params"]) -def test_get_cmds_from_backstop_and_add_cmds(version_env): +def test_get_cmds_from_backstop_and_add_cmds(): bs_file = Path(parse_cm.tests.__file__).parent / "data" / "CR182_0803.backstop" bs_cmds = commands.get_cmds_from_backstop(bs_file, remove_starcat=True) cmds = commands.get_cmds(start="2018:182:00:00:00", stop="2018:182:08:00:00") assert len(bs_cmds) == 674 - assert len(cmds) == 56 if version_env == "1" else 57 + assert len(cmds) == 57 # Get rid of source and timeline_id columns which can vary between v1 and v2 for cs in bs_cmds, cmds: @@ -272,15 +226,13 @@ def test_get_cmds_from_backstop_and_add_cmds(version_env): @pytest.mark.skipif("not HAS_MPDIR") -def test_commands_create_archive_regress(tmpdir, version_env, fast_sun_position_method): +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") +def test_commands_create_archive_regress(tmpdir, fast_sun_position_method): """Create cmds archive from scratch and test that it matches flight This tests over an eventful month that includes IU reset/NSM, SCS-107 (radiation), fast replan, loads approved but not uplinked, etc. """ - update_cmds = update_cmds_v2 if version_env == "2" else update_cmds_v1 - commands = commands_v2 if version_env == "2" else commands_v1 - kadi_orig = os.environ.get("KADI") start = CxoTime("2021:290") stop = start + 30 * u.day @@ -290,17 +242,17 @@ def test_commands_create_archive_regress(tmpdir, version_env, fast_sun_position_ with conf.set_temp("commands_dir", str(tmpdir)): try: os.environ["KADI"] = str(tmpdir) - update_cmds.main( + update_cmds_v2.main( ( - "--lookback=30" if version_env == "2" else f"--start={start.date}", + "--lookback=30", f"--stop={stop.date}", f"--data-root={tmpdir}", ) ) # Force reload of LazyVal - del commands.IDX_CMDS._val - del commands.PARS_DICT._val - del commands.REV_PARS_DICT._val + del commands_v2.IDX_CMDS._val + del commands_v2.PARS_DICT._val + del commands_v2.REV_PARS_DICT._val # Make sure we are seeing the temporary cmds archive cmds_empty = commands.get_cmds(start - 60 * u.day, start - 50 * u.day) @@ -352,24 +304,18 @@ def test_commands_create_archive_regress(tmpdir, version_env, fast_sun_position_ else: os.environ["KADI"] = kadi_orig - # Force reload - if version_env == "1": - del commands.IDX_CMDS._val - del commands.PARS_DICT._val - del commands.REV_PARS_DICT._val - else: - commands_v2.clear_caches() + commands.clear_caches() def stop_date_fixture_factory(stop_date): @pytest.fixture() def stop_date_fixture(monkeypatch): - commands_v2.clear_caches() + commands.clear_caches() monkeypatch.setenv("KADI_COMMANDS_DEFAULT_STOP", stop_date) cmds_dir = Path(conf.commands_dir) / CxoTime(stop_date).iso[:9] - with commands_v2.conf.set_temp("commands_dir", str(cmds_dir)): + with commands.conf.set_temp("commands_dir", str(cmds_dir)): yield - commands_v2.clear_caches() + commands.clear_caches() return stop_date_fixture @@ -381,19 +327,19 @@ def stop_date_fixture(monkeypatch): @pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") def test_get_cmds_v2_arch_only(stop_date_2020_12_03): # noqa: ARG001 - cmds = commands_v2.get_cmds(start="2020-01-01", stop="2020-01-02") + cmds = commands.get_cmds(start="2020-01-01", stop="2020-01-02") cmds = cmds[cmds["tlmsid"] != "OBS"] assert len(cmds) == 153 assert np.all(cmds["idx"] != -1) # Also do a zero-length query - cmds = commands_v2.get_cmds(start="2020-01-01", stop="2020-01-01") + cmds = commands.get_cmds(start="2020-01-01", stop="2020-01-01") assert len(cmds) == 0 - commands_v2.clear_caches() + commands.clear_caches() @pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") def test_get_cmds_v2_arch_recent(stop_date_2020_12_03): # noqa: ARG001 - cmds = commands_v2.get_cmds(start="2020-09-01", stop="2020-12-01") + cmds = commands.get_cmds(start="2020-09-01", stop="2020-12-01") cmds = cmds[cmds["tlmsid"] != "OBS"] # Since recent matches arch in the past, even though the results are a mix @@ -404,14 +350,14 @@ def test_get_cmds_v2_arch_recent(stop_date_2020_12_03): # noqa: ARG001 # PR #248: made this change from 17640 to 17644 assert 17640 <= len(cmds) <= 17644 - commands_v2.clear_caches() + commands.clear_caches() @pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") def test_get_cmds_v2_recent_only(stop_date_2020_12_03): # noqa: ARG001 # This query stop is well beyond the default stop date, so it should get # only commands out to the end of the NOV3020A loads (~ Dec 7). - cmds = commands_v2.get_cmds(start="2020-12-01", stop="2021-01-01") + cmds = commands.get_cmds(start="2020-12-01", stop="2021-01-01") cmds = cmds[cmds["tlmsid"] != "OBS"] assert len(cmds) == 1523 assert np.all(cmds["idx"] == -1) @@ -432,21 +378,21 @@ def test_get_cmds_v2_recent_only(stop_date_2020_12_03): # noqa: ARG001 ] # fmt: on # Same for no stop date - cmds = commands_v2.get_cmds(start="2020-12-01", stop=None) + cmds = commands.get_cmds(start="2020-12-01", stop=None) cmds = cmds[cmds["tlmsid"] != "OBS"] assert len(cmds) == 1523 assert np.all(cmds["idx"] == -1) # zero-length query - cmds = commands_v2.get_cmds(start="2020-12-01", stop="2020-12-01") + cmds = commands.get_cmds(start="2020-12-01", stop="2020-12-01") assert len(cmds) == 0 - commands_v2.clear_caches() + commands.clear_caches() @pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") def test_get_cmds_nsm_2021(stop_date_2021_10_24): # noqa: ARG001 """NSM at ~2021:296:10:41. This tests non-load commands from cmd_events.""" - cmds = commands_v2.get_cmds("2021:296:10:35:00") # , '2021:298:01:58:00') + cmds = commands.get_cmds("2021:296:10:35:00") # , '2021:298:01:58:00') cmds = cmds[cmds["tlmsid"] != "OBS"] exp = [ "2021:296:10:35:00.000 | COMMAND_HW | CIMODESL | OCT1821A | " @@ -521,7 +467,7 @@ def test_get_cmds_nsm_2021(stop_date_2021_10_24): # noqa: ARG001 "event_type=SCHEDULED_STOP_TIME, scs=0", ] assert cmds.pformat_like_backstop(max_params_width=200) == exp - commands_v2.clear_caches() + commands.clear_caches() @pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") @@ -529,7 +475,7 @@ def test_cmds_scenario(stop_date_2020_12_03): # noqa: ARG001 """Test custom scenario with a couple of ACIS commands""" # First make the cmd_events.csv file for the scenario scenario = "test_acis" - cmds_dir = Path(commands_v2.conf.commands_dir) / scenario + cmds_dir = Path(commands.conf.commands_dir) / scenario cmds_dir.mkdir(exist_ok=True, parents=True) # Note variation in format of date, since this comes from humans. cmd_evts_text = """\ @@ -540,7 +486,7 @@ def test_cmds_scenario(stop_date_2020_12_03): # noqa: ARG001 (cmds_dir / "cmd_events.csv").write_text(cmd_evts_text) # Now get commands in a time range that includes the new command events - cmds = commands_v2.get_cmds( + cmds = commands.get_cmds( "2020-12-01 00:08:00", "2020-12-01 00:09:00", scenario=scenario ) cmds = cmds[cmds["tlmsid"] != "OBS"] @@ -555,7 +501,7 @@ def test_cmds_scenario(stop_date_2020_12_03): # noqa: ARG001 " hex=7E00000, msid=CNOOPLR, scs=128", ] assert cmds.pformat_like_backstop() == exp - commands_v2.clear_caches() + commands.clear_caches() stop_date_2024_01_30 = stop_date_fixture_factory("2024-01-30") @@ -566,7 +512,7 @@ def test_nsm_offset_pitch_rasl_command_events(stop_date_2024_01_30): # noqa: AR """Test custom scenario with NSM offset pitch load event command""" # First make the cmd_events.csv file for the scenario scenario = "test_nsm_offset_pitch" - cmds_dir = Path(commands_v2.conf.commands_dir) / scenario + cmds_dir = Path(commands.conf.commands_dir) / scenario cmds_dir.mkdir(exist_ok=True, parents=True) # Note variation in format of date, since this comes from humans. cmd_evts_text = """\ @@ -578,7 +524,7 @@ def test_nsm_offset_pitch_rasl_command_events(stop_date_2024_01_30): # noqa: AR (cmds_dir / "cmd_events.csv").write_text(cmd_evts_text) # Now get commands in a time range that includes the new command events - cmds = commands_v2.get_cmds( + cmds = commands.get_cmds( "2024-01-24 12:00:00", "2024-01-25 05:00:00", scenario=scenario ) cmds = cmds[(cmds["tlmsid"] != "OBS") & (cmds["type"] != "ORBPOINT")] @@ -641,7 +587,7 @@ def test_nsm_offset_pitch_rasl_command_events(stop_date_2024_01_30): # noqa: AR assert np.isclose(pitch2, 160, atol=0.5) assert np.isclose((rasl2 - rasl1) % 360, 90, atol=0.5) - commands_v2.clear_caches() + commands.clear_caches() def test_command_set_bsh(): @@ -661,7 +607,7 @@ def test_command_set_bsh(): 2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0""" # noqa assert cmds.pformat_like_backstop(max_params_width=None) == exp.splitlines() - commands_v2.clear_caches() + commands.clear_caches() def test_command_set_safe_mode(): @@ -684,7 +630,7 @@ def test_command_set_safe_mode(): 2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 2000:001:00:01:17.960 | COMMAND_SW | AODSDITH | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0""" # noqa assert cmds.pformat_like_backstop(max_params_width=None) == exp.splitlines() - commands_v2.clear_caches() + commands.clear_caches() @pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") @@ -702,9 +648,7 @@ def test_bright_star_hold_event(cmds_dir, stop_date_2020_12_03): # noqa: ARG001 Definitive,2020:337:00:00:00,Bright star hold,,Tom Aldcroft, """ ) - cmds = commands_v2.get_cmds( - start="2020:336:21:48:00", stop="2020:338", scenario="bsh" - ) + cmds = commands.get_cmds(start="2020:336:21:48:00", stop="2020:338", scenario="bsh") exp = [ "2020:336:21:48:03.312 | LOAD_EVENT | OBS | NOV3020A | " "manvr_start=2020:336:21:09:24.361, prev_att=(-0.242373434, -0.348723922, " @@ -757,7 +701,7 @@ def test_bright_star_hold_event(cmds_dir, stop_date_2020_12_03): # noqa: ARG001 "event_type=XALT1, scs=0", ] assert cmds.pformat_like_backstop() == exp - commands_v2.clear_caches() + commands.clear_caches() @pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") @@ -890,7 +834,7 @@ def test_get_starcats_each_year(year): def test_get_starcats_with_cmds(): start, stop = "2021:365:19:00:00", "2022:002:01:25:00" - cmds = commands_v2.get_cmds(start, stop, scenario="flight") + cmds = commands.get_cmds(start, stop, scenario="flight") starcats0 = get_starcats(start, stop) starcats1 = get_starcats(cmds=cmds) assert len(starcats0) == len(starcats1) @@ -943,7 +887,7 @@ def test_get_starcats_date(): sc = get_starcats(obsid=8008, scenario="flight")[0] obs = get_observations(obsid=8008, scenario="flight")[0] assert sc.date == obs["starcat_date"] == "2007:002:04:31:43.965" - cmds = commands_v2.get_cmds("2007:002", "2007:003") + cmds = commands.get_cmds("2007:002", "2007:003") sc_cmd = cmds[cmds["date"] == obs["starcat_date"]][0] assert sc_cmd["type"] == "MP_STARCAT" @@ -1229,7 +1173,6 @@ def test_scenario_with_rts(monkeypatch, fast_sun_position_method): # example in the documentation. from kadi import paths - monkeypatch.setenv("KADI_COMMANDS_VERSION", "2") monkeypatch.setenv("KADI_COMMANDS_DEFAULT_STOP", "2021:299") # Ensure local cmd_events.csv is up to date by requesting "recent" commands @@ -1370,7 +1313,7 @@ def test_no_rltt_for_not_run_load(stop_date_2022_236): # noqa: ARG001 "2022:233:18:10:57.000 WSPOW0002A 135", "2022:233:18:12:00.000 RS_0000001 135", ] - cmds = commands_v2.get_cmds("2022:232:03:00:00", "2022:233:18:30:00") + cmds = commands.get_cmds("2022:232:03:00:00", "2022:233:18:30:00") cmds = cmds[cmds["type"] == "ACISPKT"] assert cmds["date", "tlmsid", "scs"].pformat() == exp @@ -1378,16 +1321,17 @@ def test_no_rltt_for_not_run_load(stop_date_2022_236): # noqa: ARG001 stop_date_2022_352 = stop_date_fixture_factory("2022-12-17") +@pytest.mark.skipif(not HAS_INTERNET, reason="No internet connection") def test_30_day_lookback_issue(stop_date_2022_352): # noqa: ARG001 """Test for fix in PR #265 of somewhat obscure issue where a query within the default 30-day lookback could give zero commands. Prior to the fix the query below would give zero commands (with the default stop date set accordingly).""" # noqa: D205, D209 - cmds = commands_v2.get_cmds("2022:319", "2022:324") + cmds = commands.get_cmds("2022:319", "2022:324") assert len(cmds) > 200 # Hit the CMDS_RECENT cache as well - cmds = commands_v2.get_cmds("2022:319:00:00:01", "2022:324:00:00:01") + cmds = commands.get_cmds("2022:319:00:00:01", "2022:324:00:00:01") assert len(cmds) > 200 @@ -1470,7 +1414,7 @@ def test_hrc_not_run_scenario(stop_date_2023200): # noqa: ARG001 # JUL1023A loads which start at 2023:191:03:43:28.615 scenario = "hrc_not_run" - cmds_dir = Path(commands_v2.conf.commands_dir) / scenario + cmds_dir = Path(commands.conf.commands_dir) / scenario cmds_dir.mkdir(exist_ok=True, parents=True) # Note variation in format of date, since this comes from humans. cmd_evts_text = """\ @@ -1506,7 +1450,7 @@ def test_hrc_not_run_scenario(stop_date_2023200): # noqa: ARG001 states_out = states[["datestart"] + keys].pformat_all() assert states_out == states_exp - commands_v2.clear_caches() + commands.clear_caches() test_command_not_run_cases = [ diff --git a/kadi/commands/tests/test_states.py b/kadi/commands/tests/test_states.py index 19a72281..c71a22ee 100644 --- a/kadi/commands/tests/test_states.py +++ b/kadi/commands/tests/test_states.py @@ -2,7 +2,6 @@ import gzip import hashlib import os -import warnings from pathlib import Path import numpy as np @@ -12,13 +11,6 @@ from chandra_time import DateTime from cheta import fetch from cxotime import CxoTime -from testr.test_helper import has_internet - -warnings.filterwarnings( - "ignore", - category=FutureWarning, - message="kadi commands v1 is deprecated, use v2 instead", -) from kadi import commands # noqa: E402 from kadi.commands import states # noqa: E402 @@ -29,23 +21,6 @@ except Exception: HAS_PITCH = False -VERSIONS = ["1", "2"] if has_internet() else ["1"] - - -@pytest.fixture(scope="module", params=VERSIONS) -def version(request): - return request.param - - -@pytest.fixture(autouse=True) -def version_env(monkeypatch, version): - if version is None: - monkeypatch.delenv("KADI_COMMANDS_VERSION", raising=False) - else: - monkeypatch.setenv("KADI_COMMANDS_VERSION", version) - return version - - # Canonical state0 giving spacecraft state at beginning of timelines # 2002:007:13 fetch --start 2002:007:13:00:00 --stop 2002:007:13:02:00 aoattqt1 # aoattqt2 aoattqt3 aoattqt4 cobsrqid aopcadmd tscpos diff --git a/kadi/commands/validate.py b/kadi/commands/validate.py index cf962493..1f3ec7fa 100644 --- a/kadi/commands/validate.py +++ b/kadi/commands/validate.py @@ -8,6 +8,7 @@ or from the command-line application ``kadi_validate_states`` (defined in ``kadi.scripts.validate_states``). """ + import functools import logging from abc import ABC diff --git a/kadi/config.py b/kadi/config.py index 31d4423b..52fffba9 100644 --- a/kadi/config.py +++ b/kadi/config.py @@ -8,7 +8,66 @@ """ # noqa from astropy import config +from astropy.config import ConfigNamespace class ConfigItem(config.ConfigItem): rootname = "kadi" + + +class Conf(ConfigNamespace): + """ + Configuration parameters for kadi. + """ + + default_lookback = ConfigItem( + 30, "Default lookback for previous approved loads (days)." + ) + cache_loads_in_astropy_cache = ConfigItem( + False, + "Cache backstop downloads in the astropy cache. Should typically be False, " + "but useful during development to avoid re-downloading backstops.", + ) + cache_starcats = ConfigItem( + True, + "Cache star catalogs that are retrieved to a file to avoid repeating the " + "slow process of identifying fid and stars in catalogs. The cache file is " + "conf.commands_dir/starcats.db.", + ) + clean_loads_dir = ConfigItem( + True, + "Clean backstop loads (like APR1421B.pkl.gz) in the loads directory that are " + "older than the default lookback. Most users will want this to be True, but " + "for development or if you always want a copy of the loads set to False.", + ) + commands_dir = ConfigItem( + "~/.kadi", + "Directory where command loads and command events are stored after " + "downloading from Google Sheets and OCCweb.", + ) + + cmd_events_flight_id = ConfigItem( + "19d6XqBhWoFjC-z1lS1nM6wLE_zjr4GYB1lOvrEGCbKQ", + "Google Sheet ID for command events (flight scenario).", + ) + + cmd_events_exclude_intervals_gid = ConfigItem( + "1681877928", + "Google Sheet gid for validation exclude intervals in command events", + ) + + star_id_match_halfwidth = ConfigItem( + 1.5, "Half-width box size of star ID match for get_starcats() (arcsec)." + ) + + fid_id_match_halfwidth = ConfigItem( + 40, "Half-width box size of fid ID match for get_starcats() (arcsec)." + ) + + include_in_work_command_events = ConfigItem( + False, "Include In-work command events that are not yet approved." + ) + + +# Create a configuration instance for the user +conf = Conf() diff --git a/kadi/scripts/update_cmds_v1.py b/kadi/scripts/update_cmds_v1.py deleted file mode 100644 index 8eda7816..00000000 --- a/kadi/scripts/update_cmds_v1.py +++ /dev/null @@ -1,607 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -import argparse -import difflib -import os -import pickle -from pathlib import Path - -import numpy as np -import pyyaks.logger -import ska_dbi -import ska_file -import tables -from chandra_time import DateTime -from ska_helpers.retry import retry_call -from ska_helpers.run_info import log_run_info - -from kadi import __version__ -from kadi.paths import IDX_CMDS_PATH, PARS_DICT_PATH - -MPLOGS_DIR = Path(os.environ["SKA"], "data", "mpcrit1", "mplogs") - -MIN_MATCHING_BLOCK_SIZE = 500 -BACKSTOP_CACHE = {} -CMDS_DTYPE = [ - ("idx", np.int32), - ("date", "|S21"), - ("type", "|S12"), - ("tlmsid", "|S10"), - ("scs", np.uint8), - ("step", np.uint16), - ("timeline_id", np.uint32), - ("vcdu", np.int32), -] - -logger = None # This is set as a global in main. Define here for pyflakes. - - -class UpdatedDict(dict): - """ - Dict with an ``n_updated`` attribute that gets incremented when any key value is set. - """ - - n_updated = 0 - - def __setitem__(self, *args, **kwargs): - self.n_updated += 1 - super(UpdatedDict, self).__setitem__(*args, **kwargs) - - -def get_opt(args=None): - """ - Get options for command line interface to update_cmd_states. - """ - parser = argparse.ArgumentParser(description="Update HDF5 cmds table") - parser.add_argument( - "--mp-dir", - help=( - "MP load directory (default=/data/mpcrit1/mplogs) " - "or $SKA/data/mpcrit1/mplogs)" - ), - ) - parser.add_argument("--start", help="Start date for update (default=stop-42 days)") - parser.add_argument("--stop", help="Stop date for update (default=Now+21 days)") - parser.add_argument( - "--log-level", - type=int, - default=10, - help="Log level (10=debug, 20=info, 30=warnings)", - ) - parser.add_argument("--data-root", default=".", help="Data root (default='.')") - parser.add_argument( - "--version", - action="version", - version="%(prog)s {version}".format(version=__version__), - ) - - args = parser.parse_args(args) - return args - - -def fix_nonload_cmds(nl_cmds): - """ - Convert non-load commands commands dict format from chandra_cmd_states - to the values/structure needed here. A typical value is shown below: - {'cmd': u'SIMTRANS', # Needs to be 'type' - 'date': u'2017:066:00:24:22.025', - 'id': 371228, # Store as params['nonload_id'] for provenence - 'msid': None, # Goes into params - 'params': {u'POS': -99616}, - 'scs': None, # Set to 0 - 'step': None, # Set to 0 - 'time': 605233531.20899999, # Ignored - 'timeline_id': None, # Set to 0 - 'tlmsid': None, # 'None' if None - 'vcdu': None}, # Set to -1 - """ - new_cmds = [] - for cmd in nl_cmds: - new_cmd = {} - new_cmd["date"] = str(cmd["date"]) - new_cmd["type"] = str(cmd["cmd"]) - new_cmd["tlmsid"] = str(cmd["tlmsid"]) - for key in ("scs", "step", "timeline_id"): - new_cmd[key] = 0 - new_cmd["vcdu"] = -1 - - new_cmd["params"] = {} - new_cmd["params"]["nonload_id"] = int(cmd["id"]) - if cmd["msid"] is not None: - new_cmd["params"]["msid"] = str(cmd["msid"]) - - # De-numpy (otherwise unpickling on PY3 has problems). - if "params" in cmd: - params = new_cmd["params"] - for key, val in cmd["params"].items(): - key = str(key) - try: - val = val.item() - except AttributeError: - pass - params[key] = val - - new_cmds.append(new_cmd) - - return new_cmds - - -def _tl_to_bs_cmds(tl_cmds, tl_id, db): - """ - Convert the commands ``tl_cmds`` (numpy recarray) that occur in the - timeline ``tl_id'' to a format mimicking backstop commands from - ska_parsecm.read_backstop(). This includes reading parameter values - from the ``db``. - - Parameters - ---------- - tl_cmds - numpy recarray of commands from timeline load segment - tl_id - timeline id - db - ska_dbi db object - - Returns - ------- - list of command dicts - """ - bs_cmds = [dict((col, row[col]) for col in tl_cmds.dtype.names) for row in tl_cmds] - cmd_index = dict((x["id"], x) for x in bs_cmds) - - # Add 'params' dict of command parameter key=val pairs to each tl_cmd - for par_table in ("cmd_intpars", "cmd_fltpars"): - tl_params = db.fetchall( - "SELECT * FROM %s WHERE timeline_id %s" - % (par_table, "= %d" % tl_id if tl_id else "IS NULL") - ) - - # Build up the params dict for each command in timeline load segment - for par in tl_params: - # I.e. cmd_index[par.cmd_id]['params'][par.name] = par.value - # but create the ['params'] dict as needed. - if par.cmd_id in cmd_index: - cmd_index[par.cmd_id].setdefault("params", {})[par.name] = par.value - - return bs_cmds - - -def get_cmds(start, stop, mp_dir=MPLOGS_DIR): - """ - Get backstop commands corresponding to the supplied timeline load segments. - The timeline load segments must be ordered by 'id'. - - Return cmds in the format defined by ska_parsecm.read_backstop(). - """ - mp_dir = str(mp_dir) - - # Get timeline_loads within date range. Also get non-load commands - # within the date range covered by the timelines. - server = os.path.join(os.environ["SKA"], "data", "cmd_states", "cmd_states.db3") - with ska_dbi.DBI(dbi="sqlite", server=server) as db: - timeline_loads = db.fetchall( - """SELECT * from timeline_loads - WHERE datestop > '{}' AND datestart < '{}' - ORDER BY id""".format(start.date, stop.date) - ) - - # Get non-load commands (from autonomous or ground SCS107, NSM, etc) in the - # time range that the timelines span. - tl_datestart = min(timeline_loads["datestart"]) - nl_cmds = db.fetchall( - "SELECT * from cmds where timeline_id IS NULL and " - 'date >= "{}" and date <= "{}"'.format(tl_datestart, stop.date) - ) - - # Private method from cmd_states.py fetches the actual int/float param values - # and returns list of dict. - nl_cmds = _tl_to_bs_cmds(nl_cmds, None, db) - nl_cmds = fix_nonload_cmds(nl_cmds) - logger.info( - f"Found {len(nl_cmds)} non-load commands between {tl_datestart} :" - f" {stop.date}" - ) - - logger.info( - "Found {} timelines included within {} to {}".format( - len(timeline_loads), start.date, stop.date - ) - ) - - if np.min(np.diff(timeline_loads["id"])) < 1: - raise ValueError("Timeline loads id not monotonically increasing") - - cmds = [] - orbit_cmds = [] - orbit_cmd_files = set() - - for tl in timeline_loads: - bs_file = ska_file.get_globfiles( - os.path.join(mp_dir + tl.mp_dir, "*.backstop") - )[0] - if bs_file not in BACKSTOP_CACHE: - bs_cmds = read_backstop(bs_file) - logger.info("Read {} commands from {}".format(len(bs_cmds), bs_file)) - BACKSTOP_CACHE[bs_file] = bs_cmds - else: - bs_cmds = BACKSTOP_CACHE[bs_file] - - # Process ORBPOINT (orbit event) pseudo-commands in backstop. These - # have scs=0 and need to be treated separately since during a replan - # or shutdown we still want these ORBPOINT to be in the cmds archive - # and not be excluded by timeline intervals. - if bs_file not in orbit_cmd_files: - bs_orbit_cmds = [x for x in bs_cmds if x["type"] == "ORBPOINT"] - for orbit_cmd in bs_orbit_cmds: - orbit_cmd["timeline_id"] = tl["id"] - if "EVENT_TYPE" not in orbit_cmd["params"]: - orbit_cmd["params"]["EVENT_TYPE"] = orbit_cmd["params"]["TYPE"] - del orbit_cmd["params"]["TYPE"] - orbit_cmds.extend(bs_orbit_cmds) - orbit_cmd_files.add(bs_file) - - # Only store commands for this timeline (match SCS and date) - bs_cmds = [ - x - for x in bs_cmds - if tl["datestart"] <= x["date"] <= tl["datestop"] and x["scs"] == tl["scs"] - ] - - for bs_cmd in bs_cmds: - bs_cmd["timeline_id"] = tl["id"] - - logger.info( - " Got {} backstop commands for timeline_id={} and SCS={}".format( - len(bs_cmds), tl["id"], tl["scs"] - ) - ) - cmds.extend(bs_cmds) - - orbit_cmds = get_unique_orbit_cmds(orbit_cmds) - logger.debug("Read total of {} orbit commands".format(len(orbit_cmds))) - - cmds.extend(nl_cmds) - cmds.extend(orbit_cmds) - - # Sort by date and SCS step number. - cmds = sorted(cmds, key=lambda y: (y["date"], y["step"])) - logger.debug( - "Read total of {} commands ({} non-load commands)".format( - len(cmds), len(nl_cmds) - ) - ) - - return cmds - - -def get_unique_orbit_cmds(orbit_cmds): - """ - Given list of ``orbit_cmds`` find the quasi-unique set. In the event of a - replan/reopen or other schedule oddity, it can happen that there are multiple cmds - that describe the same orbit event. Since the detailed timing might change between - schedule runs, cmds are considered the same if the date is within 3 minutes. - """ - if len(orbit_cmds) == 0: - return [] - - # Sort by (event_type, date) - orbit_cmds.sort(key=lambda y: (y["params"]["EVENT_TYPE"], y["date"])) - - uniq_cmds = [orbit_cmds[0]] - # Step through one at a time and add to uniq_cmds only if the candidate is - # "different" from uniq_cmds[-1]. - for cmd in orbit_cmds: - last_cmd = uniq_cmds[-1] - if ( - cmd["params"]["EVENT_TYPE"] == last_cmd["params"]["EVENT_TYPE"] - and abs(DateTime(cmd["date"]).secs - DateTime(last_cmd["date"]).secs) < 180 - ): - # Same event as last (even if date is a bit different). Now if this one - # has a larger timeline_id that means it is from a more recent schedule, so - # use that one. - if cmd["timeline_id"] > last_cmd["timeline_id"]: - uniq_cmds[-1] = cmd - else: - uniq_cmds.append(cmd) - - uniq_cmds.sort(key=lambda y: y["date"]) - - return uniq_cmds - - -def get_idx_cmds(cmds, pars_dict): - """ - For the input `cmds` (list of dicts), convert to the indexed command format where - parameters are specified as an index into `pars_dict`, a dict of unique parameter - values. - - Returns `idx_cmds` as a list of tuples: - (par_idx, date, time, cmd, tlmsid, scs, step, timeline_id, vcdu) - """ - idx_cmds = [] - - for i, cmd in enumerate(cmds): - if i % 10000 == 9999: - logger.info(" Iteration {}".format(i)) - - # Define a consistently ordered tuple that has all command parameter information - pars = cmd["params"] - keys = set(pars.keys()) - {"SCS", "STEP", "TLMSID"} - if cmd["tlmsid"] == "AOSTRCAT": - # Skip star catalog command because that has many (uninteresting) parameters - # and increases the file size and load speed by an order of magnitude. - pars_tup = () - else: - pars_tup = tuple((key.lower(), pars[key]) for key in sorted(keys)) - - try: - par_idx = pars_dict[pars_tup] - except KeyError: - # Along with transition to 32-bit idx in #190, ensure that idx=65535 - # never gets used. Prior to #190 this value was being used by - # get_cmds_from_backstop() assuming that it will never occur as a - # key in the pars_dict. Adding 65536 allows older versions to work - # with the new cmds.pkl pars_dict. - par_idx = len(pars_dict) + 65536 - pars_dict[pars_tup] = par_idx - - idx_cmds.append( - ( - par_idx, - cmd["date"], - cmd["type"], - cmd.get("tlmsid"), - cmd["scs"], - cmd["step"], - cmd["timeline_id"], - cmd["vcdu"], - ) - ) - - return idx_cmds - - -def add_h5_cmds(h5file, idx_cmds): - """ - Add `idx_cmds` to HDF5 file `h5file` of indexed spacecraft commands. - If file does not exist then create it. - """ - # Note: reading this file uncompressed is about 5 times faster, so sacrifice file size - # for read speed and do not use compression. - h5 = retry_call( - tables.open_file, [h5file], {"mode": "a"}, tries=4, delay=1, backoff=4 - ) - - # Convert cmds (list of tuples) to numpy structured array. This also works for an - # existing structured array. - cmds = np.array(idx_cmds, dtype=CMDS_DTYPE) - - # TODO : make sure that changes in non-load commands triggers an update - - try: - h5d = h5.root.data - logger.info("Opened h5 cmds table {}".format(h5file)) - except tables.NoSuchNodeError: - h5.create_table(h5.root, "data", cmds, "cmds", expectedrows=2e6) - logger.info("Created h5 cmds table {}".format(h5file)) - else: - date0 = min(idx_cmd[1] for idx_cmd in idx_cmds) - h5_date = h5d.cols.date[:] - idx_recent = np.searchsorted(h5_date, date0) - logger.info("Selecting commands from h5d[{}:]".format(idx_recent)) - logger.info(" {}".format(str(h5d[idx_recent]))) - h5d_recent = h5d[idx_recent:] # recent h5d entries - - # Define the column names that specify a complete and unique row - key_names = ("date", "type", "tlmsid", "scs", "step", "timeline_id", "vcdu") - - h5d_recent_vals = [ - tuple( - row[x].decode("ascii") if isinstance(row[x], bytes) else str(row[x]) - for x in key_names - ) - for row in h5d_recent - ] - idx_cmds_vals = [tuple(str(x) for x in row[1:]) for row in idx_cmds] - - diff = difflib.SequenceMatcher( - a=h5d_recent_vals, b=idx_cmds_vals, autojunk=False - ) - blocks = diff.get_matching_blocks() - logger.info("Matching blocks for existing HDF5 and timeline commands") - for block in blocks: - logger.info(" {}".format(block)) - opcodes = diff.get_opcodes() - logger.info("Diffs between existing HDF5 and timeline commands") - for opcode in opcodes: - logger.info(" {}".format(opcode)) - # Find the first matching block that is sufficiently long - for block in blocks: - if block.size > MIN_MATCHING_BLOCK_SIZE: - break - else: - raise ValueError( - "No matching blocks at least {} long".format(MIN_MATCHING_BLOCK_SIZE) - ) - - # Index into idx_cmds at the end of the large matching block. block.b is the - # beginning of the match. - idx_cmds_idx = block.b + block.size - - if idx_cmds_idx < len(cmds): - # Index into h5d at the point of the first diff after the large matching block - h5d_idx = block.a + block.size + idx_recent - - if h5d_idx < len(h5d): - logger.debug( - "Deleted relative cmds indexes {} .. {}".format( - h5d_idx - idx_recent, len(h5d) - idx_recent - ) - ) - logger.debug("Deleted cmds indexes {} .. {}".format(h5d_idx, len(h5d))) - h5d.truncate(h5d_idx) - - h5d.append(cmds[idx_cmds_idx:]) - logger.info( - "Added {} commands to HDF5 cmds table".format(len(cmds[idx_cmds_idx:])) - ) - else: - logger.info("No new timeline commands, HDF5 cmds table not updated") - - h5.flush() - logger.info("Upated HDF5 cmds table {}".format(h5file)) - h5.close() - - -def main(args=None): - global logger - - opt = get_opt(args) - - logger = pyyaks.logger.get_logger( - name="kadi_update_cmds", level=opt.log_level, format="%(asctime)s %(message)s" - ) - - log_run_info(logger.info, opt) - - # Set the global root data directory. This gets used in ..paths to - # construct file names. The use of an env var is needed to allow - # configurability of the root data directory within django. - os.environ["KADI"] = os.path.abspath(opt.data_root) - idx_cmds_path = IDX_CMDS_PATH() - pars_dict_path = PARS_DICT_PATH() - - try: - with open(pars_dict_path, "rb") as fh: - pars_dict = pickle.load(fh) - logger.info( - "Read {} pars_dict values from {}".format(len(pars_dict), pars_dict_path) - ) - except IOError: - logger.info( - "No pars_dict file {} found, starting from empty dict".format( - pars_dict_path - ) - ) - pars_dict = {} - - if not opt.mp_dir: - for prefix in ("/", os.environ["SKA"]): - pth = Path(prefix, "data", "mpcrit1", "mplogs") - if pth.exists(): - opt.mp_dir = str(pth) - break - else: - raise FileNotFoundError( - "no mission planning directories found (need --mp-dir)" - ) - logger.info(f"Using mission planning files at {opt.mp_dir}") - - # Recast as dict subclass that remembers if any element was updated - pars_dict = UpdatedDict(pars_dict) - - stop = DateTime(opt.stop) if opt.stop else DateTime() + 21 - start = DateTime(opt.start) if opt.start else stop - 42 - - cmds = get_cmds(start, stop, opt.mp_dir) - idx_cmds = get_idx_cmds(cmds, pars_dict) - add_h5_cmds(idx_cmds_path, idx_cmds) - - if pars_dict.n_updated > 0: - with open(pars_dict_path, "wb") as fh: - # Dump as pickle, first converting pars_dict from UpdatedDict to - # plain dict. The n_updated attribute is not used outside of - # update_cmds.py. See historical note in NOTES.build for context. - pickle.dump(dict(pars_dict), fh, protocol=2) - logger.info( - "Wrote {} pars_dict values ({} new) to {}".format( - len(pars_dict), pars_dict.n_updated, pars_dict_path - ) - ) - else: - logger.info("pars_dict was unmodified, not writing") - - -def _coerce_type(val): - """Coerce the supplied ``val`` (typically a string) into an int or float if - possible, otherwise as a string. - """ - try: - val = int(val) - except ValueError: - try: - val = float(val) - except ValueError: - val = str(val) - return val - - -def parse_params(paramstr): - """ - Parse parameters key1=val1,key2=val2,... from ``paramstr`` - - Parameter values are cast to the first type (int, float, or str) that - succeeds. - - Parameters - ---------- - paramstr - Comma separated string of key=val pairs - - Returns - ------- - dict - Dict of key=val pairs - """ - params = {} - for opt in paramstr.split(","): - try: - key, val = opt.split("=") - params[key] = val if key == "HEX" else _coerce_type(val) - except Exception: - pass # backstop has some quirks like blank or '??????' fields - - return params - - -def read_backstop(filename): - """ - Read commands from backstop file. - - Create dict with keys as follows for each command. ``paramstr`` is the - actual string with comma-separated parameters and ``params`` is the - corresponding dict of key=val pairs. - - Parameters - ---------- - filename - Backstop file name - - Returns - ------- - list of dict - List of dict for each command - """ - bs = [] - for bs_line in open(filename): - bs_line = bs_line.replace(" ", "") - date, vcdu, cmd_type, paramstr = bs_line.split("|") - vcdu = int( - vcdu[:-1] - ) # Get rid of final '0' from '8023268 0' (where space was stripped) - params = parse_params(paramstr) - bs.append( - { - "date": date, - "type": cmd_type, - "params": params, - "tlmsid": params.get("TLMSID"), - "scs": params.get("SCS"), - "step": params.get("STEP"), - "vcdu": vcdu, - } - ) - return bs - - -if __name__ == "__main__": - main() diff --git a/kadi/settings.py b/kadi/settings.py index 694cd721..e82785c4 100644 --- a/kadi/settings.py +++ b/kadi/settings.py @@ -8,6 +8,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.6/ref/settings/ """ + # Build paths inside the project like this: join(BASE_DIR, ...) import os from os.path import dirname, join, realpath diff --git a/setup.py b/setup.py index 1e1a54a0..cd03a999 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ entry_points = { "console_scripts": [ "get_chandra_states = kadi.commands.states:get_chandra_states", - "kadi_update_cmds = kadi.scripts.update_cmds_v1:main", "kadi_update_cmds_v2 = kadi.scripts.update_cmds_v2:main", "kadi_update_events = kadi.scripts.update_events:main", "kadi_validate_states = kadi.scripts.validate_states:main", diff --git a/task_schedule_cmds.cfg b/task_schedule_cmds.cfg index 3c7c662b..03577a52 100644 --- a/task_schedule_cmds.cfg +++ b/task_schedule_cmds.cfg @@ -39,6 +39,5 @@ alert aca@head.cfa.harvard.edu cron * * * * * - exec kadi_update_cmds --data-root=$ENV{SKA}/data/kadi exec kadi_update_cmds_v2 --data-root=$ENV{SKA}/data/kadi