From 7a350cd316a0f4ab056c37a6ae2710a89b3bcc18 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Wed, 27 Nov 2024 19:34:31 -0500 Subject: [PATCH] Modernize ruff (#342) * Modernize ruff * Fun never ends --- .pre-commit-config.yaml | 2 +- kadi/cmds/__init__.py | 2 +- kadi/cmds/cmds.py | 35 ++++---- kadi/commands/commands_v2.py | 4 +- kadi/commands/core.py | 11 ++- kadi/commands/states.py | 26 +++--- kadi/commands/tests/test_commands.py | 44 +++++----- kadi/commands/tests/test_states.py | 48 ++++++----- kadi/commands/validate.py | 16 ++-- kadi/config.py | 2 +- kadi/events/__init__.py | 3 +- kadi/events/json_field.py | 2 +- kadi/events/models.py | 10 ++- kadi/events/orbit_funcs.py | 4 +- kadi/events/plot.py | 2 +- kadi/events/query.py | 2 +- kadi/occweb.py | 18 ++-- kadi/scripts/update_events.py | 9 +- kadi/settings.py | 2 +- kadi/tests/test_events.py | 8 +- manual_test_cmds.py | 3 +- ruff-base.toml | 61 ++++++++++++++ ruff.toml | 83 +++++++------------ .../states-regression/states-regression.ipynb | 2 +- 24 files changed, 229 insertions(+), 170 deletions(-) create mode 100644 ruff-base.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 56dd3c77..ab160ded 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.6.1 + rev: v0.7.2 hooks: # Run the linter. - id: ruff diff --git a/kadi/cmds/__init__.py b/kadi/cmds/__init__.py index 38638780..57216005 100644 --- a/kadi/cmds/__init__.py +++ b/kadi/cmds/__init__.py @@ -1,2 +1,2 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from .cmds import * # noqa +from .cmds import * diff --git a/kadi/cmds/cmds.py b/kadi/cmds/cmds.py index 5c8a64a2..34a7d83d 100644 --- a/kadi/cmds/cmds.py +++ b/kadi/cmds/cmds.py @@ -70,13 +70,14 @@ def load_pars_dict(): def filter(start=None, stop=None, **kwargs): """ - Get commands with ``start`` <= date < ``stop``. Additional ``key=val`` pairs - can be supplied to further filter the results. Both ``key`` and ``val`` - are case insensitive. In addition to the any of the command parameters - such as TLMSID, MSID, SCS, STEP, or POS, the ``key`` can be: + Get commands with ``start`` <= date < ``stop``. - date : Exact date of command e.g. '2013:003:22:11:45.530' - type : Command type e.g. COMMAND_SW, COMMAND_HW, ACISPKT, SIMTRANS + Additional ``key=val`` pairs can be supplied to further filter the results. Both + ``key`` and ``val`` are case insensitive. In addition to the any of the command + parameters such as TLMSID, MSID, SCS, STEP, or POS, the ``key`` can be: + + - date : Exact date of command e.g. '2013:003:22:11:45.530' + - type : Command type e.g. COMMAND_SW, COMMAND_HW, ACISPKT, SIMTRANS Examples:: @@ -107,13 +108,14 @@ def filter(start=None, stop=None, **kwargs): def _find(start=None, stop=None, **kwargs): """ - Get commands ``start`` <= date < ``stop``. Additional ``key=val`` pairs - can be supplied to further filter the results. Both ``key`` and ``val`` - are case insensitive. In addition to the any of the command parameters - such as TLMSID, MSID, SCS, STEP, or POS, the ``key`` can be: + Get commands ``start`` <= date < ``stop``. + + Additional ``key=val`` pairs can be supplied to further filter the results. Both + ``key`` and ``val`` are case insensitive. In addition to the any of the command + parameters such as TLMSID, MSID, SCS, STEP, or POS, the ``key`` can be: - date : Exact date of command e.g. '2013:003:22:11:45.530' - type : Command type e.g. COMMAND_SW, COMMAND_HW, ACISPKT, SIMTRANS + - date : Exact date of command e.g. '2013:003:22:11:45.530' + - type : Command type e.g. COMMAND_SW, COMMAND_HW, ACISPKT, SIMTRANS Examples:: @@ -215,12 +217,9 @@ def __getitem__(self, item): if item in cmds.colnames: return cmds[item] - out = [] - for idx in cmds["idx"]: - # Find the parameters dict for this command from the reverse - # lookup table which maps index to the params tuple. - out.append(dict(rev_pars_dict[idx]).get(item)) - out = np.array(out) + # Find the parameters dict for this command from the reverse lookup table + # which maps index to the params tuple. + out = np.array([dict(rev_pars_dict[idx]).get(item) for idx in cmds["idx"]]) elif isinstance(item, int): out = Cmd(cmds[item]) diff --git a/kadi/commands/commands_v2.py b/kadi/commands/commands_v2.py index bb90311c..6d486ee3 100644 --- a/kadi/commands/commands_v2.py +++ b/kadi/commands/commands_v2.py @@ -750,7 +750,7 @@ def manvr_duration(q1, q2): q_manvr_3 = abs(-q2[0] * q1[0] - q2[1] * q1[1] - q2[2] * q1[2] + q2[3] * -q1[3]) # 4th component is cos(theta/2) - if q_manvr_3 > 1: # noqa PLR1730 + if q_manvr_3 > 1: # noqa: PLR1730 q_manvr_3 = 1 phimax = 2 * math.acos(q_manvr_3) @@ -764,7 +764,7 @@ def manvr_duration(q1, q2): np.sqrt(MANVR_DELTA**2 + 4 * phimax / MANVR_ALPHAMAX) / 2 - 1.5 * MANVR_DELTA ) - if eps < 0: # noqa PLR1730 + if eps < 0: # noqa: PLR1730 eps = 0 tm = 4 * MANVR_DELTA + 2 * eps + tau diff --git a/kadi/commands/core.py b/kadi/commands/core.py index 4ce93076..39acce9a 100644 --- a/kadi/commands/core.py +++ b/kadi/commands/core.py @@ -459,14 +459,17 @@ def get_starcat_keys_types(): def encode_starcat_params(params_dict): - assert set(params_dict.keys()) == set(STARCAT_KEYS) + if set(params_dict.keys()) != set(STARCAT_KEYS): + raise ValueError( + f"Star catalog keys must be {STARCAT_KEYS}, got {set(params_dict.keys())}" + ) args = tuple(params_dict[key] for key in STARCAT_KEYS) return struct.pack(STARCAT_TYPES, *args) def decode_starcat_params(params_bytes): vals = struct.unpack(STARCAT_TYPES, params_bytes) - return {key: val for key, val in zip(STARCAT_KEYS, vals)} + return dict(zip(STARCAT_KEYS, vals)) class CommandRow(Row): @@ -510,7 +513,7 @@ def values(self): return [self[key] for key in self.keys()] # noqa: SIM118 def items(self): - return [(key, value) for key, value in zip(self.keys(), self.values())] + return list(zip(self.keys(), self.values())) def __repr__(self): out = f"" @@ -1059,7 +1062,7 @@ def get_par_idx_update_pars_dict(pars_dict, cmd, params=None, rev_pars_dict=None # Define a consistently ordered tuple that has all command parameter information if params is None: params = cmd["params"] - keys = set(params.keys()) - set(("SCS", "STEP", "TLMSID")) + keys = set(params.keys()) - {"SCS", "STEP", "TLMSID"} if cmd["tlmsid"] == "AOSTRCAT": pars_tup = encode_starcat_params(params) if params else () diff --git a/kadi/commands/states.py b/kadi/commands/states.py index 1d92e75f..f7c38f78 100644 --- a/kadi/commands/states.py +++ b/kadi/commands/states.py @@ -366,7 +366,7 @@ def set_transitions(cls, transitions_list: list[Transition], cmds, start, stop): attrs = [attrs] for cmd in state_cmds: - transitions_list.append(Transition(cmd["date"], zip(attrs, vals))) + transitions_list.append(Transition(cmd["date"], zip(attrs, vals))) # noqa: PERF401 class ParamTransition(BaseTransition): @@ -878,7 +878,7 @@ def set_transitions(cls, transitions_list: list[Transition], cmds, start, stop): state_cmds = cls.get_state_changing_commands(cmds) for cmd in state_cmds: - transitions_list.append( + transitions_list.append( # noqa: PERF401 Transition(cmd["date"], sun_pos_mon=TransitionCallback(cls.callback)) ) @@ -933,7 +933,7 @@ def set_transitions(cls, transitions_list: list[Transition], cmds, start, stop): state_cmds = cls.get_state_changing_commands(cmds) for cmd in state_cmds: - transitions_list.append( + transitions_list.append( # noqa: PERF401 Transition( cmd["date"], eclipse_enable_spm=TransitionCallback(cls.callback) ) @@ -990,7 +990,7 @@ def set_transitions(cls, transitions_list: list[Transition], cmds, start, stop): state_cmds = cls.get_state_changing_commands(cmds) for cmd in state_cmds: - transitions_list.append( + transitions_list.append( # noqa: PERF401 Transition(cmd["date"], battery_connect=cmd["date"]) ) @@ -1171,7 +1171,7 @@ def set_transitions(cls, transitions_list: list[Transition], cmds, start, stop): # Now with the dates, finally make all the transition dicts which will # call `update_pitch_state` during state processing. for date in dates: - transitions_list.append( + transitions_list.append( # noqa: PERF401 Transition( date, update_sun_vector=TransitionCallback(cls.update_sun_vector_state), @@ -1365,7 +1365,7 @@ def set_transitions(cls, transitions_list: list[Transition], cmds, start, stop): state_cmds = cls.get_state_changing_commands(cmds) for cmd in state_cmds: - transitions_list.append( + transitions_list.append( # noqa: PERF401 Transition( cmd["date"], maneuver_transition=TransitionCallback(cls.callback) ) @@ -1561,7 +1561,7 @@ def set_transitions(cls, transitions_list: list[Transition], cmds, start, stop): state_cmds = cls.get_state_changing_commands(cmds) for cmd in state_cmds: - transitions_list.append( + transitions_list.append( # noqa: PERF401 Transition( cmd["date"], maneuver_transition=TransitionCallback( @@ -1611,7 +1611,7 @@ def set_transitions(cls, transitions_list: list[Transition], cmds, start, stop): state_cmds = cls.get_state_changing_commands(cmds) for cmd in state_cmds: - transitions_list.append( + transitions_list.append( # noqa: PERF401 Transition( cmd["date"], maneuver_transition=TransitionCallback( @@ -2477,9 +2477,13 @@ def get_continuity( # that were beyond the stop time and did not get processed. continuity_transitions.extend(states.meta["continuity_transitions"]) - colnames = set(states.colnames) - set( - ["datestart", "datestop", "tstart", "tstop", "trans_keys"] - ) + colnames = set(states.colnames) - { + "datestart", + "datestop", + "tstart", + "tstop", + "trans_keys", + } for colname in colnames: if states[colname][-1] is not None: # Reduce states to only the desired state_key diff --git a/kadi/commands/tests/test_commands.py b/kadi/commands/tests/test_commands.py index 21000290..7b2aa080 100644 --- a/kadi/commands/tests/test_commands.py +++ b/kadi/commands/tests/test_commands.py @@ -435,18 +435,18 @@ def test_get_cmds_v2_recent_only(stop_date_2020_12_03): # noqa: ARG001 assert np.all(cmds["idx"] == -1) # fmt: off assert cmds[:5].pformat_like_backstop() == [ - "2020:336:00:08:38.610 | COMMAND_HW | CNOOP | NOV3020A | hex=7E00000, msid=CNOOPLR, scs=128", # noqa - "2020:336:00:08:39.635 | COMMAND_HW | CNOOP | NOV3020A | hex=7E00000, msid=CNOOPLR, scs=128", # noqa - "2020:336:00:12:55.214 | ACISPKT | AA00000000 | NOV3020A | cmds=3, words=3, scs=131", # noqa - "2020:336:00:12:55.214 | ORBPOINT | None | NOV3020A | event_type=XEF1000, scs=0", # noqa - "2020:336:00:12:59.214 | ACISPKT | AA00000000 | NOV3020A | cmds=3, words=3, scs=131", # noqa + "2020:336:00:08:38.610 | COMMAND_HW | CNOOP | NOV3020A | hex=7E00000, msid=CNOOPLR, scs=128", + "2020:336:00:08:39.635 | COMMAND_HW | CNOOP | NOV3020A | hex=7E00000, msid=CNOOPLR, scs=128", + "2020:336:00:12:55.214 | ACISPKT | AA00000000 | NOV3020A | cmds=3, words=3, scs=131", + "2020:336:00:12:55.214 | ORBPOINT | None | NOV3020A | event_type=XEF1000, scs=0", + "2020:336:00:12:59.214 | ACISPKT | AA00000000 | NOV3020A | cmds=3, words=3, scs=131", ] assert cmds[-5:].pformat_like_backstop() == [ - "2020:342:03:15:02.313 | COMMAND_SW | OFMTSNRM | NOV3020A | hex=8010A00, msid=OFMTSNRM, scs=130", # noqa - "2020:342:03:15:02.313 | COMMAND_SW | COSCSEND | NOV3020A | hex=C800000, msid=OBC_END_SCS, scs=130", # noqa - "2020:342:06:04:34.287 | ACISPKT | AA00000000 | NOV3020A | cmds=3, words=3, scs=133", # noqa - "2020:342:06:04:34.287 | COMMAND_SW | COSCSEND | NOV3020A | hex=C800000, msid=OBC_END_SCS, scs=133", # noqa - "2020:342:06:04:34.287 | LOAD_EVENT | None | NOV3020A | event_type=SCHEDULED_STOP_TIME, scs=0", # noqa + "2020:342:03:15:02.313 | COMMAND_SW | OFMTSNRM | NOV3020A | hex=8010A00, msid=OFMTSNRM, scs=130", + "2020:342:03:15:02.313 | COMMAND_SW | COSCSEND | NOV3020A | hex=C800000, msid=OBC_END_SCS, scs=130", + "2020:342:06:04:34.287 | ACISPKT | AA00000000 | NOV3020A | cmds=3, words=3, scs=133", + "2020:342:06:04:34.287 | COMMAND_SW | COSCSEND | NOV3020A | hex=C800000, msid=OBC_END_SCS, scs=133", + "2020:342:06:04:34.287 | LOAD_EVENT | None | NOV3020A | event_type=SCHEDULED_STOP_TIME, scs=0", ] # fmt: on # Same for no stop date @@ -676,7 +676,7 @@ def test_command_set_bsh(): 2000:001:00:00:01.025 | SIMTRANS | None | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, pos=-99616, scs=0 2000:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 2000:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0 -2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0""" # noqa +2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=Bright_star_hold, event_date=2000:001:00:00:00, scs=0""" assert cmds.pformat_like_backstop(max_params_width=None) == exp.splitlines() commands.clear_caches() @@ -699,7 +699,7 @@ def test_command_set_safe_mode(): 2000:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 2000:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 2000:001:00:01:17.960 | ACISPKT | WSPOW00000 | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0 -2000:001:00:01:17.960 | COMMAND_SW | AODSDITH | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0""" # noqa +2000:001:00:01:17.960 | COMMAND_SW | AODSDITH | CMD_EVT | event=Safe_mode, event_date=2000:001:00:00:00, scs=0""" assert cmds.pformat_like_backstop(max_params_width=None) == exp.splitlines() commands.clear_caches() @@ -1072,16 +1072,16 @@ def test_get_cmds_from_event_case(par_str): ) cmd_events_all_exps = [ [ - "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Observing_not_run, event_date=2020:001:00:00:00, event_type=OBSERVING_NOT_RUN, load=FEB1422A, scs=0" # noqa + "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Observing_not_run, event_date=2020:001:00:00:00, event_type=OBSERVING_NOT_RUN, load=FEB1422A, scs=0" ], [ - "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Load_not_run, event_date=2020:001:00:00:00, event_type=LOAD_NOT_RUN, load=OCT2521A, scs=0" # noqa + "2020:001:00:00:00.000 | LOAD_EVENT | None | CMD_EVT | event=Load_not_run, event_date=2020:001:00:00:00, event_type=LOAD_NOT_RUN, load=OCT2521A, scs=0" ], [ - "2020:001:00:00:00.000 | ACISPKT | AA00000000 | CMD_EVT | event=Command, event_date=2020:001:00:00:00, cmds=3, words=3, scs=0" # noqa + "2020:001:00:00:00.000 | ACISPKT | AA00000000 | CMD_EVT | event=Command, event_date=2020:001:00:00:00, cmds=3, words=3, scs=0" ], [ - "2020:001:00:00:00.000 | NOT_RUN | 4OHETGIN | CMD_EVT | event=Command_not_run, event_date=2020:001:00:00:00, hex=8050300, msid=4OHETGIN, __type__=COMMAND_SW, scs=0" # noqa + "2020:001:00:00:00.000 | NOT_RUN | 4OHETGIN | CMD_EVT | event=Command_not_run, event_date=2020:001:00:00:00, hex=8050300, msid=4OHETGIN, __type__=COMMAND_SW, scs=0" ], [ "2020:001:00:00:00.000 | COMMAND_SW | OORMPEN | CMD_EVT | event=RTS," @@ -1122,7 +1122,7 @@ def test_get_cmds_from_event_case(par_str): " event_date=2020:001:00:00:00, scs=135", ], [ - "2020:001:00:00:00.000 | MP_OBSID | COAOSQID | CMD_EVT | event=Obsid, event_date=2020:001:00:00:00, id=65527, scs=0" # noqa + "2020:001:00:00:00.000 | MP_OBSID | COAOSQID | CMD_EVT | event=Obsid, event_date=2020:001:00:00:00, id=65527, scs=0" ], [ "2020:001:00:00:00.000 | COMMAND_SW | AONMMODE | CMD_EVT |" @@ -1248,9 +1248,9 @@ def test_get_cmds_from_event_case(par_str): "2020:001:00:00:00.000 | COMMAND_SW | OORMPDS | CMD_EVT |" " event=Bright_star_hold, event_date=2020:001:00:00:00, scs=0", "2020:001:00:00:01.025 | COMMAND_HW | AFIDP | CMD_EVT |" - " event=Bright_star_hold, event_date=2020:001:00:00:00, msid=AFLCRSET, scs=0", # noqa + " event=Bright_star_hold, event_date=2020:001:00:00:00, msid=AFLCRSET, scs=0", "2020:001:00:00:01.025 | SIMTRANS | None | CMD_EVT |" - " event=Bright_star_hold, event_date=2020:001:00:00:00, pos=-99616, scs=0", # noqa + " event=Bright_star_hold, event_date=2020:001:00:00:00, pos=-99616, scs=0", "2020:001:00:01:06.685 | ACISPKT | AA00000000 | CMD_EVT |" " event=Bright_star_hold, event_date=2020:001:00:00:00, scs=0", "2020:001:00:01:07.710 | ACISPKT | AA00000000 | CMD_EVT |" @@ -1259,7 +1259,7 @@ def test_get_cmds_from_event_case(par_str): " event=Bright_star_hold, event_date=2020:001:00:00:00, scs=0", ], [ - "2020:001:00:00:00.000 | COMMAND_SW | AOENDITH | CMD_EVT | event=Dither, event_date=2020:001:00:00:00, scs=0" # noqa + "2020:001:00:00:00.000 | COMMAND_SW | AOENDITH | CMD_EVT | event=Dither, event_date=2020:001:00:00:00, scs=0" ], ] @@ -1384,7 +1384,7 @@ def test_scenario_with_rts(monkeypatch, fast_sun_position_method): 2021:298:01:57:00.000 | ACISPKT | AA00000000 | OCT2521B | cmds=3, words=3, scs=131 2021:298:01:57:03.000 | ACISPKT | AA00000000 | OCT2521B | cmds=3, words=3, scs=131 2021:298:01:57:33.000 | COMMAND_SW | CODISASX | OCT2521B | msid=CODISASX, codisas1=135 , scs=131 -2021:298:01:57:34.000 | COMMAND_SW | COCLRSX | OCT2521B | msid=COCLRSX, coclrs1=135 , scs=131""" # noqa: E501 +2021:298:01:57:34.000 | COMMAND_SW | COCLRSX | OCT2521B | msid=COCLRSX, coclrs1=135 , scs=131""" out = "\n".join(cmds.pformat_like_backstop(max_params_width=60)) assert out == exp @@ -1641,7 +1641,7 @@ def test_command_not_run(case): 2023:351:13:30:33.849 | 4 0 | COMMAND_SW | TLMSID= COACTSX, HEX= 8402600, MSID= COACTSX, COACTS1=38 , COACTS2=0 , SCS= 128, STEP= 691 2023:351:13:30:55.373 | 5 0 | COMMAND_HW | TLMSID= 4MC5AEN, HEX= 4800012, MSID= 4MC5AEN, SCS= 131, STEP= 892 2023:351:19:38:41.550 | 6 0 | SIMTRANS | POS= 92904, SCS= 131, STEP= 1191 - """ # noqa + """ cmds = commands.read_backstop(backstop_text.strip().splitlines()) cmds["source"] = "DEC1123A" cmds_exp = cmds.copy() diff --git a/kadi/commands/tests/test_states.py b/kadi/commands/tests/test_states.py index 761059a3..654a5ab9 100644 --- a/kadi/commands/tests/test_states.py +++ b/kadi/commands/tests/test_states.py @@ -615,7 +615,7 @@ def test_get_continuity_keys(): """Test that output has only the desired state keys. Also test that one can provide a string instead of list of state keys""" continuity = states.get_continuity("2017:014:12:00:00", "clocking") - assert set(continuity) == set(["clocking", "__dates__"]) + assert set(continuity) == {"clocking", "__dates__"} def test_get_continuity_fail(): @@ -681,10 +681,10 @@ def test_reduce_states_merge_identical(all_keys): assert np.all(dr["val1"] == [1, 0, 1, 1]) assert np.all(dr["val2"] == [1, 1, 1, 0]) assert str(dr["trans_keys"][0]) == "val1,val2" - assert dr["trans_keys"][0] == set(["val1", "val2"]) - assert dr["trans_keys"][1] == set(["val1"]) - assert dr["trans_keys"][2] == set(["val1"]) - assert dr["trans_keys"][3] == set(["val2"]) + assert dr["trans_keys"][0] == {"val1", "val2"} + assert dr["trans_keys"][1] == {"val1"} + assert dr["trans_keys"][2] == {"val1"} + assert dr["trans_keys"][3] == {"val2"} def cmd_states_fetch_states(*args, **kwargs): @@ -734,9 +734,13 @@ def test_reduce_states_cmd_states(fast_sun_position_method): "2018:235:12:00:00", "2018:245:12:00:00", allow_identical=True ) - state_keys = set(STATE0) - set( - ["datestart", "datestop", "trans_keys", "tstart", "tstop"] - ) + state_keys = set(STATE0) - { + "datestart", + "datestop", + "trans_keys", + "tstart", + "tstop", + } # Default setting is reduce states with merge_identical=False, which is the same # as cmd_states. @@ -745,7 +749,7 @@ def test_reduce_states_cmd_states(fast_sun_position_method): assert len(ksr) == len(cs) - assert_all_close_states(cs, ksr, set(state_keys) - set(["trans_keys", "si_mode"])) + assert_all_close_states(cs, ksr, set(state_keys) - {"trans_keys", "si_mode"}) assert np.all(ksr["datestart"][1:] == cs["datestart"][1:]) assert np.all(ksr["datestop"][:-1] == cs["datestop"][:-1]) @@ -1052,7 +1056,7 @@ def test_backstop_sun_pos_mon_lunar(): 2017:087:08:10:35.838 ORBPOINT scs=0 step=0 timeline_id=426102503 event_type=LSPEXIT We expect SPM enable at the LSPEXIT time + 11 minutes = 2017:087:08:21:35.838 - """ # noqa + """ history = """ datestart datestop sun_pos_mon 2017:087:03:04:02.242 2017:087:07:21:40.189 DISA @@ -1683,17 +1687,17 @@ def test_grating_motion_states(): del sts["tstop"] # fmt: off exp = [ - " datestart datestop letg hetg grating trans_keys ", # noqa - "--------------------- --------------------- --------- --------- ------- ------------", # noqa - "2021:227:12:00:00.000 2021:227:23:06:03.276 RETR RETR NONE ", # noqa - "2021:227:23:06:03.276 2021:227:23:08:40.276 RETR INSR_MOVE HETG grating,hetg", # noqa - "2021:227:23:08:40.276 2021:228:08:15:00.722 RETR INSR HETG hetg", # noqa - "2021:228:08:15:00.722 2021:228:08:17:33.722 RETR RETR_MOVE NONE grating,hetg", # noqa - "2021:228:08:17:33.722 2021:229:17:41:45.525 RETR RETR NONE hetg", # noqa - "2021:229:17:41:45.525 2021:229:17:45:08.525 INSR_MOVE RETR LETG grating,letg", # noqa - "2021:229:17:45:08.525 2021:230:00:37:56.002 INSR RETR LETG letg", # noqa - "2021:230:00:37:56.002 2021:230:00:41:19.002 RETR_MOVE RETR NONE grating,letg", # noqa - "2021:230:00:41:19.002 2021:230:12:00:00.000 RETR RETR NONE letg", # noqa + " datestart datestop letg hetg grating trans_keys ", + "--------------------- --------------------- --------- --------- ------- ------------", + "2021:227:12:00:00.000 2021:227:23:06:03.276 RETR RETR NONE ", + "2021:227:23:06:03.276 2021:227:23:08:40.276 RETR INSR_MOVE HETG grating,hetg", + "2021:227:23:08:40.276 2021:228:08:15:00.722 RETR INSR HETG hetg", + "2021:228:08:15:00.722 2021:228:08:17:33.722 RETR RETR_MOVE NONE grating,hetg", + "2021:228:08:17:33.722 2021:229:17:41:45.525 RETR RETR NONE hetg", + "2021:229:17:41:45.525 2021:229:17:45:08.525 INSR_MOVE RETR LETG grating,letg", + "2021:229:17:45:08.525 2021:230:00:37:56.002 INSR RETR LETG letg", + "2021:230:00:37:56.002 2021:230:00:41:19.002 RETR_MOVE RETR NONE grating,letg", + "2021:230:00:41:19.002 2021:230:12:00:00.000 RETR RETR NONE letg", ] # fmt: on assert sts.pformat_all() == exp @@ -1745,7 +1749,7 @@ def test_hrc_states_with_scs_commanding(): 2023:039:23:12:15.958 | 3286720 0 | COMMAND_SW | HEX= 8408600, MSID= COACTSX, COACTS1=134 , COACTS2=0 , SCS= 132, STEP= 705, TLMSID= COACTSX 2023:040:04:55:45.483 | 3367148 0 | COMMAND_HW | HEX= 6420000, MSID= 215PCAOF, SCS= 132, STEP= 1027, TLMSID= 215PCAOF 2023:042:04:02:57.978 | 4029128 0 | COMMAND_SW | HEX= 8408600, MSID= COACTSX, COACTS1=134 , COACTS2=0 , SCS= 133, STEP= 314, TLMSID= COACTSX -2023:042:08:35:28.888 | 4092937 0 | COMMAND_HW | HEX= 6420000, MSID= 215PCAOF, SCS= 133, STEP= 460, TLMSID= 215PCAOF""" # noqa +2023:042:08:35:28.888 | 4092937 0 | COMMAND_HW | HEX= 6420000, MSID= 215PCAOF, SCS= 133, STEP= 460, TLMSID= 215PCAOF""" cmds = commands.read_backstop(backstop.splitlines()) sts = states.get_states( cmds=cmds, diff --git a/kadi/commands/validate.py b/kadi/commands/validate.py index 56b80f51..35122725 100644 --- a/kadi/commands/validate.py +++ b/kadi/commands/validate.py @@ -95,7 +95,7 @@ class PlotAttrs: max_gap_time: float = 300 -class Validate(ABC): +class Validate(ABC): # noqa: B024 """Validate kadi command states against telemetry base class. Class attributes are as follows: @@ -209,8 +209,9 @@ def exclude_intervals(self): return _exclude_intervals def add_exclude_intervals(self): - """Base method to exclude intervals, starting with intervals defined in the - Chandra Command Events Google Sheet. + """Base method to exclude intervals + + Starts with intervals defined in the Chandra Command Events Google Sheet. This method gets called at the end of self.tlm. @@ -278,8 +279,8 @@ def add_exclude_interval( self.exclude_intervals.sort("start") def exclude_ofp_intervals_except(self, states_expected: List[str]): - """Exclude intervals where OFP (on-board flight program) is not in the expected - state. + """ + Exclude intervals where OFP (on-board flight program) is not in expected state. This includes a padding of 30 minutes after SAFE mode and 5 minutes for non-NRML states other than SAFE like STUP, SYON, SYSF etc. @@ -555,8 +556,7 @@ def get_violations_mask(self): return bad def add_exclude_intervals(self): - """Exclude any intervals where online flight processing is not normal or safe - mode""" + """Exclude intervals where online flight processing is not normal or safe mode""" super().add_exclude_intervals() self.exclude_ofp_intervals_except(states_expected=["NRML", "SAFE"]) @@ -734,7 +734,7 @@ def get_overlap_mask(times: np.ndarray, intervals: Table): @functools.lru_cache(maxsize=1) -def get_command_sheet_exclude_intervals(state_key: str = None) -> Table: +def get_command_sheet_exclude_intervals() -> Table: url = EXCLUDE_INTERVALS_SHEET_URL.format( doc_id=kadi.commands.conf.cmd_events_flight_id, gid=kadi.commands.conf.cmd_events_exclude_intervals_gid, diff --git a/kadi/config.py b/kadi/config.py index eb704bfb..12614a26 100644 --- a/kadi/config.py +++ b/kadi/config.py @@ -5,7 +5,7 @@ See https://docs.astropy.org/en/stable/config/index.html#customizing-config-location-in-affiliated-packages and https://github.com/astropy/astropy/issues/12960. -""" # noqa +""" from astropy import config from astropy.config import ConfigNamespace diff --git a/kadi/events/__init__.py b/kadi/events/__init__.py index 737ec6c8..39a3049f 100644 --- a/kadi/events/__init__.py +++ b/kadi/events/__init__.py @@ -1,6 +1,5 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -"""\ -Access and manipulate events related to the Chandra X-ray Observatory +"""Access and manipulate events related to the Chandra X-ray Observatory Available events are: diff --git a/kadi/events/json_field.py b/kadi/events/json_field.py index 39fe692a..950b121e 100644 --- a/kadi/events/json_field.py +++ b/kadi/events/json_field.py @@ -28,7 +28,7 @@ def default(self, obj): if isinstance(obj, Decimal): return str(obj) elif isinstance(obj, datetime.datetime): - assert settings.TIME_ZONE == "UTC" + assert settings.TIME_ZONE == "UTC" # noqa: S101 return obj.strftime("%Y-%m-%dT%H:%M:%SZ") return simplejson.JSONEncoder.default(self, obj) diff --git a/kadi/events/models.py b/kadi/events/models.py index 7f3ba0a6..0a9cb18d 100644 --- a/kadi/events/models.py +++ b/kadi/events/models.py @@ -82,11 +82,13 @@ def _get_start_stop_vals(tstart, tstop, msidset, msids): return out -def _get_msid_changes(msids, sortmsids={}): +def _get_msid_changes(msids, sortmsids=None): """ For the list of fetch MSID objects, return a sorted structured array of each time any MSID value changes. """ + if sortmsids is None: + sortmsids = {} changes = [] for msid in msids: i_changes = np.flatnonzero(msid.vals[1:] != msid.vals[:-1]) @@ -169,7 +171,7 @@ def fuzz_states(states, t_fuzz): state0["tstop"] = state1["tstop"] state0["datestop"] = state1["datestop"] state0["duration"] = state0["tstop"] - state0["tstart"] - states = table.vstack([states[: i + 1], states[i + 2 :]]) # noqa + states = table.vstack([states[: i + 1], states[i + 2 :]]) break else: done = True @@ -2474,7 +2476,7 @@ def get_events(cls, start, stop=None): datestart = DateTime(start).date datestop = DateTime(stop).date - years = sorted(set(x[:4] for x in (datestart, datestop))) + years = sorted({x[:4] for x in (datestart, datestop)}) file_dates = [] for year in years: file_dates.extend(orbit_funcs.get_tlr_files(year)) @@ -2608,7 +2610,7 @@ def get_events(cls, start, stop=None): filename = Path(cls.intervals_file) if not filename.absolute(): filename = Path(DATA_DIR(), filename) - intervals = table.Table.read(str(filename), **cls.table_read_kwargs) # noqa + intervals = table.Table.read(str(filename), **cls.table_read_kwargs) # Custom in-place processing of raw intervals cls.process_intervals(intervals) diff --git a/kadi/events/orbit_funcs.py b/kadi/events/orbit_funcs.py index 57335cd2..712b42cf 100644 --- a/kadi/events/orbit_funcs.py +++ b/kadi/events/orbit_funcs.py @@ -325,14 +325,14 @@ def process_orbit_points(orbit_points): # Add a new orbit point for the ascending node EXIT which is the end of each orbit. # This simplifies bookkeeping later. for op in orbit_points[orbit_points["name"] == "EASCNCR"]: - new_ops.append( + new_ops.append( # noqa: PERF401 (op["date"], "XASCNCR", op["orbit_num"] - 1, op["descr"] + " EXIT") ) # Add corresponding XASCNCR for any new EASCNCR points for op in new_ops: if op[1] == "EASCNCR": - new_ops.append((op[0], "XASCNCR", op[2] - 1, op[3] + " EXIT")) + new_ops.append((op[0], "XASCNCR", op[2] - 1, op[3] + " EXIT")) # noqa: PERF401 logger.info("Adding {} new orbit points".format(len(new_ops))) new_ops = np.array(new_ops, dtype=ORBIT_POINTS_DTYPE) diff --git a/kadi/events/plot.py b/kadi/events/plot.py index 2626f7df..924c5253 100644 --- a/kadi/events/plot.py +++ b/kadi/events/plot.py @@ -15,7 +15,7 @@ def fix_ylim(ax, min_ylim): ax.set_ylim(y0, y1) -def tlm_event(evt, figsize=None, fig=None): +def tlm_event(evt, figsize=None, fig=None): # noqa: ARG001 """ Generic plot for a telemetry event """ diff --git a/kadi/events/query.py b/kadi/events/query.py index 398ee433..e1c75d5d 100644 --- a/kadi/events/query.py +++ b/kadi/events/query.py @@ -399,4 +399,4 @@ def __call__(self, pad=None, **filter_kwargs): query_instance = event_query_class(cls=model_class) query_instance.__doc__ = model_class.__doc__ globals()[query_name] = query_instance - __all__.append(query_name) + __all__.append(query_name) # noqa: PYI056 diff --git a/kadi/occweb.py b/kadi/occweb.py index 7860eef7..ae606986 100644 --- a/kadi/occweb.py +++ b/kadi/occweb.py @@ -1,7 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ -Provide an interface for getting pages from the OCCweb. For good measure leave -this off of github. +Provide an interface for getting pages from the OCCweb. """ import hashlib @@ -170,6 +169,7 @@ def get_ifot( def ftp_put_to_lucky(ftp_dirname, local_files, user=None, logger=None): """Put the ``local_files`` onto lucky in /``user``/``ftp_dirname``. + First put it at the top level, then when complete move it into a subdir eng_archive. This lets the OCC side just watch for fully-uploaded files in that directory. @@ -209,10 +209,12 @@ def ftp_put_to_lucky(ftp_dirname, local_files, user=None, logger=None): def ftp_get_from_lucky(ftp_dirname, local_files, user=None, logger=None): """ - Get files from lucky. This looks in remote ``ftp_dirname`` for files that - have basenames matching those of ``local_files``. The remote files - are copied to the corresponding local_files names. This is the converse - of ftp_put_to_lucky and thus requires unique basenames for all files. + Get files from lucky. + + This looks in remote ``ftp_dirname`` for files that have basenames matching those of + ``local_files``. The remote files are copied to the corresponding local_files + names. This is the converse of ftp_put_to_lucky and thus requires unique basenames + for all files. """ import ska_file import ska_ftp @@ -362,7 +364,9 @@ def get_occweb_dir(path, timeout=30, cache=False, user=None, password=None): astropy Table Table of directory entries """ - html = get_occweb_page(path, timeout=timeout, cache=cache) + html = get_occweb_page( + path, timeout=timeout, cache=cache, user=user, password=password + ) out = Table.read(html, format="ascii.html", guess=False) del out["col0"] del out["Description"] diff --git a/kadi/scripts/update_events.py b/kadi/scripts/update_events.py index a9701630..393534b0 100644 --- a/kadi/scripts/update_events.py +++ b/kadi/scripts/update_events.py @@ -11,7 +11,7 @@ from chandra_time import DateTime from ska_helpers.run_info import log_run_info -from kadi import __version__ # noqa +from kadi import __version__ # noqa: F401 logger = None # for pyflakes @@ -56,8 +56,9 @@ def get_opt(args=None): def try4times(func, *arg, **kwarg): """ - Work around problems with sqlite3 database getting locked out from writing, - presumably due to read activity. Not completely understood. + Work around problems with sqlite3 database getting locked out from writing + + This is presumably due to read activity. Not completely understood. This function will try to run func(*arg, **kwarg) a total of 4 times with an increasing sequence of wait times between tries. It catches only a database @@ -234,7 +235,7 @@ def get_events_and_event_models(EventModel, cls_name, events_in_dates): def main(): - global logger + global logger # noqa: PLW0603 # TODO - remove this global, use ska_helpers logger opt = get_opt() diff --git a/kadi/settings.py b/kadi/settings.py index e82785c4..08d3a15b 100644 --- a/kadi/settings.py +++ b/kadi/settings.py @@ -16,7 +16,7 @@ BASE_DIR = dirname(dirname(realpath(__file__))) # Data paths for kadi project -from kadi.paths import DATA_DIR, EVENTS_DB_PATH # noqa +from kadi.paths import DATA_DIR, EVENTS_DB_PATH # Make sure there is an events database if not os.path.exists(EVENTS_DB_PATH()): diff --git a/kadi/tests/test_events.py b/kadi/tests/test_events.py index a796cef5..a673aeb3 100644 --- a/kadi/tests/test_events.py +++ b/kadi/tests/test_events.py @@ -107,10 +107,10 @@ def test_basic_query(): rzt["dur"].format = ".3f" # fmt: off assert rzt.pformat(max_width=-1) == [ - " start stop tstart tstop dur orbit orbit_num perigee ", # noqa - "--------------------- --------------------- ------------- ------------- --------- ----- --------- ---------------------", # noqa - "2013:003:16:19:36.289 2013:004:02:21:34.289 473617243.473 473653361.473 36118.000 1852 1852 2013:003:22:29:59.302", # noqa - "2013:006:08:22:22.982 2013:006:17:58:48.982 473847810.166 473882396.166 34586.000 1853 1853 2013:006:13:58:21.389", # noqa + " start stop tstart tstop dur orbit orbit_num perigee ", + "--------------------- --------------------- ------------- ------------- --------- ----- --------- ---------------------", + "2013:003:16:19:36.289 2013:004:02:21:34.289 473617243.473 473653361.473 36118.000 1852 1852 2013:003:22:29:59.302", + "2013:006:08:22:22.982 2013:006:17:58:48.982 473847810.166 473882396.166 34586.000 1853 1853 2013:006:13:58:21.389", ] # fmt: on diff --git a/manual_test_cmds.py b/manual_test_cmds.py index fd4bbc4b..a56467e6 100755 --- a/manual_test_cmds.py +++ b/manual_test_cmds.py @@ -15,7 +15,8 @@ def test_ingest(): - """ + """Test ingest one day at a time. + Test that doing the ingest a day at a time (which is the normal operational scenario) gives the same result as a one-time bulk ingest. The latter is "easier" because there are no intermediate deletes and re-inserts. diff --git a/ruff-base.toml b/ruff-base.toml new file mode 100644 index 00000000..b2949f34 --- /dev/null +++ b/ruff-base.toml @@ -0,0 +1,61 @@ +# Copied originally from pandas. This config requires ruff >= 0.2. +target-version = "py311" + +# fix = true +lint.unfixable = [] + +lint.select = [ + "I", # isort + "F", # pyflakes + "E", "W", # pycodestyle + "YTT", # flake8-2020 + "B", # flake8-bugbear + "Q", # flake8-quotes + "T10", # flake8-debugger + "INT", # flake8-gettext + "PLC", "PLE", "PLR", "PLW", # pylint + "PIE", # misc lints + "PYI", # flake8-pyi + "TID", # tidy imports + "ISC", # implicit string concatenation + "TCH", # type-checking imports + "C4", # comprehensions + "PGH" # pygrep-hooks +] + +# Some additional rules that are useful +lint.extend-select = [ +"UP009", # UTF-8 encoding declaration is unnecessary +"SIM118", # Use `key in dict` instead of `key in dict.keys()` +"D205", # One blank line required between summary line and description +"ARG001", # Unused function argument +"RSE102", # Unnecessary parentheses on raised exception +"PERF401", # Use a list comprehension to create a transformed list +"S101", # Use of `assert` detected +] + +lint.ignore = [ + "ISC001", # Disable this for compatibility with ruff format + "E402", # module level import not at top of file + "E731", # do not assign a lambda expression, use a def + "PLR2004", # Magic number + "B028", # No explicit `stacklevel` keyword argument found + "PLR0913", # Too many arguments to function call + "PLR1730", # Checks for if statements that can be replaced with min() or max() calls +] + +extend-exclude = [ + "docs", +] + +[lint.pycodestyle] +max-line-length = 100 # E501 reports lines that exceed the length of 100. + +[lint.extend-per-file-ignores] +"__init__.py" = ["E402", "F401", "F403"] +# For tests: +# - D205: Don't worry about test docstrings +# - ARG001: Unused function argument false positives for some fixtures +# - E501: Line-too-long +# - S101: Do not use assert +"**/tests/test_*.py" = ["D205", "ARG001", "E501", "S101"] diff --git a/ruff.toml b/ruff.toml index 7f28aac8..6379b266 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,64 +1,43 @@ -# Copied originally from pandas -target-version = "py310" +extend = "ruff-base.toml" -# fix = true -lint.unfixable = [] - -lint.select = [ - "I", # isort - "F", # pyflakes - "E", "W", # pycodestyle - "YTT", # flake8-2020 - "B", # flake8-bugbear - "Q", # flake8-quotes - "T10", # flake8-debugger - "INT", # flake8-gettext - "PLC", "PLE", "PLR", "PLW", # pylint - "PIE", # misc lints - "PYI", # flake8-pyi - "TID", # tidy imports - "ISC", # implicit string concatenation - "TCH", # type-checking imports - "C4", # comprehensions - "PGH" # pygrep-hooks -] - -lint.ignore = [ - "ISC001", # Disable this for compatibility with ruff format - "B028", # No explicit `stacklevel` keyword argument found +# TODO : fix these and stop ignoring +lint.extend-ignore = [ "B905", # `zip()` without an explicit `strict=` parameter - "E402", # module level import not at top of file - "E731", # do not assign a lambda expression, use a def - "PLC1901", # compare-to-empty-string - "PLR0911", # Too many returns - "PLR0912", # Too many branches - "PLR0913", # Too many arguments to function call - "PLR2004", # Magic number - "PYI021", # Docstrings should not be included in stubs "PLR0915", # Too many statements - "PLW0603", # Global statements are discouraged + "PLR0912", # Too many branches "PLW2901", # Redefined loop name -] - -# TODO : fix these and stop ignoring -lint.extend-ignore = [ - "B006", # Do not use mutable data structures for argument defaults - "C401", # Unnecessary generator (rewrite as a `set` comprehension) - "C402", # Unnecessary generator (rewrite as a dict comprehension) - "C405", # Unnecessary `list` literal (rewrite as a `set` literal) "C408", # Unnecessary `dict` call (rewrite as a literal) - "C416", # Unnecessary `dict` comprehension (rewrite using `dict()`) - "G010", # warn is deprecated in favor of warning - "PGH004", # Use specific rule codes when using `noqa` - "PYI056", # Calling `.append()` on `__all__` may not be supported by all type checkers - "B024", # Abstract base class, but it has no abstract methods ] extend-exclude = [ "docs", "utils", - "validate", + + # Older notebooks in git repo "kadi_demo.ipynb", + "validate/compare-commands-acis-cti.ipynb", + "validate/compare-commands-v1-v2.ipynb", + "validate/play-obs-stop-issue.ipynb", + "validate/pr304-func-test.ipynb", + "validate/pr305-func-test.ipynb", + "validate/validate-cmd-interrupt-refactor.ipynb", + "validate/validate-cmds-v7.0.ipynb", + "validate/validate-get-starcats.ipynb", + "validate/gratings/plot_grating.py", + "validate/performance_states.py", + "validate/pr319/run_updates.py", + "validate/gratings/compare_grating_moves.py", + + # Notebooks in @taldcroft's git directory but not versioned + "notebooks/commands-archive-v2.0-play.ipynb", + "notebooks/commands-v2-test.ipynb", + "notebooks/compare-agasc-kadi-starcats.ipynb", + "notebooks/hrc-states-play.ipynb", + "notebooks/play-spm-eclipse-trouble.ipynb", + "notebooks/play-spm-fix.ipynb", + "notebooks/profile-command-states.ipynb", + "notebooks/sun-position-errors.ipynb", + "notebooks/validate-states-play.ipynb", ] [lint.pycodestyle] @@ -69,4 +48,6 @@ max-line-length = 100 # E501 reports lines that exceed the length of 100. "command_sets.py" = ["ARG001"] "**/tests/**" = ["D", "E501"] "states.py" = ["N801", "ARG003"] -"**/*.ipynb" = ["B018"] \ No newline at end of file +"**/*.ipynb" = ["B018"] +"manual_test_cmds.py" = ["S101"] # assert OK +"kadi/events/*.py" = ["D205"] # Docstring starts with a single line \ No newline at end of file diff --git a/validate/states-regression/states-regression.ipynb b/validate/states-regression/states-regression.ipynb index e36f8c31..016df70d 100644 --- a/validate/states-regression/states-regression.ipynb +++ b/validate/states-regression/states-regression.ipynb @@ -48,7 +48,7 @@ "metadata": {}, "outputs": [], "source": [ - "git_branch = subprocess.getoutput([\"git branch --show-current\"])\n" + "git_branch = subprocess.getoutput([\"git branch --show-current\"])" ] }, {