added functionality to write a YAML file to update obs-status of susp…

…ect obs.

agasc/supplement/magnitudes/mag_estimate.py

@@ -48,7 +48,8 @@
     2: 'No telemetry data',
     3: 'Mismatch in telemetry between aca_l0 and cheta',
     4: 'Time mismatch between cheta and level0',
-    5: 'Failed job'
+    5: 'Failed job',
+    6: 'Skipped'
 }
 EXCEPTION_CODES = collections.defaultdict(lambda: -1)
 EXCEPTION_CODES.update({msg: code for code, msg in EXCEPTION_MSG.items() if code > 0})
@@ -736,6 +737,20 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None):
     if len(star_obs) > 1:
         star_obs = star_obs.loc['mp_starcat_time', sorted(star_obs['mp_starcat_time'])]
 
+    # exclude star_obs that are in obs_status_override with status != 0
+    excluded_obs = np.array([((oi, ai) in obs_status_override
+                             and obs_status_override[(oi, ai)]['status'] != 0)
+                             for oi, ai in star_obs[['obsid', 'agasc_id']]])
+
+    logger.debug('  Excluding observations flagged in obs-status table: '
+                 f'{list(star_obs[excluded_obs]["obsid"])}')
+
+    if not np.any(~excluded_obs):
+        logger.debug('  Skipping star. All observations are flagged as not good.')
+        raise MagStatsException('Skipped')
+
+    star_obs = star_obs[~excluded_obs]
+
     failures = []
     all_telem = []
     stats = []
@@ -752,7 +767,9 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None):
 
     if len(all_telem) == 0:
         logger.debug(f'  Error in get_agasc_id_stats({agasc_id=}): No telemetry data')
-        raise MagStatsException('No telemetry data', agasc_id=agasc_id)
+        raise MagStatsException('No telemetry data',
+                                agasc_id=agasc_id,
+                                obsid=list(star_obs['obsid']))
 
     stats = Table(stats)
     n_obsids = len(star_obs)

agasc/supplement/magnitudes/update_mag_supplement.py

@@ -6,6 +6,7 @@
 import logging
 from functools import partial
 from multiprocessing import Pool
+import jinja2
 
 from tqdm import tqdm
 import tables
@@ -72,7 +73,8 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres
             obs_stats.append(obs_stat)
             fails += obs_fail
         except mag_estimate.MagStatsException as e:
-            fails.append(dict(e))
+            if e.msg not in ['Skipped']:
+                fails.append(dict(e))
         except Exception as e:
             # transform Exception to MagStatsException for standard book keeping
             logger.debug(f'Unexpected Error: {e}')
@@ -287,6 +289,50 @@ def update_supplement(agasc_stats, filename, include_all=True):
     return new_stars, updated_stars
 
 
+def write_obs_status_yaml(obs_stats, fails=(), filename=None):
+    obs = []
+    if obs_stats and len(obs_stats):
+        obs_stats = obs_stats[~obs_stats['obs_ok']]
+        obsids = np.unique(obs_stats['obsid'])
+        for obsid in obsids:
+            rows = obs_stats[obs_stats['obsid'] == obsid]
+            rows.sort(keys='agasc_id')
+            obs.append({
+                'obsid': obsid,
+                'agasc_id': list(rows['agasc_id']),
+                'status': 1
+            })
+    for fail in fails:
+        if fail['agasc_id'] is None or fail['obsid'] is None:
+            continue
+        obsids = fail['obsid'] if type(fail['obsid']) is list else [fail['obsid']]
+        agasc_id = fail['agasc_id']
+        for obsid in obsids:
+            obs.append({
+                'obsid': obsid,
+                'agasc_id': [agasc_id],
+                'status': 1
+            })
+    if len(obs) == 0:
+        return
+
+    yaml_template = """obs:
+  {%- for obs in observations %}
+  - obsid: {{ obs.obsid }}
+    status: {{ obs.status }}
+    agasc_id: [{% for agasc_id in obs.agasc_id -%}
+                  {{ agasc_id }}{%- if not loop.last %}, {% endif -%}
+               {%- endfor -%}]
+    comments:
+  {%- endfor %}
+  """
+    tpl = jinja2.Template(yaml_template)
+    if filename:
+        with open(filename, 'w') as fh:
+            fh.write(tpl.render(observations=obs))
+    return tpl.render(observations=obs)
+
+
 def do(output_dir,
        reports_dir,
        agasc_ids=None,
@@ -420,11 +466,19 @@ def do(output_dir,
         output_dir.mkdir(parents=True)
 
     update_mag_stats(obs_stats, agasc_stats, fails, output_dir)
+
+    obs_status_file = output_dir / 'obs_status.yml'
+    try:
+        write_obs_status_yaml(obs_stats, fails=fails, filename=obs_status_file)
+    except Exception as e:
+        logger.warning(f'Failed to write {obs_status_file}: {e}')
+
     if agasc_stats is not None and len(agasc_stats):
         new_stars, updated_stars = update_supplement(agasc_stats,
                                                      filename=filename)
 
         logger.info(f'  {len(new_stars)} new stars, {len(updated_stars)} updated stars')
+
         if email:
             try:
                 bad_obs = (