Ingest each AG frame to DB after exposure

src/lvmguider/cameras.py

@@ -21,10 +21,12 @@
 
 from sdsstools.time import get_sjd
 
-from lvmguider import __version__
+from lvmguider import __version__, config
+from lvmguider.dataclasses import CameraSolution
 from lvmguider.extraction import extract_sources as extract_sources_func
 from lvmguider.maskbits import GuiderStatus
 from lvmguider.tools import (
+    dataframe_to_database,
     elapsed_time,
     header_from_model,
     run_in_executor,
@@ -197,6 +199,7 @@ async def expose(
         if not command.actor.status & GuiderStatus.NON_IDLE:
             command.actor.status |= GuiderStatus.IDLE
 
+        # Update FITS file PROC extension.
         with elapsed_time(command, "updating lvm.agcam file"):
             await asyncio.gather(
                 *[
@@ -205,6 +208,20 @@ async def expose(
                 ]
             )
 
+        # Store data to DB.
+        with elapsed_time(command, "storing lvm.agcam to DB"):
+            camera_frame_dfs = []
+            for fn in filenames:
+                cs = CameraSolution.open(fn)
+                camera_frame_dfs.append(cs.to_framedata().to_dataframe())
+
+            await run_in_executor(
+                dataframe_to_database,
+                pandas.concat(camera_frame_dfs, axis=0, ignore_index=True),
+                config["database"]["agcam_frame_table"],
+                delete_columns=["frameno", "telescope", "camera"],
+            )
+
         return (list(filenames), next_seqno, list(sources) if extract_sources else None)
 
     def reset_seqno(self):

src/lvmguider/coadd.py

@@ -54,7 +54,6 @@
     get_raw_extension,
     get_spec_frameno,
     header_from_model,
-    isot_to_sjd,
     nan_or_none,
     polyfit_with_sigclip,
     sort_files_by_camera,
@@ -1605,31 +1604,6 @@ def coadd_to_database(
     except Exception as err:
         log.error(f"Failed loading frames to database: {str(err).strip()}")
 
-    # Load AG frames.
-    ag_frames_file = coadd_file.with_name(coadd_file.stem + "_frames.parquet")
-    if ag_frames_file.exists():
-        log.debug("Loading AG frames to database.")
-        try:
-            ag_frames = pandas.read_parquet(ag_frames_file, dtype_backend="pyarrow")
-            if exposure_no:
-                ag_frames["exposure_no"] = exposure_no
-            if len(ag_frames) > 0:
-                ag_frames_mjd = isot_to_sjd(ag_frames["date_obs"].iloc[0])
-                ag_frames["mjd"] = ag_frames_mjd
-                ag_frames["stacked"] = ag_frames.stacked.astype(bool)
-                ag_frames["solved"] = ag_frames.solved.astype(bool)
-                table_name = config["database"]["agcam_frame_table"]
-                dataframe_to_database(
-                    ag_frames,
-                    table_name,
-                    delete_columns=delete_columns,
-                    **db_connection_params,
-                )
-        except Exception as err:
-            log.error(f"Failed loading frames to database: {str(err).strip()}")
-    else:
-        log.error(f"Cannot find AG frames file {ag_frames_file}")
-
     # Load guider data.
     guider_frames_file = coadd_file.with_name(coadd_file.stem + "_guiderdata.parquet")
     if guider_frames_file.exists():
@@ -1643,8 +1617,6 @@ def coadd_to_database(
                 guider_frames["exposure_no"] = exposure_no
             if len(guider_frames) > 0:
                 guider_frames["mjd"] = sjd
-                guider_frames["solved"] = guider_frames.solved.astype(bool)
-                guider_frames["applied"] = guider_frames.applied.astype(bool)
                 table_name = config["database"]["guider_frame_table"]
                 dataframe_to_database(
                     guider_frames,

src/lvmguider/dataclasses.py

@@ -73,7 +73,8 @@ def zero_point(self):
         """Returns the median zero point."""
 
         if self.sources is not None:
-            return float(self.sources.zp.dropna().median())
+            if len(data := self.sources.zp.dropna()) > 0:
+                return float(data.median())
 
         return numpy.nan
 
@@ -409,8 +410,8 @@ def to_dataframe(self):
             dec=numpy.float64(pointing[1]),
             pa=numpy.float32(self.solution.pa),
             zero_point=numpy.float32(self.solution.zero_point),
-            stacked=int(self.stacked),
-            solved=int(self.solution.solved),
+            stacked=bool(self.stacked),
+            solved=bool(self.solution.solved),
             wcs_mode=self.solution.wcs_mode,
         )
         df.loc[0, list(new_row)] = list(new_row.values())
@@ -549,7 +550,7 @@ def guider_data(self):
                 fwhm=numpy.float32(gs.fwhm),
                 pa=numpy.float32(pa),
                 zero_point=numpy.float32(gs.zero_point),
-                solved=int(gs.solved),
+                solved=bool(gs.solved),
                 n_cameras_solved=int(gs.n_cameras_solved),
                 guide_mode=gs.guide_mode,
                 x_ff_pixel=numpy.float32(gs.guide_pixel[0]),
@@ -564,7 +565,7 @@ def guider_data(self):
                 pa_off=numpy.float32(gs.pa_off),
                 axis0_off=numpy.float32(gs.axis0_off),
                 axis1_off=numpy.float32(gs.axis1_off),
-                applied=int(gs.correction_applied),
+                applied=bool(gs.correction_applied),
                 ax0_applied=numpy.float32(gs.correction[0]),
                 ax1_applied=numpy.float32(gs.correction[1]),
                 rot_applied=numpy.float32(gs.correction[2]),

src/lvmguider/etc/datamodel.yml

@@ -132,16 +132,16 @@ FRAMEDATA:
   dec: 'f8[pyarrow]'
   pa: 'f4[pyarrow]'
   zero_point: 'f4[pyarrow]'
-  stacked: 'int8[pyarrow]'
-  solved: 'int8[pyarrow]'
+  stacked: 'bool[pyarrow]'
+  solved: 'bool[pyarrow]'
   wcs_mode: 'string[pyarrow]'
 
 GUIDERDATA_FRAME:
   frameno: 'int16[pyarrow]'
   telescope: 'string[pyarrow]'
   mjd: 'int32[pyarrow]'
   date_obs: 'string[pyarrow]'
-  solved: 'int8[pyarrow]'
+  solved: 'bool[pyarrow]'
   n_cameras_solved: 'int8[pyarrow]'
   guide_mode: 'string[pyarrow]'
   fwhm: 'f4[pyarrow]'
@@ -159,7 +159,7 @@ GUIDERDATA_FRAME:
   pa_off: 'f4[pyarrow]'
   axis0_off: 'f4[pyarrow]'
   axis1_off: 'f4[pyarrow]'
-  applied: 'int8[pyarrow]'
+  applied: 'bool[pyarrow]'
   ax0_applied: 'f4[pyarrow]'
   ax1_applied: 'f4[pyarrow]'
   rot_applied: 'f4[pyarrow]'

src/lvmguider/guider.py

@@ -22,10 +22,11 @@
 
 from sdsstools.time import get_sjd
 
-from lvmguider import __version__
+from lvmguider import __version__, config
 from lvmguider.dataclasses import CameraSolution, GuiderSolution
 from lvmguider.maskbits import GuiderStatus
 from lvmguider.tools import (
+    dataframe_to_database,
     elapsed_time,
     estimate_zeropoint,
     get_dark_subtracted_data,
@@ -646,6 +647,7 @@ async def update_fits(self, guider_solution: GuiderSolution):
         """Updates the ``lvm.agcam`` files and creates the ``lvm.guider`` file."""
 
         coros = []
+        update_agcam_database: bool = False
 
         assert guider_solution.sources is not None
 
@@ -674,6 +676,7 @@ async def update_fits(self, guider_solution: GuiderSolution):
             }
 
             coros.append(run_in_executor(update_fits, file, "PROC", header=proc_update))
+            update_agcam_database = True
 
             # Update the sources file for each camera.
             sources_cam_path = file.with_suffix(".parquet")
@@ -742,3 +745,19 @@ async def update_fits(self, guider_solution: GuiderSolution):
 
         with elapsed_time(self.command, "update PROC HDU and create lvm.guider file"):
             await asyncio.gather(*coros)
+
+        # If we have solved the cameras, update the database.
+        if update_agcam_database:
+            with elapsed_time(self.command, "updating lvm.agcam DB records"):
+                camera_frame_dfs = []
+                for solution in guider_solution.solutions:
+                    fn = solution.path.absolute()
+                    cs = CameraSolution.open(fn)
+                    camera_frame_dfs.append(cs.to_framedata().to_dataframe())
+
+                await run_in_executor(
+                    dataframe_to_database,
+                    pandas.concat(camera_frame_dfs, axis=0, ignore_index=True),
+                    config["database"]["agcam_frame_table"],
+                    delete_columns=["frameno", "telescope", "camera"],
+                )