From 31e383a57eb1005da449afef334130812ba425d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Sun, 4 Feb 2024 23:43:54 +0000 Subject: [PATCH] Ingest each AG frame to DB after exposure --- src/lvmguider/cameras.py | 19 ++++++++++++++++++- src/lvmguider/coadd.py | 28 ---------------------------- src/lvmguider/dataclasses.py | 11 ++++++----- src/lvmguider/etc/datamodel.yml | 8 ++++---- src/lvmguider/guider.py | 21 ++++++++++++++++++++- 5 files changed, 48 insertions(+), 39 deletions(-) diff --git a/src/lvmguider/cameras.py b/src/lvmguider/cameras.py index ce91ef7..0711cd8 100644 --- a/src/lvmguider/cameras.py +++ b/src/lvmguider/cameras.py @@ -21,10 +21,12 @@ from sdsstools.time import get_sjd -from lvmguider import __version__ +from lvmguider import __version__, config +from lvmguider.dataclasses import CameraSolution from lvmguider.extraction import extract_sources as extract_sources_func from lvmguider.maskbits import GuiderStatus from lvmguider.tools import ( + dataframe_to_database, elapsed_time, header_from_model, run_in_executor, @@ -197,6 +199,7 @@ async def expose( if not command.actor.status & GuiderStatus.NON_IDLE: command.actor.status |= GuiderStatus.IDLE + # Update FITS file PROC extension. with elapsed_time(command, "updating lvm.agcam file"): await asyncio.gather( *[ @@ -205,6 +208,20 @@ async def expose( ] ) + # Store data to DB. + with elapsed_time(command, "storing lvm.agcam to DB"): + camera_frame_dfs = [] + for fn in filenames: + cs = CameraSolution.open(fn) + camera_frame_dfs.append(cs.to_framedata().to_dataframe()) + + await run_in_executor( + dataframe_to_database, + pandas.concat(camera_frame_dfs, axis=0, ignore_index=True), + config["database"]["agcam_frame_table"], + delete_columns=["frameno", "telescope", "camera"], + ) + return (list(filenames), next_seqno, list(sources) if extract_sources else None) def reset_seqno(self): diff --git a/src/lvmguider/coadd.py b/src/lvmguider/coadd.py index fde5129..91e38a0 100644 --- a/src/lvmguider/coadd.py +++ b/src/lvmguider/coadd.py @@ -54,7 +54,6 @@ get_raw_extension, get_spec_frameno, header_from_model, - isot_to_sjd, nan_or_none, polyfit_with_sigclip, sort_files_by_camera, @@ -1605,31 +1604,6 @@ def coadd_to_database( except Exception as err: log.error(f"Failed loading frames to database: {str(err).strip()}") - # Load AG frames. - ag_frames_file = coadd_file.with_name(coadd_file.stem + "_frames.parquet") - if ag_frames_file.exists(): - log.debug("Loading AG frames to database.") - try: - ag_frames = pandas.read_parquet(ag_frames_file, dtype_backend="pyarrow") - if exposure_no: - ag_frames["exposure_no"] = exposure_no - if len(ag_frames) > 0: - ag_frames_mjd = isot_to_sjd(ag_frames["date_obs"].iloc[0]) - ag_frames["mjd"] = ag_frames_mjd - ag_frames["stacked"] = ag_frames.stacked.astype(bool) - ag_frames["solved"] = ag_frames.solved.astype(bool) - table_name = config["database"]["agcam_frame_table"] - dataframe_to_database( - ag_frames, - table_name, - delete_columns=delete_columns, - **db_connection_params, - ) - except Exception as err: - log.error(f"Failed loading frames to database: {str(err).strip()}") - else: - log.error(f"Cannot find AG frames file {ag_frames_file}") - # Load guider data. guider_frames_file = coadd_file.with_name(coadd_file.stem + "_guiderdata.parquet") if guider_frames_file.exists(): @@ -1643,8 +1617,6 @@ def coadd_to_database( guider_frames["exposure_no"] = exposure_no if len(guider_frames) > 0: guider_frames["mjd"] = sjd - guider_frames["solved"] = guider_frames.solved.astype(bool) - guider_frames["applied"] = guider_frames.applied.astype(bool) table_name = config["database"]["guider_frame_table"] dataframe_to_database( guider_frames, diff --git a/src/lvmguider/dataclasses.py b/src/lvmguider/dataclasses.py index f6f5fe1..1dc0656 100644 --- a/src/lvmguider/dataclasses.py +++ b/src/lvmguider/dataclasses.py @@ -73,7 +73,8 @@ def zero_point(self): """Returns the median zero point.""" if self.sources is not None: - return float(self.sources.zp.dropna().median()) + if len(data := self.sources.zp.dropna()) > 0: + return float(data.median()) return numpy.nan @@ -409,8 +410,8 @@ def to_dataframe(self): dec=numpy.float64(pointing[1]), pa=numpy.float32(self.solution.pa), zero_point=numpy.float32(self.solution.zero_point), - stacked=int(self.stacked), - solved=int(self.solution.solved), + stacked=bool(self.stacked), + solved=bool(self.solution.solved), wcs_mode=self.solution.wcs_mode, ) df.loc[0, list(new_row)] = list(new_row.values()) @@ -549,7 +550,7 @@ def guider_data(self): fwhm=numpy.float32(gs.fwhm), pa=numpy.float32(pa), zero_point=numpy.float32(gs.zero_point), - solved=int(gs.solved), + solved=bool(gs.solved), n_cameras_solved=int(gs.n_cameras_solved), guide_mode=gs.guide_mode, x_ff_pixel=numpy.float32(gs.guide_pixel[0]), @@ -564,7 +565,7 @@ def guider_data(self): pa_off=numpy.float32(gs.pa_off), axis0_off=numpy.float32(gs.axis0_off), axis1_off=numpy.float32(gs.axis1_off), - applied=int(gs.correction_applied), + applied=bool(gs.correction_applied), ax0_applied=numpy.float32(gs.correction[0]), ax1_applied=numpy.float32(gs.correction[1]), rot_applied=numpy.float32(gs.correction[2]), diff --git a/src/lvmguider/etc/datamodel.yml b/src/lvmguider/etc/datamodel.yml index 8962ce9..eaa8ced 100644 --- a/src/lvmguider/etc/datamodel.yml +++ b/src/lvmguider/etc/datamodel.yml @@ -132,8 +132,8 @@ FRAMEDATA: dec: 'f8[pyarrow]' pa: 'f4[pyarrow]' zero_point: 'f4[pyarrow]' - stacked: 'int8[pyarrow]' - solved: 'int8[pyarrow]' + stacked: 'bool[pyarrow]' + solved: 'bool[pyarrow]' wcs_mode: 'string[pyarrow]' GUIDERDATA_FRAME: @@ -141,7 +141,7 @@ GUIDERDATA_FRAME: telescope: 'string[pyarrow]' mjd: 'int32[pyarrow]' date_obs: 'string[pyarrow]' - solved: 'int8[pyarrow]' + solved: 'bool[pyarrow]' n_cameras_solved: 'int8[pyarrow]' guide_mode: 'string[pyarrow]' fwhm: 'f4[pyarrow]' @@ -159,7 +159,7 @@ GUIDERDATA_FRAME: pa_off: 'f4[pyarrow]' axis0_off: 'f4[pyarrow]' axis1_off: 'f4[pyarrow]' - applied: 'int8[pyarrow]' + applied: 'bool[pyarrow]' ax0_applied: 'f4[pyarrow]' ax1_applied: 'f4[pyarrow]' rot_applied: 'f4[pyarrow]' diff --git a/src/lvmguider/guider.py b/src/lvmguider/guider.py index 13b6a6e..76583b0 100644 --- a/src/lvmguider/guider.py +++ b/src/lvmguider/guider.py @@ -22,10 +22,11 @@ from sdsstools.time import get_sjd -from lvmguider import __version__ +from lvmguider import __version__, config from lvmguider.dataclasses import CameraSolution, GuiderSolution from lvmguider.maskbits import GuiderStatus from lvmguider.tools import ( + dataframe_to_database, elapsed_time, estimate_zeropoint, get_dark_subtracted_data, @@ -646,6 +647,7 @@ async def update_fits(self, guider_solution: GuiderSolution): """Updates the ``lvm.agcam`` files and creates the ``lvm.guider`` file.""" coros = [] + update_agcam_database: bool = False assert guider_solution.sources is not None @@ -674,6 +676,7 @@ async def update_fits(self, guider_solution: GuiderSolution): } coros.append(run_in_executor(update_fits, file, "PROC", header=proc_update)) + update_agcam_database = True # Update the sources file for each camera. sources_cam_path = file.with_suffix(".parquet") @@ -742,3 +745,19 @@ async def update_fits(self, guider_solution: GuiderSolution): with elapsed_time(self.command, "update PROC HDU and create lvm.guider file"): await asyncio.gather(*coros) + + # If we have solved the cameras, update the database. + if update_agcam_database: + with elapsed_time(self.command, "updating lvm.agcam DB records"): + camera_frame_dfs = [] + for solution in guider_solution.solutions: + fn = solution.path.absolute() + cs = CameraSolution.open(fn) + camera_frame_dfs.append(cs.to_framedata().to_dataframe()) + + await run_in_executor( + dataframe_to_database, + pandas.concat(camera_frame_dfs, axis=0, ignore_index=True), + config["database"]["agcam_frame_table"], + delete_columns=["frameno", "telescope", "camera"], + )