diff --git a/bin/build_super_arc b/bin/build_super_arc index 3c197c28..f4bfc0c9 100644 --- a/bin/build_super_arc +++ b/bin/build_super_arc @@ -35,7 +35,7 @@ def _header_fix(fits_file, target_mjd, target_tileid=None): f[0].header["MJD"] = target_mjd if target_tileid is not None: f[0].header["TILE_ID"] = target_tileid - + f.writeto(fits_file, overwrite=True) def parse_arguments(args=None): @@ -55,7 +55,7 @@ def parse_arguments(args=None): if __name__ == "__main__": # parse arguments cmd_args = parse_arguments(args=sys.argv[1:]) - + # extract metadata for target MJDs raw_paths = md.locate_new_frames(hemi="s", mjd=cmd_args.mjds, camera="*", expnum="????????") _ = md.extract_metadata(raw_paths) @@ -67,7 +67,7 @@ if __name__ == "__main__": # get max MJD and tileid for new supers target_mjd = arcs.mjd.max() target_tileid = arcs.loc[arcs.mjd == target_mjd].tileid.max() - + # initialize the masters paths list masters_paths = [] @@ -76,10 +76,10 @@ if __name__ == "__main__": # set current lamp status lamps_current = lamps_status.copy() lamps_current[lamp] = True - + # group arcs into analogs of current lamp - analogs, cpaths, mpaths = md.get_analog_groups(tileid=1111, mjd=target_mjd, imagetyp="arc", **lamps_current) - + analogs, cpaths, mpaths = md.get_analog_groups(tileid=11111, mjd=target_mjd, imagetyp="arc", **lamps_current) + for i, (keys, analog) in enumerate(analogs.items()): # unpack keys tileid, mjd, imagetyp, camera = keys @@ -88,13 +88,13 @@ if __name__ == "__main__": mpixmask = path.full("lvm_master", drpver=drpver, kind="mpixmask", **masters["pixmask"]) mbias = path.full("lvm_master", drpver=drpver, kind="mbias", **masters["bias"]) mdark = path.full("lvm_master", drpver=drpver, kind="mdark", **masters["dark"]) - + # preproc and detrend raw arcs for j, (_, arc) in enumerate(analog.iterrows()): rpath = path.full("lvm_raw", camspec=arc.camera, **arc.to_dict()) ppath = path.full("lvm_anc", drpver=drpver, kind="p", imagetype=arc.imagetyp, **arc.to_dict()) cpath = cpaths[keys][j] - + os.makedirs(os.path.dirname(ppath), exist_ok=True) if os.path.isfile(ppath): log.info(f"skipping {ppath}, file already exist") @@ -104,7 +104,7 @@ if __name__ == "__main__": log.info(f"skipping {cpath}, file already exist") else: imageMethod.detrend_frame(in_image=ppath, out_image=cpath, in_bias=mbias, in_dark=mdark, in_slitmap=Table(drp.fibermap.data)) - + # define master path mpath = mpaths[i].replace("marc", f"marc_{lamp}") # create master arcs @@ -118,7 +118,7 @@ if __name__ == "__main__": # update masters paths list masters_paths.append(mpath) - + # extract frames master_paths = [os.path.join(root, file) for root, _, files in os.walk(os.getenv("LVM_SPECTRO_REDUX")) for file in files if file.startswith("lvm-marc_") and file.endswith(".fits")] master_arcs = md.extract_metadata(kind="master", frames_paths=master_paths) @@ -126,7 +126,7 @@ if __name__ == "__main__": lamp = marc[lamps].astype(int).idxmax(0) mpath = path.full("lvm_master", drpver=drpver, kind=f"marc_{lamp}", **marc.to_dict()) xpath = path.full("lvm_master", drpver=drpver, kind=f"xmarc_{lamp}", **marc.to_dict()) - + # match masters masters = md.match_master_metadata(target_imagetyp="arc", target_mjd=target_mjd, target_camera=marc.camera) @@ -136,7 +136,7 @@ if __name__ == "__main__": # update masters paths list masters_paths.append(xpath) - + # update masters metadata new_masters = md.extract_metadata(kind="master", frames_paths=masters_paths) print(new_masters.to_string()) diff --git a/bin/build_super_trace b/bin/build_super_trace index 73829a7c..ad5e73ba 100644 --- a/bin/build_super_trace +++ b/bin/build_super_trace @@ -58,7 +58,7 @@ def _header_fix(fits_file, target_mjd, target_tileid=None): f[0].header["MJD"] = target_mjd if target_tileid is not None: f[0].header["TILE_ID"] = target_tileid - + f.writeto(fits_file, overwrite=True) def parse_arguments(args=None): @@ -78,7 +78,7 @@ def parse_arguments(args=None): if __name__ == "__main__": # parse arguments cmd_args = parse_arguments(args=sys.argv[1:]) - + # extract metadata for target MJDs raw_paths = md.locate_new_frames(hemi="s", mjd=cmd_args.mjds, camera="*", expnum="????????") _ = md.extract_metadata(raw_paths) @@ -90,7 +90,7 @@ if __name__ == "__main__": # get max MJD and tileid for new supers target_mjd = flats.mjd.max() target_tileid = flats.loc[flats.mjd == target_mjd].tileid.max() - + # initialize the masters paths list masters_paths = [] @@ -107,10 +107,10 @@ if __name__ == "__main__": else: lamps_current = {"ldls": False, "quartz": True} lamp = "quartz" - + # group flats into analogs of current lamp - analogs, cpaths, mpaths = md.get_analog_groups(tileid=1111, mjd=target_mjd, imagetyp="flat", camera=camera, **lamps_current) - + analogs, cpaths, mpaths = md.get_analog_groups(tileid=11111, mjd=target_mjd, imagetyp="flat", camera=camera, **lamps_current) + for i, (keys, analog) in enumerate(analogs.items()): # unpack keys tileid, mjd, imagetyp, camera = keys @@ -119,13 +119,13 @@ if __name__ == "__main__": mpixmask = path.full("lvm_master", drpver=drpver, kind="mpixmask", **masters["pixmask"]) mbias = path.full("lvm_master", drpver=drpver, kind="mbias", **masters["bias"]) mdark = path.full("lvm_master", drpver=drpver, kind="mdark", **masters["dark"]) - + # preproc and detrend raw flats for j, (_, flat) in enumerate(analog.iterrows()): rpath = path.full("lvm_raw", camspec=flat.camera, **flat.to_dict()) ppath = path.full("lvm_anc", drpver=drpver, kind="p", imagetype=flat.imagetyp, **flat.to_dict()) cpath = cpaths[keys][j] - + os.makedirs(os.path.dirname(ppath), exist_ok=True) if os.path.isfile(ppath): log.info(f"skipping {ppath}, file already exist") @@ -135,7 +135,7 @@ if __name__ == "__main__": log.info(f"skipping {cpath}, file already exist") else: imageMethod.detrend_frame(in_image=ppath, out_image=cpath, in_bias=mbias, in_dark=mdark, in_slitmap=Table(drp.fibermap.data)) - + # define master path mpath = path.full("lvm_master", drpver=drpver, kind=f"mflat_{lamp}", tileid=target_tileid, mjd=target_mjd, camera=camera) # create master flats @@ -146,7 +146,7 @@ if __name__ == "__main__": # update masters paths list masters_paths.append(mpath) - + # extract frames master_paths = [os.path.join(root, file) for root, _, files in os.walk(os.getenv("LVM_SPECTRO_REDUX")) for file in files if file.startswith("lvm-mflat_") and file.endswith(".fits")] master_flats = md.extract_metadata(kind="master", frames_paths=master_paths) @@ -154,7 +154,7 @@ if __name__ == "__main__": lamp = mflat[lamps].astype(int).idxmax(0) mpath = path.full("lvm_master", drpver=drpver, kind=f"mflat_{lamp}", **mflat.to_dict()) xpath = path.full("lvm_master", drpver=drpver, kind=f"xmflat_{lamp}", **mflat.to_dict()) - + mtrace_peak = path.full("lvm_master", drpver=drpver, kind="mtrace", **mflat.to_dict()) mtrace_fwhm = path.full("lvm_master", drpver=drpver, kind="mfwhm", **mflat.to_dict()) # run tracing @@ -166,7 +166,7 @@ if __name__ == "__main__": # update masters paths list # masters_paths.append(xpath) - + # update masters metadata new_masters = md.extract_metadata(kind="master", frames_paths=masters_paths) print(new_masters.to_string()) diff --git a/bin/drp b/bin/drp index 040fbf44..e1dac83d 100755 --- a/bin/drp +++ b/bin/drp @@ -2,6 +2,10 @@ # encoding: utf-8 # +import os +import pathlib +import shutil + import click import cloup from cloup.constraints import mutually_exclusive, RequireExactly, IsSet, If @@ -123,5 +127,16 @@ def quick_reduction(expnum: int, use_fiducial_master: bool, skip_sky_subtraction cli.add_command(quick_reduction) +@cli.command('erase', short_help='Remove the DRP reductions') +@click.option('-d', '--drpver', type=str, help='the DRP version', required=True) +def erase(drpver: str): + """ Clean out the DRP reductions for a given version """ + path = pathlib.Path(os.getenv('LVM_SPECTRO_REDUX')) / drpver + if not path.exists(): + click.echo(f'Path {path} does not exist.') + return + shutil.rmtree(path) + + if __name__ == "__main__": cli() diff --git a/python/lvmdrp/core/image.py b/python/lvmdrp/core/image.py index 6d879fe9..5e3a19ac 100644 --- a/python/lvmdrp/core/image.py +++ b/python/lvmdrp/core/image.py @@ -386,7 +386,7 @@ def __truediv__(self, other): new_error = self._error / other else: new_error = None - + img = copy(self) img.setData(data=new_data, error=new_error) return img @@ -804,11 +804,11 @@ def convertUnit(self, to, assume="adu", gain_field="GAIN", inplace=False): # early return if no data or header to compute conversion if new_image._header is None or new_image._data is None: return new_image - + current = self._header.get("BUNIT", assume) if current == to: return new_image - + if current != to: exptime = self.getHdrValue("EXPTIME") gains = self.getHdrValue(f"AMP? {gain_field}") @@ -829,7 +829,7 @@ def convertUnit(self, to, assume="adu", gain_field="GAIN", inplace=False): factor = exptime else: raise ValueError(f"Cannot convert from {current} to {to}") - + new_image.setSection( section=sects[i], subimg=new_image.getSection(section=sects[i]) * factor, @@ -1117,13 +1117,13 @@ def writeFitsData( hdu = pyfits.PrimaryHDU(self._error) elif extension_error > 0 and extension_error is not None: hdus[extension_error] = pyfits.ImageHDU(self._error, name="ERROR") - + # frames hdu if extension_frames == 0: hdu = pyfits.PrimaryHDU(self._individual_frames) elif extension_frames > 0 and extension_frames is not None: hdus[extension_frames] = pyfits.BinTableHDU(self._individual_frames, name="FRAMES") - + # slitmap hdu if extension_slitmap == 0: hdu = pyfits.PrimaryHDU(self._slitmap) @@ -1695,7 +1695,7 @@ def fitPoly(self, axis="y", order=4, plot=-1): new_mask = numpy.isnan(fit_result) else: new_mask = None - + new_img = copy(self) new_img.setData(data=fit_result, mask=new_mask) return new_img @@ -1743,7 +1743,7 @@ def traceFWHM( 1 ] # traceFWHM.setSlice(i, axis='y', data = fwhm_fit[0], mask = fwhm_fit[1]) # insert the result into the trace mask # return traceFWHM - return (fwhm, mask) + return (fwhm, mask) def extractSpecAperture(self, TraceMask, aperture): @@ -1784,7 +1784,7 @@ def extractSpecAperture(self, TraceMask, aperture): ) if self._mask is not None: mask[good_pix[:, i], i] = numpy.sum(self._mask[:, i][pixels], 1) > 0 - + # update mask with trace mask mask |= bad_pix return data, error, mask @@ -2374,7 +2374,7 @@ def setIndividualFrames(self, images): self._individual_frames = Table(names=["TILEID", "MJD", "EXPNUM", "SPEC", "CAMERA", "EXPTIME"], dtype=(int, int, int, str, str, float)) for img in images: self._individual_frames.add_row([ - img._header.get("TILEID", 1111), + img._header.get("TILEID", 11111), img._header.get("MJD"), img._header.get("EXPOSURE"), img._header.get("SPEC"), @@ -2384,7 +2384,7 @@ def setIndividualFrames(self, images): def getSlitmap(self): return self._slitmap - + def setSlitmap(self, slitmap): self._slitmap = slitmap @@ -2597,7 +2597,7 @@ def combineImages( new_header["ISMASTER"] = (True, "Is this a combined (master) frame") new_header["NFRAMES"] = (nexp, "Number of exposures combined") new_header["STATCOMB"] = (method, "Statistic used to combine images") - + # add combined lamps to header if images[0]._header["IMAGETYP"] == "flat": lamps = CON_LAMPS @@ -2605,7 +2605,7 @@ def combineImages( lamps = ARC_LAMPS else: lamps = [] - + if lamps: new_lamps = set() for image in images: diff --git a/python/lvmdrp/functions/run_drp.py b/python/lvmdrp/functions/run_drp.py index 8cdb62b1..cbfe628c 100644 --- a/python/lvmdrp/functions/run_drp.py +++ b/python/lvmdrp/functions/run_drp.py @@ -26,6 +26,7 @@ resample_wavelength, join_spec_channels, stack_rss) from lvmdrp.utils.metadata import (get_frames_metadata, get_master_metadata, extract_metadata, get_analog_groups, match_master_metadata, create_master_path) +from lvmdrp.utils.convert import tileid_grp from lvmdrp.functions.run_quickdrp import quick_science_reduction from lvmdrp import config, log, path, __version__ as drpver @@ -382,8 +383,8 @@ def create_output_path(kind: str, flavor: str, mjd: int, tileid: int, camera: st Creates the output file path for the science frames or the master arc/flats. For example, the extracted fiber spectra is - "1111/60115/ancillary/lvm-xobject-b1-00060115.fits" for science frames - or "1111/60115/calib/lvm-xmarc-b1.fits" for the master arc frame. + "11111/60115/ancillary/lvm-xobject-b1-00060115.fits" for science frames + or "11111/60115/calib/lvm-xmarc-b1.fits" for the master arc frame. Parameters ---------- @@ -784,9 +785,10 @@ def start_logging(mjd: int, tileid: int): tileid : int The tile ID of the observations """ + tilegrp = tileid_grp(tileid) lpath = (os.path.join(os.getenv('LVM_SPECTRO_REDUX'), - "{drpver}/{tileid}/{mjd}/lvm-drp-{tileid}-{mjd}.log")) - logpath = lpath.format(drpver=drpver, mjd=mjd, tileid=tileid) + "{drpver}/{tilegrp}/{tileid}/{mjd}/lvm-drp-{tileid}-{mjd}.log")) + logpath = lpath.format(drpver=drpver, mjd=mjd, tileid=tileid, tilegrp=tilegrp) logpath = pathlib.Path(logpath) # if logpath.exists(): @@ -1335,7 +1337,8 @@ def create_status_file(tileid: int, mjd: int, status: str = 'started'): status : str, optional the DRP status, by default 'started' """ - root = pathlib.Path(os.getenv("LVM_SPECTRO_REDUX")) / f'{drpver}' / f'{tileid}' / 'logs' + tilegrp = tileid_grp(tileid) + root = pathlib.Path(os.getenv("LVM_SPECTRO_REDUX")) / f'{drpver}/{tilegrp}/{tileid}/logs' root.mkdir(parents=True, exist_ok=True) path = root / f'lvm-drp-{tileid}-{mjd}.{status}' path.touch() @@ -1356,7 +1359,8 @@ def remove_status_file(tileid: int, mjd: int, remove_all: bool = False): remove_all : bool, optional Flag to remove all status files, by default False """ - root = pathlib.Path(os.getenv("LVM_SPECTRO_REDUX")) / f'{drpver}' / f'{tileid}' / 'logs' + tilegrp = tileid_grp(tileid) + root = pathlib.Path(os.getenv("LVM_SPECTRO_REDUX")) / f'{drpver}/{tilegrp}/{tileid}/logs' if remove_all: shutil.rmtree(root) @@ -1384,29 +1388,31 @@ def status_file_exists(tileid: int, mjd: int, status: str = 'started') -> bool: bool Flag if the file exists """ - root = pathlib.Path(os.getenv("LVM_SPECTRO_REDUX")) / f'{drpver}' / f'{tileid}' / 'logs' + tilegrp = tileid_grp(tileid) + root = pathlib.Path(os.getenv("LVM_SPECTRO_REDUX")) / f'{drpver}/{tilegrp}/{tileid}/logs' path = root / f'lvm-drp-{tileid}-{mjd}.{status}' return path.exists() def update_error_file(tileid: int, mjd: int, expnum: int, error: str, reset: bool = False): - """_summary_ + """ Update the DRP error file - _extended_summary_ + Appends to the "drp_errors.txt" file whenever + there is an error during a reduction. Parameters ---------- tileid : int - _description_ + the tile id mjd : int - _description_ + the MJD expnum : int - _description_ + the exposure number error : str - _description_ + the traceback reset : bool, optional - _description_, by default False + Flag to reset the text file, by default False """ path = pathlib.Path(os.getenv("LVM_SPECTRO_REDUX")) / f'{drpver}' / 'drp_errors.txt' diff --git a/python/lvmdrp/utils/convert.py b/python/lvmdrp/utils/convert.py index cc1b0243..5a678aa9 100644 --- a/python/lvmdrp/utils/convert.py +++ b/python/lvmdrp/utils/convert.py @@ -2,6 +2,7 @@ # encoding: utf-8 import pathlib +from typing import Union from astropy.time import Time @@ -108,3 +109,29 @@ def correct_sjd(path: pathlib.Path, sjd: int) -> int: # raw exp_sjd = int(path.parent.stem) return sjd if exp_sjd == sjd else exp_sjd + + +def tileid_grp(tileid: Union[int, str]) -> str: + """ Convert a tile id to a tile group + + This is for manual use without ``sdss_access``. If the + group definition changes, the definition in the + ``tilegrp`` function in ``sdss_access.path.path.py`` + also needs updating. + + The raw_metadata code uses a tileid of "*" for + pattern matching. In this case, we use "*XX" for + the tile group. + + Parameters + ---------- + tileid : Union[int, str] + the LVM tile id + + Returns + ------- + str + the LVM tile id group + """ + return '*XX' if tileid == '*' else f'{int(tileid) // 1000:0>4d}XX' + diff --git a/python/lvmdrp/utils/examples.py b/python/lvmdrp/utils/examples.py index eaf42569..e0aac6e0 100644 --- a/python/lvmdrp/utils/examples.py +++ b/python/lvmdrp/utils/examples.py @@ -137,7 +137,7 @@ def get_frames_metadata(mjd: Union[str, int] = None, path: str = None, suffix: s camera, expnum = parse_sdr_name(frame_path) spec = f"sp{camera[-1]}" exptime = header["EXPTIME"] - tileid = header.get("TILEID", 1111) + tileid = header.get("TILEID", 11111) qual = header.get("QUALITY", "excellent") frames_table.add_row([imagetyp, spec, mjd, camera, expnum, exptime, tileid, qual, str(frame_path), frame_path.stem]) diff --git a/python/lvmdrp/utils/metadata.py b/python/lvmdrp/utils/metadata.py index 00df7436..72972097 100644 --- a/python/lvmdrp/utils/metadata.py +++ b/python/lvmdrp/utils/metadata.py @@ -25,7 +25,7 @@ ) from lvmdrp import log, __version__, path from lvmdrp.utils.hdrfix import apply_hdrfix -from lvmdrp.utils.convert import dateobs_to_sjd, correct_sjd +from lvmdrp.utils.convert import dateobs_to_sjd, correct_sjd, tileid_grp DRPVER = __version__ @@ -57,6 +57,7 @@ ("status", ReductionStatus), ("drpqual", QualityFlag), ("name", str), + ("tilegrp", str) ] MASTER_METADATA_COLUMNS = [ ("tileid", int), @@ -80,6 +81,7 @@ ("drpqual", QualityFlag), ("nframes", int), ("name", str), + ("tilegrp", str) ] @@ -138,8 +140,9 @@ def _get_metadata_paths(tileid=None, mjd=None, kind="raw", filter_exist=True): raise ValueError( "`tileid` and `mjd` are needed to define a path for raw metadata" ) + tilegrp = tileid_grp(tileid) path_pattern = os.path.join( - METADATA_PATH, str(tileid), str(mjd), "raw_metadata.hdf5" + METADATA_PATH, tilegrp, str(tileid), str(mjd), "raw_metadata.hdf5" ) elif kind == "master": path_pattern = os.path.join(METADATA_PATH, "master_metadata.hdf5") @@ -576,9 +579,12 @@ def extract_metadata(frames_paths: list, kind: str = "raw") -> pd.DataFrame: # set on-lamp conditions onlamp = ["ON", True, 'T', 1] - # get the tile id; set null tile ids -999 to 1111 - tileid = header.get("TILE_ID") or header.get("TILEID", 1111) - tileid = 1111 if tileid in (-999, None) else tileid + # get the tile id; set null tile ids -999 to 11111 + tileid = header.get("TILE_ID") or header.get("TILEID", 11111) + tileid = 11111 if tileid in (-999, None) else tileid + + # get the tile group + tilegrp = tileid_grp(tileid) if kind == "raw": new_metadata[i] = [ @@ -604,6 +610,7 @@ def extract_metadata(frames_paths: list, kind: str = "raw") -> pd.DataFrame: header.get("DRPSTAT", ReductionStatus(0)), header.get("DRPQUAL", QualityFlag(0)), frame_path.stem, + tilegrp, ] elif kind == "master": new_metadata[i] = [ @@ -628,6 +635,7 @@ def extract_metadata(frames_paths: list, kind: str = "raw") -> pd.DataFrame: header.get("DRPQUAL", QualityFlag(0)), header.get("NFRAMES", 1), frame_path.stem, + tilegrp, ] # define dataframe diff --git a/tests/conftest.py b/tests/conftest.py index 1c1fb931..2d9cfada 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,7 +56,7 @@ def _datadir(mjds): @pytest.fixture(scope='module') def make_meta(): """ fixture factory to make a fake metadata frame """ - def _make_meta(expnum=6817, tileid=1111, mjd=61234): + def _make_meta(expnum=6817, tileid=11111, mjd=61234): defaults = {'hemi': 's', 'status': 0, 'stage': 1, 'qual': 0, 'quality': 'excellent', 'quartz': False, 'ldls': False, 'argon': False, 'xenon': False, 'krypton': False, 'hgne': False, 'neon': False, 'exptime': 900.0, 'imgtype': 'object'} @@ -67,6 +67,7 @@ def _make_meta(expnum=6817, tileid=1111, mjd=61234): defaults['expnum'] = expnum defaults['tileid'] = tileid + defaults['tilegrp'] = '0011XX' defaults['mjd'] = mjd defaults['rmjd'] = mjd return pd.DataFrame(defaults) @@ -96,7 +97,7 @@ def multimeta(make_multi): yield make_multi(expnum=[6818, 6819]) -def create_fake_fits(path, tileid=1111, mjd=61234, expnum=6817, cameras=None): +def create_fake_fits(path, tileid=11111, mjd=61234, expnum=6817, cameras=None): """ create a fake raw frame FITS file """ out = {} cameras = cameras or ['b1'] diff --git a/tests/functions/test_rundrp.py b/tests/functions/test_rundrp.py index c3c99685..b82866e2 100644 --- a/tests/functions/test_rundrp.py +++ b/tests/functions/test_rundrp.py @@ -16,9 +16,9 @@ @pytest.fixture() def status_file(): """ fixture to create a status file """ - create_status_file(1111, 61234, status='started') + create_status_file(11111, 61234, status='started') yield (pathlib.Path(os.getenv("LVM_SPECTRO_REDUX")) - / f'{drpver}/1111/logs/lvm-drp-1111-61234.started') + / f'{drpver}/0011XX/11111/logs/lvm-drp-11111-61234.started') @pytest.fixture() @@ -38,16 +38,16 @@ def test_create_status_file(status_file): def test_remove_status_file(status_file): """ test we can remove the status file """ - remove_status_file(1111, 61234) + remove_status_file(11111, 61234) assert not status_file.exists() def test_remove_status_all(status_file): """ test we can remove all the status files """ - create_status_file(1111, 61235, status='started') - remove_status_file(1111, 61234, remove_all=True) + create_status_file(11111, 61235, status='started') + remove_status_file(11111, 61234, remove_all=True) - path2 = pathlib.Path(os.getenv("LVM_SPECTRO_REDUX")) / f'{drpver}/1111/logs/lvm-drp-1111-61235.started' + path2 = pathlib.Path(os.getenv("LVM_SPECTRO_REDUX")) / f'{drpver}/0011XX/11111/logs/lvm-drp-11111-61235.started' assert not status_file.exists() assert not path2.exists() @@ -55,7 +55,7 @@ def test_remove_status_all(status_file): def test_status_exists(status_file): """ test we can check the status file """ - assert status_file_exists(1111, 61234, status='started') + assert status_file_exists(11111, 61234, status='started') def test_update_error_file(): @@ -70,12 +70,12 @@ def test_update_error_file(): ValueError: This is a bad error on b2 """ - update_error_file(1111, 61234, 1011, errors) + update_error_file(11111, 61234, 1011, errors) assert path.exists() with open(path) as f: data = f.read() - assert 'ERROR on tileid, mjd, exposure: 1111, 61234, 1011' in data + assert 'ERROR on tileid, mjd, exposure: 11111, 61234, 1011' in data assert 'This is a bad error on b2' in data diff --git a/tests/utils/test_convert.py b/tests/utils/test_convert.py new file mode 100644 index 00000000..84c91cdf --- /dev/null +++ b/tests/utils/test_convert.py @@ -0,0 +1,56 @@ +# encoding: utf-8 +# + +import os +import pathlib +import pytest + +from lvmdrp.utils.convert import (tileid_grp, correct_sjd, sjd_to_mjd, + mjd_to_sjd, dateobs_to_sjd, dateobs_to_mjd) + + +@pytest.mark.parametrize('tileid, exp', + [('11111', '0011XX'), + (11111, '0011XX'), + (1055360, '1055XX'), + ('*', '*XX')], + ids=['str', 'int', 'main', 'pattern']) +def test_tileid_grp(tileid, exp): + """ test we can correctly get the tile id group """ + grp = tileid_grp(tileid) + assert grp == exp + + +def test_sjd_to_mjd(): + """ test we can covert from SJD to MJD """ + out = sjd_to_mjd(60125) + assert out == 60124.6 + + +def test_mjd_to_sjd(): + """ test we can covert from MJD to SJD """ + out = mjd_to_sjd(60125) + assert out == 60125.4 + + +def test_dateobs_to_mjd(): + """ test we can covert from date time to MJD """ + out = int(dateobs_to_mjd('2023-06-19T22:52:00.981')) + assert out == 60114 + + +def test_dateobs_to_sjd(): + """ test we can covert from date time to SJD """ + out = int(dateobs_to_sjd('2023-06-19T22:52:00.981')) + assert out == 60115 + + +@pytest.mark.parametrize('stem', + ['sdR-s-b1-00006817.fits.gz', + 'calib/lvm-mbias-r1.fits'], + ids=['raw', 'master']) +def test_correct_sjd(stem): + """ test we can correct the SJD """ + path = pathlib.Path(os.getenv("LVM_DATA_S")) / '60112' / stem + out = correct_sjd(path, '60113') + assert out == 60112 diff --git a/tests/utils/test_metadata.py b/tests/utils/test_metadata.py index 8ccb71cd..f15cd022 100644 --- a/tests/utils/test_metadata.py +++ b/tests/utils/test_metadata.py @@ -24,7 +24,8 @@ def test_get_frames_metadata(make_fits): meta = meta.sort_values('camera') assert len(meta) == 3 assert 61234 in meta['mjd'].unique() - assert 1111 in meta['tileid'].unique() + assert 11111 in meta['tileid'].unique() + assert '0011XX' in meta['tilegrp'].unique() assert 6817 in meta['expnum'].unique() assert meta.iloc[0]['name'] == 'sdR-s-b1-00006817.fits' assert set(meta['camera']) == {'b1', 'b2', 'b3'}