Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Moves reductions to a new tile id group sub-directory #43

Merged
merged 7 commits into from
Nov 29, 2023
Merged

Moves reductions to a new tile id group sub-directory #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d69d953
Merge branch 'master' of github.com:sdss/lvmdrp
havok2063 Nov 28, 2023
7228cfe
Merge branch 'master' of github.com:sdss/lvmdrp
havok2063 Nov 28, 2023
16976d8
updating code to use new tilegrp redux subdirectory
havok2063 Nov 28, 2023
0711eee
updating docstring
havok2063 Nov 28, 2023
44b1789
fixing and adding tests
havok2063 Nov 28, 2023
5edf144
adding click command to clean out DRP reductions
havok2063 Nov 28, 2023
f13eccb
changing default tile in places
havok2063 Nov 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions bin/build_super_arc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def _header_fix(fits_file, target_mjd, target_tileid=None):
f[0].header["MJD"] = target_mjd
if target_tileid is not None:
f[0].header["TILE_ID"] = target_tileid

f.writeto(fits_file, overwrite=True)

def parse_arguments(args=None):
Expand All @@ -55,7 +55,7 @@ def parse_arguments(args=None):
if __name__ == "__main__":
# parse arguments
cmd_args = parse_arguments(args=sys.argv[1:])

# extract metadata for target MJDs
raw_paths = md.locate_new_frames(hemi="s", mjd=cmd_args.mjds, camera="*", expnum="????????")
_ = md.extract_metadata(raw_paths)
Expand All @@ -67,7 +67,7 @@ if __name__ == "__main__":
# get max MJD and tileid for new supers
target_mjd = arcs.mjd.max()
target_tileid = arcs.loc[arcs.mjd == target_mjd].tileid.max()

# initialize the masters paths list
masters_paths = []

Expand All @@ -76,10 +76,10 @@ if __name__ == "__main__":
# set current lamp status
lamps_current = lamps_status.copy()
lamps_current[lamp] = True

# group arcs into analogs of current lamp
analogs, cpaths, mpaths = md.get_analog_groups(tileid=1111, mjd=target_mjd, imagetyp="arc", **lamps_current)
analogs, cpaths, mpaths = md.get_analog_groups(tileid=11111, mjd=target_mjd, imagetyp="arc", **lamps_current)

for i, (keys, analog) in enumerate(analogs.items()):
# unpack keys
tileid, mjd, imagetyp, camera = keys
Expand All @@ -88,13 +88,13 @@ if __name__ == "__main__":
mpixmask = path.full("lvm_master", drpver=drpver, kind="mpixmask", **masters["pixmask"])
mbias = path.full("lvm_master", drpver=drpver, kind="mbias", **masters["bias"])
mdark = path.full("lvm_master", drpver=drpver, kind="mdark", **masters["dark"])

# preproc and detrend raw arcs
for j, (_, arc) in enumerate(analog.iterrows()):
rpath = path.full("lvm_raw", camspec=arc.camera, **arc.to_dict())
ppath = path.full("lvm_anc", drpver=drpver, kind="p", imagetype=arc.imagetyp, **arc.to_dict())
cpath = cpaths[keys][j]

os.makedirs(os.path.dirname(ppath), exist_ok=True)
if os.path.isfile(ppath):
log.info(f"skipping {ppath}, file already exist")
Expand All @@ -104,7 +104,7 @@ if __name__ == "__main__":
log.info(f"skipping {cpath}, file already exist")
else:
imageMethod.detrend_frame(in_image=ppath, out_image=cpath, in_bias=mbias, in_dark=mdark, in_slitmap=Table(drp.fibermap.data))

# define master path
mpath = mpaths[i].replace("marc", f"marc_{lamp}")
# create master arcs
Expand All @@ -118,15 +118,15 @@ if __name__ == "__main__":

# update masters paths list
masters_paths.append(mpath)

# extract frames
master_paths = [os.path.join(root, file) for root, _, files in os.walk(os.getenv("LVM_SPECTRO_REDUX")) for file in files if file.startswith("lvm-marc_") and file.endswith(".fits")]
master_arcs = md.extract_metadata(kind="master", frames_paths=master_paths)
for _, marc in master_arcs.iterrows():
lamp = marc[lamps].astype(int).idxmax(0)
mpath = path.full("lvm_master", drpver=drpver, kind=f"marc_{lamp}", **marc.to_dict())
xpath = path.full("lvm_master", drpver=drpver, kind=f"xmarc_{lamp}", **marc.to_dict())

# match masters
masters = md.match_master_metadata(target_imagetyp="arc", target_mjd=target_mjd, target_camera=marc.camera)

Expand All @@ -136,7 +136,7 @@ if __name__ == "__main__":

# update masters paths list
masters_paths.append(xpath)

# update masters metadata
new_masters = md.extract_metadata(kind="master", frames_paths=masters_paths)
print(new_masters.to_string())
24 changes: 12 additions & 12 deletions bin/build_super_trace
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def _header_fix(fits_file, target_mjd, target_tileid=None):
f[0].header["MJD"] = target_mjd
if target_tileid is not None:
f[0].header["TILE_ID"] = target_tileid

f.writeto(fits_file, overwrite=True)

def parse_arguments(args=None):
Expand All @@ -78,7 +78,7 @@ def parse_arguments(args=None):
if __name__ == "__main__":
# parse arguments
cmd_args = parse_arguments(args=sys.argv[1:])

# extract metadata for target MJDs
raw_paths = md.locate_new_frames(hemi="s", mjd=cmd_args.mjds, camera="*", expnum="????????")
_ = md.extract_metadata(raw_paths)
Expand All @@ -90,7 +90,7 @@ if __name__ == "__main__":
# get max MJD and tileid for new supers
target_mjd = flats.mjd.max()
target_tileid = flats.loc[flats.mjd == target_mjd].tileid.max()

# initialize the masters paths list
masters_paths = []

Expand All @@ -107,10 +107,10 @@ if __name__ == "__main__":
else:
lamps_current = {"ldls": False, "quartz": True}
lamp = "quartz"

# group flats into analogs of current lamp
analogs, cpaths, mpaths = md.get_analog_groups(tileid=1111, mjd=target_mjd, imagetyp="flat", camera=camera, **lamps_current)
analogs, cpaths, mpaths = md.get_analog_groups(tileid=11111, mjd=target_mjd, imagetyp="flat", camera=camera, **lamps_current)

for i, (keys, analog) in enumerate(analogs.items()):
# unpack keys
tileid, mjd, imagetyp, camera = keys
Expand All @@ -119,13 +119,13 @@ if __name__ == "__main__":
mpixmask = path.full("lvm_master", drpver=drpver, kind="mpixmask", **masters["pixmask"])
mbias = path.full("lvm_master", drpver=drpver, kind="mbias", **masters["bias"])
mdark = path.full("lvm_master", drpver=drpver, kind="mdark", **masters["dark"])

# preproc and detrend raw flats
for j, (_, flat) in enumerate(analog.iterrows()):
rpath = path.full("lvm_raw", camspec=flat.camera, **flat.to_dict())
ppath = path.full("lvm_anc", drpver=drpver, kind="p", imagetype=flat.imagetyp, **flat.to_dict())
cpath = cpaths[keys][j]

os.makedirs(os.path.dirname(ppath), exist_ok=True)
if os.path.isfile(ppath):
log.info(f"skipping {ppath}, file already exist")
Expand All @@ -135,7 +135,7 @@ if __name__ == "__main__":
log.info(f"skipping {cpath}, file already exist")
else:
imageMethod.detrend_frame(in_image=ppath, out_image=cpath, in_bias=mbias, in_dark=mdark, in_slitmap=Table(drp.fibermap.data))

# define master path
mpath = path.full("lvm_master", drpver=drpver, kind=f"mflat_{lamp}", tileid=target_tileid, mjd=target_mjd, camera=camera)
# create master flats
Expand All @@ -146,15 +146,15 @@ if __name__ == "__main__":

# update masters paths list
masters_paths.append(mpath)

# extract frames
master_paths = [os.path.join(root, file) for root, _, files in os.walk(os.getenv("LVM_SPECTRO_REDUX")) for file in files if file.startswith("lvm-mflat_") and file.endswith(".fits")]
master_flats = md.extract_metadata(kind="master", frames_paths=master_paths)
for _, mflat in master_flats.iterrows():
lamp = mflat[lamps].astype(int).idxmax(0)
mpath = path.full("lvm_master", drpver=drpver, kind=f"mflat_{lamp}", **mflat.to_dict())
xpath = path.full("lvm_master", drpver=drpver, kind=f"xmflat_{lamp}", **mflat.to_dict())

mtrace_peak = path.full("lvm_master", drpver=drpver, kind="mtrace", **mflat.to_dict())
mtrace_fwhm = path.full("lvm_master", drpver=drpver, kind="mfwhm", **mflat.to_dict())
# run tracing
Expand All @@ -166,7 +166,7 @@ if __name__ == "__main__":

# update masters paths list
# masters_paths.append(xpath)

# update masters metadata
new_masters = md.extract_metadata(kind="master", frames_paths=masters_paths)
print(new_masters.to_string())
15 changes: 15 additions & 0 deletions bin/drp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
# encoding: utf-8
#

import os
import pathlib
import shutil

import click
import cloup
from cloup.constraints import mutually_exclusive, RequireExactly, IsSet, If
Expand Down Expand Up @@ -123,5 +127,16 @@ def quick_reduction(expnum: int, use_fiducial_master: bool, skip_sky_subtraction
cli.add_command(quick_reduction)


@cli.command('erase', short_help='Remove the DRP reductions')
@click.option('-d', '--drpver', type=str, help='the DRP version', required=True)
def erase(drpver: str):
""" Clean out the DRP reductions for a given version """
path = pathlib.Path(os.getenv('LVM_SPECTRO_REDUX')) / drpver
if not path.exists():
click.echo(f'Path {path} does not exist.')
return
shutil.rmtree(path)


if __name__ == "__main__":
cli()
26 changes: 13 additions & 13 deletions python/lvmdrp/core/image.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ def __truediv__(self, other):
new_error = self._error / other
else:
new_error = None

img = copy(self)
img.setData(data=new_data, error=new_error)
return img
Expand Down Expand Up @@ -804,11 +804,11 @@ def convertUnit(self, to, assume="adu", gain_field="GAIN", inplace=False):
# early return if no data or header to compute conversion
if new_image._header is None or new_image._data is None:
return new_image

current = self._header.get("BUNIT", assume)
if current == to:
return new_image

if current != to:
exptime = self.getHdrValue("EXPTIME")
gains = self.getHdrValue(f"AMP? {gain_field}")
Expand All @@ -829,7 +829,7 @@ def convertUnit(self, to, assume="adu", gain_field="GAIN", inplace=False):
factor = exptime
else:
raise ValueError(f"Cannot convert from {current} to {to}")

new_image.setSection(
section=sects[i],
subimg=new_image.getSection(section=sects[i]) * factor,
Expand Down Expand Up @@ -1117,13 +1117,13 @@ def writeFitsData(
hdu = pyfits.PrimaryHDU(self._error)
elif extension_error > 0 and extension_error is not None:
hdus[extension_error] = pyfits.ImageHDU(self._error, name="ERROR")

# frames hdu
if extension_frames == 0:
hdu = pyfits.PrimaryHDU(self._individual_frames)
elif extension_frames > 0 and extension_frames is not None:
hdus[extension_frames] = pyfits.BinTableHDU(self._individual_frames, name="FRAMES")

# slitmap hdu
if extension_slitmap == 0:
hdu = pyfits.PrimaryHDU(self._slitmap)
Expand Down Expand Up @@ -1695,7 +1695,7 @@ def fitPoly(self, axis="y", order=4, plot=-1):
new_mask = numpy.isnan(fit_result)
else:
new_mask = None

new_img = copy(self)
new_img.setData(data=fit_result, mask=new_mask)
return new_img
Expand Down Expand Up @@ -1743,7 +1743,7 @@ def traceFWHM(
1
] # traceFWHM.setSlice(i, axis='y', data = fwhm_fit[0], mask = fwhm_fit[1]) # insert the result into the trace mask
# return traceFWHM
return (fwhm, mask)
return (fwhm, mask)


def extractSpecAperture(self, TraceMask, aperture):
Expand Down Expand Up @@ -1784,7 +1784,7 @@ def extractSpecAperture(self, TraceMask, aperture):
)
if self._mask is not None:
mask[good_pix[:, i], i] = numpy.sum(self._mask[:, i][pixels], 1) > 0

# update mask with trace mask
mask |= bad_pix
return data, error, mask
Expand Down Expand Up @@ -2374,7 +2374,7 @@ def setIndividualFrames(self, images):
self._individual_frames = Table(names=["TILEID", "MJD", "EXPNUM", "SPEC", "CAMERA", "EXPTIME"], dtype=(int, int, int, str, str, float))
for img in images:
self._individual_frames.add_row([
img._header.get("TILEID", 1111),
img._header.get("TILEID", 11111),
img._header.get("MJD"),
img._header.get("EXPOSURE"),
img._header.get("SPEC"),
Expand All @@ -2384,7 +2384,7 @@ def setIndividualFrames(self, images):

def getSlitmap(self):
return self._slitmap

def setSlitmap(self, slitmap):
self._slitmap = slitmap

Expand Down Expand Up @@ -2597,15 +2597,15 @@ def combineImages(
new_header["ISMASTER"] = (True, "Is this a combined (master) frame")
new_header["NFRAMES"] = (nexp, "Number of exposures combined")
new_header["STATCOMB"] = (method, "Statistic used to combine images")

# add combined lamps to header
if images[0]._header["IMAGETYP"] == "flat":
lamps = CON_LAMPS
elif images[0]._header["IMAGETYP"] == "arc":
lamps = ARC_LAMPS
else:
lamps = []

if lamps:
new_lamps = set()
for image in images:
Expand Down
Loading
Loading