Merge branch 'develop' into fix_spatflex

deprecated/arc_old.py

@@ -855,3 +855,53 @@ def saturation_mask(a, satlevel):
 
     return mask.astype(int)
 
+def mask_around_peaks(spec, inbpm):
+    """
+    Find peaks in the input spectrum and mask pixels around them.
+
+    All pixels to the left and right of a peak is masked until
+    a pixel has a lower value than the adjacent pixel. At this
+    point, we assume that spec has reached the noise level.
+
+    Parameters
+    ----------
+    spec: `numpy.ndarray`_
+        Spectrum (1D array) in counts
+    inbpm: `numpy.ndarray`_
+        Input bad pixel mask
+
+    Returns
+    -------
+    outbpm: `numpy.ndarray`_
+        Bad pixel mask with pixels around peaks masked
+    """
+    # Find the peak locations
+    pks = detect_peaks(spec)
+
+    # Initialise some useful variables and the output bpm
+    xarray = np.arange(spec.size)
+    specdiff = np.append(np.diff(spec), 0.0)
+    outbpm = inbpm.copy()
+
+    # Loop over the peaks and mask pixels around them
+    for i in range(len(pks)):
+        # Find all pixels to the left of the peak that are above the noise level
+        wl = np.where((xarray <= pks[i]) & (specdiff > 0.0))[0]
+        ww = (pks[i]-wl)[::-1]
+        # Find the first pixel to the left of the peak that is below the noise level
+        nmask = np.where(np.diff(ww) > 1)[0]
+        if nmask.size != 0 and nmask[0] > 5:
+            # Mask all pixels to the left of the peak
+            mini = max(0,wl.size-nmask[0]-1)
+            outbpm[wl[mini]:pks[i]] = True
+        # Find all pixels to the right of the peak that are above the noise level
+        ww = np.where((xarray >= pks[i]) & (specdiff < 0.0))[0]
+        # Find the first pixel to the right of the peak that is below the noise level
+        nmask = np.where(np.diff(ww) > 1)[0]
+        if nmask.size != 0 and nmask[0] > 5:
+            # Mask all pixels to the right of the peak
+            maxi = min(nmask[0], ww.size)
+            outbpm[pks[i]:ww[maxi]+2] = True
+    # Return the output bpm
+    return outbpm
+

doc/api/pypeit.scripts.extract_datacube.rst

@@ -0,0 +1,8 @@
+pypeit.scripts.extract\_datacube module
+=======================================
+
+.. automodule:: pypeit.scripts.extract_datacube
+   :members:
+   :private-members:
+   :undoc-members:
+   :show-inheritance:

doc/api/pypeit.scripts.rst

@@ -26,6 +26,7 @@ Submodules
    pypeit.scripts.compare_sky
    pypeit.scripts.compile_wvarxiv
    pypeit.scripts.edge_inspector
+   pypeit.scripts.extract_datacube
    pypeit.scripts.flux_calib
    pypeit.scripts.flux_setup
    pypeit.scripts.identify

doc/api/pypeit.spectrographs.aat_uhrf.rst

@@ -0,0 +1,8 @@
+pypeit.spectrographs.aat\_uhrf module
+=====================================
+
+.. automodule:: pypeit.spectrographs.aat_uhrf
+   :members:
+   :private-members:
+   :undoc-members:
+   :show-inheritance:

doc/api/pypeit.spectrographs.rst

@@ -7,6 +7,7 @@ Submodules
 .. toctree::
    :maxdepth: 4
 
+   pypeit.spectrographs.aat_uhrf
    pypeit.spectrographs.bok_bc
    pypeit.spectrographs.gemini_flamingos
    pypeit.spectrographs.gemini_gmos

doc/calibrations/flat.rst

@@ -51,7 +51,7 @@ spectrograph that we have not included example
 screen-shots.
 
 Raw Flat
---------
+++++++++
 
 This is the processed and combined ``pixelflat`` image.
 Despite the name, it is not completely raw.
@@ -60,7 +60,7 @@ This image should look like any one of your input
 ``pixelflat`` images.
 
 Pixel Flat
-----------
+++++++++++
 
 This is the normalized to unity image which is used to
 correct for pixel-to-pixel variations across the detector.
@@ -77,17 +77,32 @@ true if there is limited flux at these ends (e.g. the data
 goes below the atmospheric cutoff).
 
 Illumination Flat
------------------
++++++++++++++++++
 
 This image should also have most values near unity, but
 there will be vertical coherence.  And the edges (left/right)
 may fall off well below unity.
 
 Flat Model
-----------
+++++++++++
 
 This image should largely resemble the `Raw Flat`_.
 
+pypeit_show_pixflat
+-------------------
+
+In addition to ``pypeit_chk_flats``, if a custom pixel flat is provided by the user,
+another script ``pypeit_show_pixflat`` is available to inspect it. The script is called as follows:
+
+.. code-block:: console
+
+        pypeit_show_pixflat PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz
+
+The script usage can be displayed by calling the script with the ``-h`` option:
+
+.. include:: ../help/pypeit_show_pixflat.rst
+
+
 Troubleshooting
 ===============
 

doc/calibrations/flat_fielding.rst

@@ -204,7 +204,8 @@ need to provide the matching flat field images in your
 In short, if ``use_pixelflat=True`` for *any* of your frame types,
 at least one of the data files in the
 :ref:`pypeit_file` :ref:`data_block` must
-be labelled as ``pixelflat`` (unless you `Feed a PixelFlat`_).
+be labelled as ``pixelflat``, or ``slitless_pixflat``
+(unless you `Feed a PixelFlat`_).
 
 And, if ``use_illumflat=True`` for *any* of your frame types,
 at least one of the data files in the
@@ -216,26 +217,47 @@ frames for the pixel and illumination corrections. This is
 supported, but we recommend that you set the ``trace`` frames
 to be the same as the ``illumflat`` frames.
 
+.. _generate-pixflat:
+
+Generate a Slitless PixelFlat
+-----------------------------
+
+If a set of ``slitless_pixflat`` frames are available in the
+:ref:`data_block` of the :ref:`pypeit_file`, PypeIt will generate
+a slitless pixel flat (unless you `Feed a PixelFlat`_ instead)
+during the main :ref:`run-pypeit`, and will apply it to frame
+types that have ``use_pixelflat=True``.
+The slitless pixel flat is generated separately for each detector
+(even in the case of a mosaic reduction) and it is stored in a FITS
+file in the reduction directory, with one extension per detector.
+In addition to saving the file in your reduction directory,
+the constructed pixelflat is saved to the PypeIt cache (see ref:`data_installation`).
+This allows you to use the file for both current and future reductions.
+To use this file in future reductions, the user should add the slitless
+pixel flat file name to the :ref:`pypeit_file` as shown in `Feed a PixelFlat`_.
+
+If you generate your own slitless pixel flat, and you think it is generally
+applicable for your instrument, please consider sharing it with the PypeIt Developers.
+
+
 Feed a PixelFlat
 ----------------
 
 If you have generated your own pixel flat (or were provided one)
 and it is trimmed and oriented following the expected :ref:`pypeit-orientation`,
-then you may feed this into PypeIt.  This is the recommended approach
-at present for :ref:`lrisb`.
+then you may feed this into PypeIt.
 
-And you perform this by modifying the :ref:`parameter_block`:
-
-.. TODO: IS THIS STILL THE CORRECT APPROACH?  WHAT DO PEOPLE DO IF THEY DON'T
-.. HAVE THE DEV SUITE?
+To use the available PixelFlat, you need to modify the :ref:`parameter_block` like, e.g.:
 
 .. code-block:: ini
 
-    [calibrations]
-        [[flatfield]]
-            pixelflat_file = /Users/joe/python/PypeIt-development-suite/CALIBS/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009.fits.gz
+   [calibrations]
+      [[flatfield]]
+         pixelflat_file = PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz
+
+If any of the frames in the :ref:`data_block` are labelled as ``pixelflat``, or ``slitless_pixflat``,
+the provided pixel flat file will still be used instead of generating a new one.
 
-None of the frames in the :ref:`data_block` should be labelled as ``pixelflat``.
 
 Algorithms
 ----------

doc/calibrations/image_proc.rst

@@ -42,8 +42,9 @@ where:
     - the quantity :math:`C=N_{\rm frames}\ c/s\prime=c/s` is the number of electron counts excited by
       photons hitting the detector,
     - :math:`1/s=N_{\rm frames}/s\prime` is a factor that accounts for the number
-      of frames contributing to the electron counts, and the relative
-      throughput factors (see below) that can be measured from flat-field frames,
+      of frames contributing to the electron counts (`N_{\rm frames}`), and (`s\prime`) the relative
+      throughput factors (see below) that can be measured from flat-field frames plus a scaling factor
+      applied if the counts of each frame are scaled to the mean counts of all frames,
     - :math:`D` is the dark-current, i.e., the rate at which the detector
       generates thermal electrons, in e-/pixel/s,
     - :math:`N_{\rm bin}` is the number of pixels in a binned pixel,

doc/coadd3d.rst

@@ -155,7 +155,7 @@ There are several recommended steps of the coadd3d process that can be run separ
 
         pypeit_sensfunc spec1d_StandardStarName.fits -o sens_StandardStarName.fits
 
-    For further details, see :doc:`_sensitivity_function`.
+    For further details, see :ref:`sensitivity_function`.
 
 #. Step 4 - Generate a datacube of the science exposures. This is done by running the following command:
 
@@ -164,7 +164,7 @@ There are several recommended steps of the coadd3d process that can be run separ
         pypeit_coadd_datacube ScienceName.coadd3d -o
 
     Note that you will need to specify the sensitivity function file using the ``sensfile`` option in the
-    :doc:`coadd3d_file` file. For further details, see :ref:`coadd3d_fluxing`.
+    :ref:`coadd3d_file` file. For further details, see :ref:`coadd3d_fluxing`.
 
 Combination options
 ===================
@@ -239,7 +239,7 @@ If you would like to flux calibrate your datacube, you need to
 produce your standard star datacube first. Then extract the spectrum
 of the standard star using the ``pypeit_extract_datacube`` script. This
 will produce a ``spec1d`` file that you will need to use to generate a
-sensitivity function in the usual way (see :doc:`_sensitivity_function`).
+sensitivity function in the usual way (see :ref:`sensitivity_function`).
 Then, when generating the datacube of the science frame you must include
 the name of the sensitivity function in your ``coadd3d`` file as follows:
 
@@ -289,7 +289,7 @@ then you can specify the ``skysub_frame`` in the ``spec2d`` block of the
 above. If you have dedicated sky frames, then it is generally
 recommended to reduce these frames as if they are regular science
 frames, but add the following keyword arguments at the top of your
-:doc:`coadd3d_file`:
+:ref:`coadd3d_file`:
 
 .. code-block:: ini
 

doc/cookbook.rst

@@ -70,7 +70,8 @@ what we recommend:
  - We will refer to that folder as ``RAWDIR``
 
 The raw images can be gzip-compressed, although this means opening files will be
-slower.
+slower.  See :ref:`setup-file-searching` for specific comments about the files
+in your raw directory.
 
 A word on calibration data
 --------------------------

doc/dev/hiresconfig.rst

@@ -0,0 +1,118 @@
+.. include:: ../include/links.rst
+
+.. _hires_config:
+
+Automated sorting of HIRES frames by instrument configuration
+=============================================================
+
+Version History
+---------------
+
+
+=========   ================   =========== ===========
+*Version*   *Author*           *Date*      ``PypeIt``
+=========   ================   =========== ===========
+1.0         Debora Pelliccia   10 Aug 2024 1.16.1.dev
+=========   ================   =========== ===========
+
+----
+
+Basics
+------
+
+To prepare for the data reduction, PypeIt, first, automatically associates fits
+files to specific :ref:`frame_types` (see :ref:`hires_frames`) and, then,
+collects groups of frames in unique instrument configurations (see below). This is performed
+by the :ref:`pypeit_setup` script, which sorts the frames and writes a
+:ref:`pypeit_file` for each unique configuration. See :ref:`setup_doc`.
+
+
+HIRES configuration identification
+---------------------------------
+
+The HIRES instrument configurations are determined by the function
+:func:`pypeit.metadata.PypeItMetaData.unique_configurations`,
+which finds unique combinations of the following keywords:
+
+===============     ============
+``fitstbl`` key     Header Key
+===============     ============
+``dispname``        ``XDISPERS``
+``decker``          ``DECKNAME``
+``binning``         ``BINNING``
+``filter1``         ``FIL1NAME``
+``echangle``        ``ECHANGL``
+``xdangle``         ``XDANGL``
+===============     ============
+
+The unique configurations are determined by collating the relevant metadata from the headers
+of all frames found by a run of :ref:`pypeit_setup`, *except* those that are designated as
+bias and slitless_pixflat frames. Bias and slitless_pixflat frames can have header data (e.g., ``filter1``)
+that do not match the instrument configuration that an observer intended for their use.
+Therefore, PypeIt uses the ``dispname`` and ``binning`` keys to match the bias and
+slitless_pixflat frames to the configurations with frames taken with the same cross-disperser
+and same binning.
+Note that when using the ``echangle`` and ``xdangle`` keys to identify configurations, PypeIt
+uses a relative tolerance of 1e-3 and absolute tolerance of 1e-2 for ``echangle``, and a relative
+tolerance of 1e-2 for ``xdangle``, to account for small differences in the values of these angles.
+
+After that, :func:`pypeit.metadata.PypeItMetaData.set_configurations` associates each frame
+to the relevant unique configuration ("setup"), by assigning a setup identifier
+(e.g., A,B,C,D...) to every frames for which the values of the above keywords match the
+values of the specific unique configuration.
+
+HIRES calibration groups
+-----------------------
+
+PypeIt uses the concept of a "calibration group" to define a complete set of
+calibration frames (e.g., arcs, flats) and the science frames to which these calibration
+frames should be applied.
+
+By default, :ref:`pypeit_setup` uses the setup identifier to assign frames to a single
+calibration group. Frames that are in the same calibration group will have the same PypeIt
+keyword ``calib``. No automated procedure exists to do anything except this.
+However, the user can edit the :ref:`pypeit_file` to, within a given configuration, assign
+specific calibration frames to specific science frames using the data in the ``calib`` column
+of the :ref:`data_block`.
+
+Testing
+-------
+
+To test that PypeIt can successfully identify multiple
+configurations among a set of files, we have added the
+``test_setup_keck_hires_multiconfig()`` test to
+``${PYPEIT_DEV}/unit_tests/test_setups.py``.
+
+Here is an example of how to run the test:
+
+.. code-block:: bash
+
+    cd ${PYPEIT_DEV}/unit_tests
+    pytest test_setup.py::test_setup_keck_hires_multiconfig -W ignore
+
+The tests require that you have downloaded the PypeIt
+:ref:`dev-suite` and defined the ``PYPEIT_DEV`` environmental
+variable that points to the relevant directory.
+
+The algorithm for this test is as follows:
+
+    1. Collect the names of all files in selected HIRES directories.
+
+    2. Use :class:`~pypeit.pypeitsetup.PypeItSetup` to automatically
+       identify the configurations for these files.
+
+    3. Check that the code found two configurations and wrote the
+       pypeit files for each.
+
+    4. For each configuration:
+
+        a. Read the pypeit file
+
+        b. Check that the name for the setup is correct ('A' or 'B')
+
+        c. Check that the calibration group is the same for all frames ('0' or '1')
+
+
+Because these tests are now included in the PypeIt
+:ref:`unit-tests`, these configuration checks are performed by the
+developers for every new version of the code.