Merge pull request #249 from gbrammer/drizzle-variance

Implement `weight_type="jwst_var"` strategy for mosaics with full error propagation

grizli/aws/aws_drizzler.py

@@ -782,21 +782,24 @@ def drizzle_images(label='macs0647-jd1', ra=101.9822125, dec=70.24326667, pixsca
         else:
             clean_i = remove
 
-        status = utils.drizzle_from_visit(visits[0],
-                                          h,
-                                          pixfrac=pixfrac, kernel=kernel, 
-                                          clean=clean_i, 
-                                          include_saturated=include_saturated,
-                                          weight_type='median_err',
-                                          skip=skip,
-                                          dryrun=dryrun)
+        status = utils.drizzle_from_visit(
+            visits[0],
+            h,
+            pixfrac=pixfrac,
+            kernel=kernel,
+            clean=clean_i,
+            include_saturated=include_saturated,
+            weight_type='median_err',
+            skip=skip,
+            dryrun=dryrun
+        )
 
         if dryrun:
             filt_dict[filt] = status
             continue
 
         elif status is not None:
-            sci, wht, outh, filt_dict[filt], wcs_tab = status
+            sci, wht, var, outh, filt_dict[filt], wcs_tab = status
 
             if subtract_median:
                 #med = np.median(sci[sci != 0])

grizli/aws/db.py

@@ -335,6 +335,48 @@ def download_s3_file(path='s3://grizli-v2/HST/Pipeline/Tiles/2529/tile.2529.010.
     #     return None
 
 
+STATUS_QUERY = f"""select status, count(distinct(assoc_name)) from assoc_table
+where instrument_name in ('NIRISS','NIRCAM','MIRI')
+group by status
+"""
+
+class DatabaseStatusCheck():
+    """
+    Helper for checking ``status`` counts in various db tables
+    """
+    def __init__(self, query=STATUS_QUERY):
+        """
+        Parameters
+        ----------
+        query : str
+            SQL query
+        """
+        self.query = query
+        self.first = SQL(self.query)
+        # self.check_status()
+
+    def update(self):
+        """
+        Return the query and print counts and differences
+        """
+        new = SQL(self.query)
+        diffs = []
+        for i, s in enumerate(new['status']):
+            ix = self.first['status'] == s
+            if ix.sum() == 0:
+                self.first.add_row({'status': s, 'count': 0})
+                diffs.append('')
+            else:
+                diff = new['count'][i] - self.first['count'][ix][0]
+                if diff != 0:
+                    diffs.append(f'{diff}')
+                else:
+                    diffs.append('')
+
+        new['change'] = diffs
+        return new
+
+
 def get_redshift_fit_status(root, id, table='redshift_fit_v2', engine=None):
     """
     Get status value from the database for root_id object

grizli/aws/field_tiles.py

@@ -808,13 +808,17 @@ def process_tile(field='cos', tile='01.01', filters=TILE_FILTERS, fetch_existing
             os.system(f'gunzip --force {file}')
 
     if not rgb_only:
-        visit_processor.cutout_mosaic(rootname=root, 
-                              skip_existing=True,
-                              ir_wcs=ir_wcs,
-                              filters=filters, 
-                              pixfrac=pixfrac,
-                              kernel='square', s3output=None, 
-                              gzip_output=False, clean_flt=clean_flt)
+        visit_processor.cutout_mosaic(
+            rootname=root,
+            skip_existing=True,
+            ir_wcs=ir_wcs,
+            filters=filters,
+            pixfrac=pixfrac,
+            kernel='square',
+            s3output=None,
+            gzip_output=False,
+            clean_flt=clean_flt
+        )
 
     files = glob.glob(f'{root}*_dr*fits*')
     if len(files) == 0:
@@ -826,19 +830,29 @@ def process_tile(field='cos', tile='01.01', filters=TILE_FILTERS, fetch_existing
     if make_catalog:
         import golfir.catalog
 
-        golfir.catalog.make_charge_detection(root, ext='ir',
-                        filters=['f160w', 'f140w', 'f125w', 'f110w', 'f105w',
-                                 'f814w', 'f850lp',
-                                 'f277w-clear','f356w-clear','f444w-clear'],
-                                 use_hst_kernel=False,
-                                 subtract_background=True)
-
-        phot = auto_script.multiband_catalog(field_root=root,
-                       phot_apertures=prep.SEXTRACTOR_PHOT_APERTURES_ARCSEC[:4]) #, **phot_kwargs)
-
+        golfir.catalog.make_charge_detection(
+            root,
+            ext='ir',
+            filters=[
+                'f160w', 'f140w', 'f125w', 'f110w', 'f105w',
+                'f814w', 'f850lp',
+                'f277w-clear','f356w-clear','f444w-clear'
+            ],
+            use_hst_kernel=False,
+            subtract_background=True
+        )
+
+        phot = auto_script.multiband_catalog(
+            field_root=root,
+            phot_apertures=prep.SEXTRACTOR_PHOT_APERTURES_ARCSEC[:4]
+        ) #, **phot_kwargs)
+
         if len(phot) == 0:
             # Empty catalog
-            db.execute(f"update combined_tiles set status=10 where tile = '{tile}' AND field = '{field}'")
+            db.execute(
+                "update combined_tiles set status=10 "
+                + f"where tile = '{tile}' AND field = '{field}'"
+            )
 
             if cleanup:
 

grizli/aws/tile_mosaic.py

@@ -1402,18 +1402,20 @@ def drizzle_tile_subregion(tile=2530, subx=522, suby=461, filter='F160W', s3outp
 
     if s3output is None:
         s3output = f's3://grizli-mosaic-tiles/Tiles/{tile}/'
-    
+
     try:
-        visit_processor.cutout_mosaic(rootname=root,
-                                  product='{rootname}.{f}',
-                                  ir_wcs=ir_wcs,
-                                  res=exp, 
-                                  s3output=s3output, 
-                                  make_figure=make_figure,
-                                  skip_existing=skip_existing,
-                                  gzip_output=gzip_output,
-                                  **kwargs)
-
+        visit_processor.cutout_mosaic(
+            rootname=root,
+            product='{rootname}.{f}',
+            ir_wcs=ir_wcs,
+            res=exp,
+            s3output=s3output,
+            make_figure=make_figure,
+            skip_existing=skip_existing,
+            gzip_output=gzip_output,
+            **kwargs
+        )
+
         # Update subtile status
         db.execute(f"""UPDATE mosaic_tiles_exposures t
               SET in_mosaic = 1
@@ -1422,6 +1424,7 @@ def drizzle_tile_subregion(tile=2530, subx=522, suby=461, filter='F160W', s3outp
               AND w.filter='{filter}' AND tile={tile}
               AND subx={subx} AND suby={suby}
                """)
+
     except TypeError:
         db.execute(f"""UPDATE mosaic_tiles_exposures t
               SET in_mosaic = 8
@@ -1430,7 +1433,7 @@ def drizzle_tile_subregion(tile=2530, subx=522, suby=461, filter='F160W', s3outp
               AND w.filter='{filter}' AND tile={tile}
               AND subx={subx} AND suby={suby}
                """)
-        
+
     status = '{root}.{f}'
     return status