Use just the frame-id for CLI

.gitignore

@@ -101,4 +101,7 @@ tests/out/*.ipynb
 !tests/prepare-for-delivery.ipynb
 
 # Jsons in Test Directory
-tests/**/*.json
+tests/**/*.json
+
+# Use geojson zip file for data
+!isce2_topsapp/data/s1_frames.geojson.zip

CHANGELOG.md

@@ -11,8 +11,9 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ### Added
 * Provide prototype (internal) for burst analysis thanks to Forrest Williams and Joseph Kennedy (see PR #73)
 * CLI (and API) can switch between burst and SLC ifg generation thanks to entry point magic (see PR #73 for details)
-* Exposes Ionosphere correction in CLI (and API)
-* Exposes ESD and ESD threshold in CLI (and API)
+* Exposes a number of new corrections/ISCE2 processing options including: `ionosphere`, and `ESD threshold` arguments in CLI. Examples in README.
+* Exposes `frame-id` parameter for fixed frame cropping. Discussion, references, and examples in README.
+* Pins ISCE2 version to 2.6.1 and numpy / scipy to previous versions (see environment.yml) - to be amended when newest ISCE2 build is sorted out
 
 ## [0.2.1]
 

README.md

@@ -59,34 +59,30 @@ This example shows how to obtain a layer with ionsopheric phase delay. The SLCs
 isce2_topsapp --reference-scenes S1B_IW_SLC__1SDV_20171117T145926_20171117T145953_008323_00EBAB_AFB8 \
               --secondary-scenes S1A_IW_SLC__1SDV_20171111T150004_20171111T150032_019219_0208AF_EE89 \
               --estimate-ionosphere-delay True \
-              --do-esd True \
               --esd-coherence-threshold .5 \
               > topsapp_img.out 2> topsapp_img.err
 ```
+Not including `--esd-coherence-threshold` means no ESD correction will be applied. The ESD threshold refers to a coherence value and therefore must be in $[0, 1]$.
 
 ### Using "fixed frames" (experimental)
 
 Sentinel-1 Frames are not constant over passes. We generate fixed frames [here](https://github.com/ACCESS-Cloud-Based-InSAR/s1-frame-generation) and enumerate interferograms using this [repo](https://github.com/ACCESS-Cloud-Based-InSAR/s1-frame-enumerator). This is highly experimental. We then ensure ISCE processes only over the frame. The key is overlap. We provide some examples of the additional options (you will need to run this in *two* separate directories because ISCE2 outputs are organized with respect to the working directory of the processing). For one frame over CA:
 ```
-isce2_topsapp --reference-scenes S1A_IW_SLC__1SDV_20230113T135954_20230113T140021_046766_059B44_981B \
-                                 S1A_IW_SLC__1SDV_20230113T140019_20230113T140046_046766_059B44_A9C1 \
-                                 S1A_IW_SLC__1SDV_20230113T140044_20230113T140111_046766_059B44_FBB8 \
-              --secondary-scenes S1A_IW_SLC__1SDV_20221208T135956_20221208T140023_046241_05897B_86FA \
-                                 S1A_IW_SLC__1SDV_20221208T140021_20221208T140048_046241_05897B_8EBC \
-                                 S1A_IW_SLC__1SDV_20221208T140046_20221208T140113_046241_05897B_28A9 \
-              --region-of-interest -120.902529 35.257855 -117.740922 37.231403 \
-              --frame-id 19965 \
-              > topsapp_img_f19965.out 2> topsapp_img_f19965.err
+isce2_topsapp --reference-scenes S1A_IW_SLC__1SDV_20230125T135954_20230125T140021_046941_05A132_D35C \
+                                 S1A_IW_SLC__1SDV_20230125T140019_20230125T140046_046941_05A132_82DF \
+              --secondary-scenes S1A_IW_SLC__1SDV_20221220T135956_20221220T140023_046416_058F77_B248 \
+                                 S1A_IW_SLC__1SDV_20221220T140020_20221220T140047_046416_058F77_5213 \
+              --frame-id 22438 \
+              > topsapp_img_f22438.out 2> topsapp_img_f22438.err
 ```
 and an overlapping frame:
 ```
-isce2_topsapp --reference-scenes S1A_IW_SLC__1SDV_20230113T140019_20230113T140046_046766_059B44_A9C1 \
-                                 S1A_IW_SLC__1SDV_20230113T140044_20230113T140111_046766_059B44_FBB8 \
-              --secondary-scenes S1A_IW_SLC__1SDV_20221208T140021_20221208T140048_046241_05897B_8EBC \
-                                 S1A_IW_SLC__1SDV_20221208T140046_20221208T140113_046241_05897B_28A9 \
-              --region-of-interest -121.167298 33.929114 -118.055825 35.904876 \
-              --frame-id 19966 \
-              > topsapp_img_f19966.out 2> topsapp_img_f19966.err
+isce2_topsapp --reference-scenes S1A_IW_SLC__1SDV_20230125T140019_20230125T140046_046941_05A132_82DF \
+                                 S1A_IW_SLC__1SDV_20230125T140044_20230125T140111_046941_05A132_59E7 \
+              --secondary-scenes S1A_IW_SLC__1SDV_20221220T140020_20221220T140047_046416_058F77_5213 \
+                                 S1A_IW_SLC__1SDV_20221220T140045_20221220T140112_046416_058F77_7692 \
+              --frame-id 22439 \
+              > topsapp_img_f22439.out 2> topsapp_img_f22439.err
 ```
 
 # Running with Docker (locally or on a server)

environment.yml

@@ -18,7 +18,7 @@ dependencies:
  - geopandas
  - hyp3lib>=1.7
  - ipykernel
- - isce2
+ - isce2==2.6.1
  - jinja2
  - joblib
  - jsonschema==3.2.0
@@ -27,17 +27,19 @@ dependencies:
  - matplotlib
  - netcdf4
  - notebook
- - numpy
+ - numpy<1.24
  - pandas
  - papermill
  - pytest
  - pytest-cov
  - pytest-mock
  - rasterio
  - rioxarray
+ - xarray
+ - scipy<1.10
  - setuptools
  - setuptools_scm
  - shapely
  - tqdm
  - dem_stitcher>=2.3.1
- - aiohttp  # only needed for manifest and swath download
+ - aiohttp  # only needed for manifest and swath download

isce2_topsapp/__main__.py

@@ -19,17 +19,18 @@
 
 def localize_data(reference_scenes: list,
                   secondary_scenes: list,
-                  region_of_interest: list,
+                  frame_id: int = -1,
                   dry_run: bool = False) -> dict:
     """The dry-run prevents gets necessary metadata from SLCs and orbits.
 
     Can be used to run workflow without redownloading data (except DEM).
 
-    region_of_interest is in xmin, ymin, xmax, ymax format (epsg: 4326)
+    Fixed frames are found here: s3://s1-gunw-frames/s1_frames.geojson
+    And discussed in the readme.
     """
     out_slc = download_slcs(reference_scenes,
                             secondary_scenes,
-                            region_of_interest=region_of_interest,
+                            frame_id=frame_id,
                             dry_run=dry_run)
 
     out_orbits = download_orbits(reference_scenes,
@@ -111,8 +112,6 @@ def gunw_slc():
     parser.add_argument('--dry-run', action='store_true')
     parser.add_argument('--reference-scenes', type=str.split, nargs='+', required=True)
     parser.add_argument('--secondary-scenes', type=str.split, nargs='+', required=True)
-    parser.add_argument('--region-of-interest', type=float, nargs=4, default=None,
-                        help='xmin ymin xmax ymax in epgs:4326', required=False)
     parser.add_argument('--estimate-ionosphere-delay', type=true_false_string_argument, default=False)
     parser.add_argument('--frame-id', type=int, default=-1)
     parser.add_argument('--esd-coherence-threshold', type=float, default=-1.)
@@ -127,12 +126,8 @@ def gunw_slc():
     loc_data = localize_data(args.reference_scenes,
                              args.secondary_scenes,
                              dry_run=args.dry_run,
-                             region_of_interest=args.region_of_interest)
-    # TODO: either remove this or ensure it is passed to CMR metadata
+                             frame_id=args.frame_id)
     loc_data['frame_id'] = args.frame_id
-    if args.frame_id >= 0:
-        if not args.region_of_interest:
-            raise RuntimeError('If you specify frame_id, then must specify region_of_interest')
 
     # Allows for easier re-inspection of processing, packaging, and delivery
     # after job completes

isce2_topsapp/localize_slc.py

@@ -1,12 +1,21 @@
 import netrc
 from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
 
 import asf_search as asf
-from shapely.geometry import GeometryCollection, box, shape
+import geopandas as gpd
+from dateparser import parse
+from shapely.geometry import GeometryCollection, shape
 from shapely.ops import unary_union
 from tqdm import tqdm
 
 
+def get_global_gunw_frames():
+    data_dir = Path(__file__).parent / 'data'
+    path_to_frames_zip = data_dir / 's1_frames.geojson.zip'
+    return gpd.read_file(path_to_frames_zip)
+
+
 def get_asf_slc_objects(slc_ids: list) -> list:
 
     response = asf.granule_search(slc_ids)
@@ -30,41 +39,78 @@ def get_session():
     return session
 
 
-def check_geometry(reference_obs: list,
-                   secondary_obs: list,
-                   region_of_interest: list) -> GeometryCollection:
+def get_intersection_geo(reference_obs: list,
+                         secondary_obs: list,
+                         frame_id: int = -1) -> GeometryCollection:
     reference_geos = [shape(r.geojson()['geometry']) for r in reference_obs]
     secondary_geos = [shape(r.geojson()['geometry']) for r in secondary_obs]
 
     reference_geo = unary_union(reference_geos)
     secondary_geo = unary_union(secondary_geos)
 
-    # Two geometries must intersect for their to be an interferogram
-    intersection_geo = secondary_geo.intersection(reference_geo)
-    if intersection_geo.is_empty:
-        raise RuntimeError('The overlap between reference and secondary scenes '
-                           'is empty')
-
-    # Update the area of interest based on user specification
-    if region_of_interest is not None:
-        region_of_interest_geo = box(*region_of_interest)
-        if not region_of_interest_geo.intersects(intersection_geo):
-            raise RuntimeError('Region of interest does not overlap with IFG '
-                               'area (ref and sec overlap)')
-        intersection_geo = region_of_interest_geo
-
     # if they are not Polygons they are multipolygons and not valid
     connected_ref = (reference_geo.geom_type == 'Polygon')
     connected_sec = (secondary_geo.geom_type == 'Polygon')
+
     if (not connected_sec) or (not connected_ref):
-        raise RuntimeError('Reference and/or secondary dates were not connected'
-                           ' in their coverage (multipolygons)')
+        raise ValueError('Reference and/or secondary dates were not connected'
+                         ' in their coverage (multipolygons)')
+
+    # Two geometries must intersect for their to be an interferogram
+    intersection_geo = secondary_geo.intersection(reference_geo)
+    if intersection_geo.is_empty:
+        raise ValueError('The overlap between reference and secondary scenes '
+                         'is empty')
+
+    # Update the area of interest based on frame_id
+    if frame_id != -1:
+        df_frames = get_global_gunw_frames()
+        ind = df_frames.frame_id == frame_id
+        df_frame = df_frames[ind].reset_index(drop=True)
+        frame_geo = df_frame.geometry[0]
+        if not frame_geo.intersects(intersection_geo):
+            raise ValueError('Frame area does not overlap with IFG '
+                             'area (i.e. ref and sec overlap)')
+        intersection_geo = frame_geo
     return intersection_geo
 
 
+def ensure_repeat_pass_time_small(slc_properties: list,
+                                  maximum_minutes_between_acq=2):
+    """Make sure all the dictionaries of startTime are within 5 minutes"""
+    dates = [parse(prop['startTime']) for prop in slc_properties]
+    dates = sorted(dates)
+    minutes_apart_from_first_acq = [(date - dates[0]).seconds for date in dates]
+    return all([minutes_apart <= maximum_minutes_between_acq * 60
+                for minutes_apart in minutes_apart_from_first_acq])
+
+
+def check_flight_direction(slc_properties: list) -> bool:
+    unique_look_direction = set([prop['flightDirection']
+                                 for prop in slc_properties])
+    return len(unique_look_direction) == 1
+
+
+def check_date_order(ref_properties: list, sec_properties: list) -> bool:
+    ref_date = parse(ref_properties[0]['startTime'])
+    sec_date = parse(sec_properties[0]['startTime'])
+    return sec_date < ref_date
+
+
+def check_track_numbers(slc_properties: list):
+    path_numbers = [prop['pathNumber'] for prop in slc_properties]
+    path_numbers = sorted(list(set(path_numbers)))
+    if len(path_numbers) == 1:
+        return True
+    if len(path_numbers) == 2:
+        if ((path_numbers[1] - path_numbers[0]) == 1):
+            return True
+    return False
+
+
 def download_slcs(reference_ids: list,
                   secondary_ids: list,
-                  region_of_interest: list = None,
+                  frame_id: int = -1,
                   max_workers_for_download: int = 5,
                   dry_run: bool = False) -> dict:
     reference_obs = get_asf_slc_objects(reference_ids)
@@ -74,13 +120,32 @@ def download_slcs(reference_ids: list,
     reference_props = [ob.properties for ob in reference_obs]
     secondary_props = [ob.properties for ob in secondary_obs]
 
+    minutes_apart = 2
+    if not ensure_repeat_pass_time_small(reference_props,
+                                         maximum_minutes_between_acq=minutes_apart):
+        raise ValueError('The reference SLCs are more than {minutes_apart} min'
+                         'apart from the initial acq. in this pass')
+    if not ensure_repeat_pass_time_small(secondary_props,
+                                         maximum_minutes_between_acq=minutes_apart):
+        raise ValueError('The secondary SLCs are more than {minutes_apart} min'
+                         'apart from the initial acq. in this pass')
+
+    if not check_flight_direction(reference_props + secondary_props):
+        raise ValueError('The SLCs are not all Descending or Ascending')
+
+    if not check_track_numbers(reference_props + secondary_props):
+        raise ValueError('The SLCs do not belong to the same track (or sequential tracks)')
+
+    if not check_date_order(reference_props, secondary_props):
+        raise ValueError('Reference date must occur after secondary date')
+
     # Check the number of objects is the same as inputs
     assert len(reference_obs) == len(reference_ids)
     assert len(secondary_obs) == len(secondary_ids)
 
-    intersection_geo = check_geometry(reference_obs,
-                                      secondary_obs,
-                                      region_of_interest=region_of_interest)
+    intersection_geo = get_intersection_geo(reference_obs,
+                                            secondary_obs,
+                                            frame_id=frame_id)
 
     def download_one(resp):
         session = get_session()