From adc8ab515abcdadf29dc7cdd82cb9e2c785d41ef Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 20 Oct 2022 14:16:53 -0700 Subject: [PATCH 01/19] adding ionosphere for isce2 outputs --- isce2_topsapp/__main__.py | 6 +++++- isce2_topsapp/templates/topsapp_template.xml | 2 ++ isce2_topsapp/topsapp_proc.py | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/isce2_topsapp/__main__.py b/isce2_topsapp/__main__.py index 1252e71..fb70030 100644 --- a/isce2_topsapp/__main__.py +++ b/isce2_topsapp/__main__.py @@ -88,6 +88,7 @@ def gunw_slc(): parser.add_argument('--dry-run', action='store_true') parser.add_argument('--reference-scenes', type=str.split, nargs='+', required=True) parser.add_argument('--secondary-scenes', type=str.split, nargs='+', required=True) + parser.add_argument('--estimate-ionosphere-delay', type=bool, default=False) args = parser.parse_args() ensure_earthdata_credentials(args.username, args.password) @@ -110,9 +111,10 @@ def gunw_slc(): secondary_slc_zips=loc_data['sec_paths'], orbit_directory=loc_data['orbit_directory'], extent=loc_data['extent'], + estimate_ionosphere_delay=args.estimate_ionosphere_delay, dem_for_proc=loc_data['full_res_dem_path'], dem_for_geoc=loc_data['low_res_dem_path'], - dry_run=args.dry_run + dry_run=args.dry_run, ) ref_properties = loc_data['reference_properties'] @@ -146,6 +148,7 @@ def gunw_burst(): parser.add_argument('--burst-number', type=int, required=True) parser.add_argument('--azimuth-looks', type=int, default=2) parser.add_argument('--range-looks', type=int, default=10) + parser.add_argument('--estimate-ionosphere-delay', type=bool, default=False) args = parser.parse_args() ensure_earthdata_credentials(args.username, args.password) @@ -184,6 +187,7 @@ def gunw_burst(): extent=roi, dem_for_proc=dem['full_res_dem_path'], dem_for_geoc=dem['low_res_dem_path'], + estimate_ionosphere_delay=args.estimate_ionosphere_delay, azimuth_looks=args.azimuth_looks, range_looks=args.range_looks, swaths=[ref_burst.swath], diff --git a/isce2_topsapp/templates/topsapp_template.xml b/isce2_topsapp/templates/topsapp_template.xml index 8f776c5..326c93d 100755 --- a/isce2_topsapp/templates/topsapp_template.xml +++ b/isce2_topsapp/templates/topsapp_template.xml @@ -22,6 +22,8 @@ {{ geocodeDemFilename }} True snaphu_mcf + {{ estimate_ionosphere_delay }} + False {{ do_esd }} {{ esd_coherence_threshold }} {{ use_virtual_files }} diff --git a/isce2_topsapp/topsapp_proc.py b/isce2_topsapp/topsapp_proc.py index c58a596..df9486d 100644 --- a/isce2_topsapp/topsapp_proc.py +++ b/isce2_topsapp/topsapp_proc.py @@ -33,6 +33,7 @@ def topsapp_processing(*, extent: list, dem_for_proc: str, dem_for_geoc: str, + estimate_ionosphere_delay: bool, azimuth_looks: int = 7, range_looks: int = 19, swaths: list = None, @@ -62,6 +63,7 @@ def topsapp_processing(*, do_unwrap=True, use_virtual_files=True, esd_coherence_threshold=-1, + estimate_ionosphere_delay=estimate_ionosphere_delay, azimuth_looks=azimuth_looks, range_looks=range_looks, swaths=swaths From 96d1447c28b05976000047553f17b1cbead67b13 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 20 Oct 2022 14:21:03 -0700 Subject: [PATCH 02/19] include example in readme --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 066dee9..915fa6a 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,18 @@ This is reflected in the [`sample_run.sh`](sample_run.sh). To be even more explicity, you can use [`tee`](https://en.wikipedia.org/wiki/Tee_(command)) to record output to both including `> >(tee -a topsapp_img.out) 2> >(tee -a topsapp_img.err >&2)`. +### Customizations + +#### Estimating Ionospheric Phase Delay + +This example shows how to obtain a layer with ionsopheric phase delay. The SLCs are over the Arabian peninusula where the ionosphere can be seen: + +``` +isce2_topsapp --reference-scenes S1A_IW_SLC__1SDV_20221002T151520_20221002T151543_045265_056931_E517 \ + --secondary-scenes S1A_IW_SLC__1SDV_20220908T151520_20220908T151542_044915_055D68_78EC \ + --estimate-ionosphere-delay True +``` + # Running with Docker (locally or on a server) 1. When running locally with root privileges (i.e. at your local workstation), build the docker image using: From 0d4a7051278125c2e09f5d62f1be6268a2acea5e Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Fri, 21 Oct 2022 11:42:04 -0700 Subject: [PATCH 03/19] include err i/o to sample --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 915fa6a..2b70485 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,8 @@ This example shows how to obtain a layer with ionsopheric phase delay. The SLCs ``` isce2_topsapp --reference-scenes S1A_IW_SLC__1SDV_20221002T151520_20221002T151543_045265_056931_E517 \ --secondary-scenes S1A_IW_SLC__1SDV_20220908T151520_20220908T151542_044915_055D68_78EC \ - --estimate-ionosphere-delay True + --estimate-ionosphere-delay True \ + > topsapp_img.out 2> topsapp_img.err ``` # Running with Docker (locally or on a server) From 6390d9774af5f8252d01549dcdcbf7485be89a15 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Fri, 21 Oct 2022 11:42:39 -0700 Subject: [PATCH 04/19] update geocoding --- isce2_topsapp/templates/topsapp_template.xml | 2 +- isce2_topsapp/topsapp_proc.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/isce2_topsapp/templates/topsapp_template.xml b/isce2_topsapp/templates/topsapp_template.xml index 326c93d..2f85dc7 100755 --- a/isce2_topsapp/templates/topsapp_template.xml +++ b/isce2_topsapp/templates/topsapp_template.xml @@ -27,6 +27,6 @@ {{ do_esd }} {{ esd_coherence_threshold }} {{ use_virtual_files }} - ['merged/phsig.cor', 'merged/filt_topophase.unw', 'merged/los.rdr', 'merged/topophase.flat', 'merged/filt_topophase.flat', 'merged/filt_topophase_2stage.unw', 'merged/topophase.cor', 'merged/filt_topophase.unw.conncomp'] + {{ geocode_list }} diff --git a/isce2_topsapp/topsapp_proc.py b/isce2_topsapp/topsapp_proc.py index df9486d..b28ea46 100644 --- a/isce2_topsapp/topsapp_proc.py +++ b/isce2_topsapp/topsapp_proc.py @@ -25,6 +25,15 @@ TEMPLATE_DIR = Path(__file__).parent/'templates' +GEOCODE_LIST_BASE = ['merged/phsig.cor', + 'merged/filt_topophase.unw', + 'merged/los.rdr', + 'merged/topophase.flat', + 'merged/filt_topophase.flat', + 'merged/filt_topophase_2stage.unw', + 'merged/topophase.cor', + 'merged/filt_topophase.unw.conncomp'] + def topsapp_processing(*, reference_slc_zips: list, @@ -50,6 +59,10 @@ def topsapp_processing(*, with open(TEMPLATE_DIR/'topsapp_template.xml', 'r') as file: template = Template(file.read()) + geocode_list = GEOCODE_LIST_BASE.copy() + if estimate_ionosphere_delay: + geocode_list.append('merged/topophase.ion') + topsApp_xml = template.render(orbit_directory=orbit_directory, output_reference_directory='reference', output_secondary_directory='secondary', @@ -66,7 +79,8 @@ def topsapp_processing(*, estimate_ionosphere_delay=estimate_ionosphere_delay, azimuth_looks=azimuth_looks, range_looks=range_looks, - swaths=swaths + swaths=swaths, + geocode_list=geocode_list ) with open('topsApp.xml', "w") as file: file.write(topsApp_xml) From 4f39ecba28571633770276bb89110cd4c0a6a15d Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 24 Oct 2022 09:54:37 -0700 Subject: [PATCH 05/19] fix names in templates --- isce2_topsapp/templates/topsapp_template.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/isce2_topsapp/templates/topsapp_template.xml b/isce2_topsapp/templates/topsapp_template.xml index 2f85dc7..eb04f68 100755 --- a/isce2_topsapp/templates/topsapp_template.xml +++ b/isce2_topsapp/templates/topsapp_template.xml @@ -22,8 +22,8 @@ {{ geocodeDemFilename }} True snaphu_mcf - {{ estimate_ionosphere_delay }} - False + {{ estimate_ionosphere_delay }} + False {{ do_esd }} {{ esd_coherence_threshold }} {{ use_virtual_files }} From 1b538498b1d7fe41ebc4f9dfd0553f0b2b1d0551 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Tue, 25 Oct 2022 16:27:13 -0700 Subject: [PATCH 06/19] include 2d layer additions with rioxarray --- environment.yml | 1 + isce2_topsapp/__main__.py | 8 ++++- isce2_topsapp/packaging.py | 8 ++++- .../packaging_utils/additional_layers.json | 10 +++++++ .../packaging_utils/additional_layers.py | 30 +++++++++++++++++++ .../templates/nc_packaging_template.json | 18 ----------- 6 files changed, 55 insertions(+), 20 deletions(-) create mode 100644 isce2_topsapp/packaging_utils/additional_layers.json create mode 100644 isce2_topsapp/packaging_utils/additional_layers.py diff --git a/environment.yml b/environment.yml index dcd5145..6878372 100644 --- a/environment.yml +++ b/environment.yml @@ -34,6 +34,7 @@ dependencies: - pytest-cov - pytest-mock - rasterio + - rioxarray - setuptools - setuptools_scm - shapely diff --git a/isce2_topsapp/__main__.py b/isce2_topsapp/__main__.py index fb70030..9dfcfa7 100644 --- a/isce2_topsapp/__main__.py +++ b/isce2_topsapp/__main__.py @@ -121,10 +121,16 @@ def gunw_slc(): sec_properties = loc_data['secondary_properties'] extent = loc_data['extent'] + additional_2d_layers = [] + if args.estimate_ionosphere_delay: + additional_2d_layers.append('ionosphere') + + additional_2d_layers = additional_2d_layers or None nc_path = package_gunw_product(isce_data_directory=Path.cwd(), reference_properties=ref_properties, secondary_properties=sec_properties, - extent=extent + extent=extent, + additional_2d_layers=additional_2d_layers ) # Move final product to current working directory diff --git a/isce2_topsapp/packaging.py b/isce2_topsapp/packaging.py index cf333a1..59d0406 100644 --- a/isce2_topsapp/packaging.py +++ b/isce2_topsapp/packaging.py @@ -10,6 +10,8 @@ import isce2_topsapp from isce2_topsapp.templates import read_netcdf_packaging_template +from isce2_topsapp.packaging_utils.additional_layers import add_2d_layer + DATASET_VERSION = '2.0.6' @@ -215,7 +217,8 @@ def package_gunw_product(*, isce_data_directory: Union[str, Path], reference_properties: list, secondary_properties: list, - extent: list) -> Path: + extent: list, + additional_2d_layers: list = None) -> Path: """Creates a GUNW standard product netcdf from the ISCE outputs and some initial metadata. @@ -244,4 +247,7 @@ def package_gunw_product(*, out_nc_file = perform_netcdf_packaging(isce_data_dir=isce_data_directory, gunw_id=gunw_id) + if additional_2d_layers is not None: + [additional_2d_layers(layer, out_nc_file) for layer in additional_2d_layers] + return out_nc_file diff --git a/isce2_topsapp/packaging_utils/additional_layers.json b/isce2_topsapp/packaging_utils/additional_layers.json new file mode 100644 index 0000000..94fb24b --- /dev/null +++ b/isce2_topsapp/packaging_utils/additional_layers.json @@ -0,0 +1,10 @@ +{"ionosphere": {"dst_group": "/science/grids/corrections/derived/ionosphere", + "input_relative_path": "merged/topophase.ion.geo", + "attr": {"_FillValue": 0, + "standard_name": "ionospherePhaseCorrection", + "long_name": "ionospherePhaseCorrection", + "description": "Estimated ionosphere phase correction" + } + } + +} \ No newline at end of file diff --git a/isce2_topsapp/packaging_utils/additional_layers.py b/isce2_topsapp/packaging_utils/additional_layers.py new file mode 100644 index 0000000..da42ca8 --- /dev/null +++ b/isce2_topsapp/packaging_utils/additional_layers.py @@ -0,0 +1,30 @@ +import json +from pathlib import Path + +import rioxarray as rxr +import xarray as xr + +LAYER_JSON = Path(__file__).parents[0] / 'additional_layers.json' +ADDITIONAL_LAYERS = json.load(open(LAYER_JSON)) + + +def add_2d_layer(layer_name: str, netcdf_path: Path) -> Path: + + layer_data = ADDITIONAL_LAYERS[layer_name] + data_array = rxr.open_rasterio(layer_data['input_relative_path']) + data_array = data_array.rename({'x': 'longitude', + 'y': 'latitude'}) + + data_array = data_array.squeeze(dim=['band'], + drop=True) + + data_array.attrs.update(layer_data['attr']) + group = xr.Dataset({layer_name: data_array}) + + group.rio.write_crs(4326, inplace=True) + + group.to_netcdf(netcdf_path, + group=layer_data['dst_group'], + mode='a') + + return netcdf_path diff --git a/isce2_topsapp/templates/nc_packaging_template.json b/isce2_topsapp/templates/nc_packaging_template.json index c00538c..a189118 100755 --- a/isce2_topsapp/templates/nc_packaging_template.json +++ b/isce2_topsapp/templates/nc_packaging_template.json @@ -255,24 +255,6 @@ { "group" : [ { - "name" : "derived", - "content" : [ - { - "group" : [ - { - "name" : "ionosphere", - "content" : [ - { - "dataset" : [ - { - "name" : "ionosphere", - "type" : "str", - "description" : "placeholder dummy: [2D]" - }] - }] - }] - }] - },{ "name" : "external", "content" : [ { From d8d46c81d063e70babbdefd1fbb29c30bec7a696 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 26 Oct 2022 10:31:56 -0700 Subject: [PATCH 07/19] fix function call --- isce2_topsapp/packaging.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/isce2_topsapp/packaging.py b/isce2_topsapp/packaging.py index 59d0406..a422706 100644 --- a/isce2_topsapp/packaging.py +++ b/isce2_topsapp/packaging.py @@ -9,9 +9,8 @@ from dateparser import parse import isce2_topsapp -from isce2_topsapp.templates import read_netcdf_packaging_template from isce2_topsapp.packaging_utils.additional_layers import add_2d_layer - +from isce2_topsapp.templates import read_netcdf_packaging_template DATASET_VERSION = '2.0.6' @@ -193,7 +192,7 @@ def perform_netcdf_packaging(*, isce_data_dir = Path(isce_data_dir) merged_dir = isce_data_dir/'merged' metadata_path = merged_dir/'metadata.h5' - assert(metadata_path.exists()) + assert metadata_path.exists() # Write config file _write_json_config(gunw_id=gunw_id, @@ -209,7 +208,7 @@ def perform_netcdf_packaging(*, out_nc_file = merged_dir/f'{gunw_id}.nc' # Check if the netcdf file was created - assert(out_nc_file.exists()) + assert out_nc_file.exists() return out_nc_file @@ -248,6 +247,6 @@ def package_gunw_product(*, gunw_id=gunw_id) if additional_2d_layers is not None: - [additional_2d_layers(layer, out_nc_file) for layer in additional_2d_layers] + [add_2d_layer(layer, out_nc_file) for layer in additional_2d_layers] return out_nc_file From ff9766393c2dc6957bf76d21aa042dd4723442cd Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 26 Oct 2022 13:38:06 -0700 Subject: [PATCH 08/19] finalize packaging --- .../packaging_utils/additional_layers.json | 10 +++---- .../packaging_utils/additional_layers.py | 30 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/isce2_topsapp/packaging_utils/additional_layers.json b/isce2_topsapp/packaging_utils/additional_layers.json index 94fb24b..0d2cde4 100644 --- a/isce2_topsapp/packaging_utils/additional_layers.json +++ b/isce2_topsapp/packaging_utils/additional_layers.json @@ -1,10 +1,10 @@ {"ionosphere": {"dst_group": "/science/grids/corrections/derived/ionosphere", "input_relative_path": "merged/topophase.ion.geo", - "attr": {"_FillValue": 0, - "standard_name": "ionospherePhaseCorrection", - "long_name": "ionospherePhaseCorrection", - "description": "Estimated ionosphere phase correction" - } + "attrs": {"_FillValue": 0, + "standard_name": "ionospherePhaseCorrection", + "long_name": "ionospherePhaseCorrection", + "description": "Estimated ionosphere phase correction" + } } } \ No newline at end of file diff --git a/isce2_topsapp/packaging_utils/additional_layers.py b/isce2_topsapp/packaging_utils/additional_layers.py index da42ca8..b1efc76 100644 --- a/isce2_topsapp/packaging_utils/additional_layers.py +++ b/isce2_topsapp/packaging_utils/additional_layers.py @@ -1,7 +1,6 @@ import json from pathlib import Path -import rioxarray as rxr import xarray as xr LAYER_JSON = Path(__file__).parents[0] / 'additional_layers.json' @@ -11,20 +10,21 @@ def add_2d_layer(layer_name: str, netcdf_path: Path) -> Path: layer_data = ADDITIONAL_LAYERS[layer_name] - data_array = rxr.open_rasterio(layer_data['input_relative_path']) - data_array = data_array.rename({'x': 'longitude', - 'y': 'latitude'}) - data_array = data_array.squeeze(dim=['band'], - drop=True) - - data_array.attrs.update(layer_data['attr']) - group = xr.Dataset({layer_name: data_array}) - - group.rio.write_crs(4326, inplace=True) - - group.to_netcdf(netcdf_path, - group=layer_data['dst_group'], - mode='a') + ds = xr.open_dataset(layer_data['input_relative_path'], + engine='rasterio') + ds = ds.rename({'x': 'longitude', + 'y': 'latitude', + 'band_data': layer_name}) + ds['latitude'].attrs.update({'long_name': 'latitude', + 'standard_name': 'latitude'}) + ds['longitude'].attrs.update({'long_name': 'longitude', + 'standard_name': 'longitude'}) + ds = ds.squeeze(['band'], drop=True) + ds[layer_name].attrs.update(layer_data['attrs']) + + ds.to_netcdf(netcdf_path, + group=layer_data['dst_group'], + mode='a') return netcdf_path From 4116d4d4e35f9f714ea55b4af6d85163c762cd4a Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 26 Oct 2022 13:43:33 -0700 Subject: [PATCH 09/19] lint --- isce2_topsapp/localize_dem.py | 2 +- isce2_topsapp/localize_slc.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/isce2_topsapp/localize_dem.py b/isce2_topsapp/localize_dem.py index b14f86a..0e7c873 100644 --- a/isce2_topsapp/localize_dem.py +++ b/isce2_topsapp/localize_dem.py @@ -13,7 +13,7 @@ def tag_dem_xml_as_ellipsoidal(dem_path: Path) -> str: xml_path = str(dem_path) + '.xml' - assert(Path(xml_path).exists()) + assert Path(xml_path).exists() tree = etree.parse(xml_path) root = tree.getroot() diff --git a/isce2_topsapp/localize_slc.py b/isce2_topsapp/localize_slc.py index e22f6e7..37a2d01 100644 --- a/isce2_topsapp/localize_slc.py +++ b/isce2_topsapp/localize_slc.py @@ -40,11 +40,11 @@ def check_geometry(reference_obs: list, # Two geometries must intersect for their to be an interferogram intersection_geo = secondary_geo.intersection(reference_geo) - assert(not intersection_geo.is_empty) + assert not intersection_geo.is_empty # if they are not Polygons they are multipolygons and not valid - assert(isinstance(secondary_geo, Polygon)) - assert(isinstance(reference_geo, Polygon)) + assert isinstance(secondary_geo, Polygon) + assert isinstance(reference_geo, Polygon) return intersection_geo @@ -60,8 +60,8 @@ def download_slcs(reference_ids: list, secondary_props = [ob.properties for ob in secondary_obs] # Check the number of objects is the same as inputs - assert(len(reference_obs) == len(reference_ids)) - assert(len(secondary_obs) == len(secondary_ids)) + assert len(reference_obs) == len(reference_ids) + assert len(secondary_obs) == len(secondary_ids) intersection_geo = check_geometry(reference_obs, secondary_obs) From 87863ef70284cf510fe7f9850f792d6baa392537 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 26 Oct 2022 14:37:04 -0700 Subject: [PATCH 10/19] do-esd first pass --- README.md | 4 +++- isce2_topsapp/__main__.py | 9 +++++++++ isce2_topsapp/topsapp_proc.py | 8 +++++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2b70485..e519de5 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ To be even more explicity, you can use [`tee`](https://en.wikipedia.org/wiki/Tee ### Customizations -#### Estimating Ionospheric Phase Delay +#### Estimating Ionospheric Phase Delay and ESD This example shows how to obtain a layer with ionsopheric phase delay. The SLCs are over the Arabian peninusula where the ionosphere can be seen: @@ -59,6 +59,8 @@ This example shows how to obtain a layer with ionsopheric phase delay. The SLCs isce2_topsapp --reference-scenes S1A_IW_SLC__1SDV_20221002T151520_20221002T151543_045265_056931_E517 \ --secondary-scenes S1A_IW_SLC__1SDV_20220908T151520_20220908T151542_044915_055D68_78EC \ --estimate-ionosphere-delay True \ + --do-esd True \ + --esd-coherence-threshold .5 \ > topsapp_img.out 2> topsapp_img.err ``` diff --git a/isce2_topsapp/__main__.py b/isce2_topsapp/__main__.py index 9dfcfa7..b026f28 100644 --- a/isce2_topsapp/__main__.py +++ b/isce2_topsapp/__main__.py @@ -89,8 +89,15 @@ def gunw_slc(): parser.add_argument('--reference-scenes', type=str.split, nargs='+', required=True) parser.add_argument('--secondary-scenes', type=str.split, nargs='+', required=True) parser.add_argument('--estimate-ionosphere-delay', type=bool, default=False) + parser.add_argument('--do-esd', type=bool, default=False) + parser.add_argument('--esd_coherence_threshold', type=float, default=-1) args = parser.parse_args() + do_esd_arg = (args.esd_coherence_treshold != -1) == args.do_esd + if not do_esd_arg: + raise ValueError('If ESD is turned on, specify esd_coherence_threshold between 0 and 1; ' + 'Otherwise, do not or set the threshold to -1') + ensure_earthdata_credentials(args.username, args.password) args.reference_scenes = [item for sublist in args.reference_scenes for item in sublist] @@ -112,6 +119,8 @@ def gunw_slc(): orbit_directory=loc_data['orbit_directory'], extent=loc_data['extent'], estimate_ionosphere_delay=args.estimate_ionosphere_delay, + do_esd=args.do_esd, + esd_coherence_threshold=args.esd_coherence_threshold, dem_for_proc=loc_data['full_res_dem_path'], dem_for_geoc=loc_data['low_res_dem_path'], dry_run=args.dry_run, diff --git a/isce2_topsapp/topsapp_proc.py b/isce2_topsapp/topsapp_proc.py index b28ea46..974e000 100644 --- a/isce2_topsapp/topsapp_proc.py +++ b/isce2_topsapp/topsapp_proc.py @@ -46,7 +46,9 @@ def topsapp_processing(*, azimuth_looks: int = 7, range_looks: int = 19, swaths: list = None, - dry_run: bool = False): + dry_run: bool = False, + do_esd: bool = False, + esd_coherence_threshold: float = .7): swaths = swaths or [1, 2, 3] # for [ymin, ymax, xmin, xmax] extent_isce = [extent[k] for k in [1, 3, 0, 2]] @@ -71,11 +73,11 @@ def topsapp_processing(*, region_of_interest=extent_isce, demFilename=dem_for_proc, geocodeDemFilename=dem_for_geoc, - do_esd=False, filter_strength=.5, do_unwrap=True, use_virtual_files=True, - esd_coherence_threshold=-1, + do_esd=do_esd, + esd_coherence_threshold=esd_coherence_threshold, estimate_ionosphere_delay=estimate_ionosphere_delay, azimuth_looks=azimuth_looks, range_looks=range_looks, From 3119d1a0e0968cafaa32faea1c06d72022319c57 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 26 Oct 2022 14:46:44 -0700 Subject: [PATCH 11/19] fix underscore and type --- isce2_topsapp/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/isce2_topsapp/__main__.py b/isce2_topsapp/__main__.py index b026f28..c2857e7 100644 --- a/isce2_topsapp/__main__.py +++ b/isce2_topsapp/__main__.py @@ -90,10 +90,10 @@ def gunw_slc(): parser.add_argument('--secondary-scenes', type=str.split, nargs='+', required=True) parser.add_argument('--estimate-ionosphere-delay', type=bool, default=False) parser.add_argument('--do-esd', type=bool, default=False) - parser.add_argument('--esd_coherence_threshold', type=float, default=-1) + parser.add_argument('--esd-coherence-threshold', type=float, default=-1) args = parser.parse_args() - do_esd_arg = (args.esd_coherence_treshold != -1) == args.do_esd + do_esd_arg = (args.esd_coherence_threshold != -1) == args.do_esd if not do_esd_arg: raise ValueError('If ESD is turned on, specify esd_coherence_threshold between 0 and 1; ' 'Otherwise, do not or set the threshold to -1') From c0efbf5aee6f7f1d9d92df8fcd1e256460a381d5 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 9 Nov 2022 11:55:30 -0800 Subject: [PATCH 12/19] use example from David Bekaert --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e519de5..4b8cf98 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ To be even more explicity, you can use [`tee`](https://en.wikipedia.org/wiki/Tee This example shows how to obtain a layer with ionsopheric phase delay. The SLCs are over the Arabian peninusula where the ionosphere can be seen: ``` -isce2_topsapp --reference-scenes S1A_IW_SLC__1SDV_20221002T151520_20221002T151543_045265_056931_E517 \ - --secondary-scenes S1A_IW_SLC__1SDV_20220908T151520_20220908T151542_044915_055D68_78EC \ +isce2_topsapp --reference-scenes S1B_IW_SLC__1SDV_20171117T145926_20171117T145953_008323_00EBAB_AFB8 \ + --secondary-scenes S1A_IW_SLC__1SDV_20171111T150004_20171111T150032_019219_0208AF_EE89 \ --estimate-ionosphere-delay True \ --do-esd True \ --esd-coherence-threshold .5 \ From 1b56dbdc7ba6f67054c8ee6b192062c28d2f2c10 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 9 Nov 2022 12:01:53 -0800 Subject: [PATCH 13/19] add comments --- isce2_topsapp/packaging_utils/additional_layers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/isce2_topsapp/packaging_utils/additional_layers.py b/isce2_topsapp/packaging_utils/additional_layers.py index b1efc76..03aea50 100644 --- a/isce2_topsapp/packaging_utils/additional_layers.py +++ b/isce2_topsapp/packaging_utils/additional_layers.py @@ -13,6 +13,8 @@ def add_2d_layer(layer_name: str, netcdf_path: Path) -> Path: ds = xr.open_dataset(layer_data['input_relative_path'], engine='rasterio') + + # Renaming ensures correct geo-referencing with spatial_ref grid mapping ds = ds.rename({'x': 'longitude', 'y': 'latitude', 'band_data': layer_name}) @@ -20,6 +22,8 @@ def add_2d_layer(layer_name: str, netcdf_path: Path) -> Path: 'standard_name': 'latitude'}) ds['longitude'].attrs.update({'long_name': 'longitude', 'standard_name': 'longitude'}) + + # removes channel (aka band) dimension ds = ds.squeeze(['band'], drop=True) ds[layer_name].attrs.update(layer_data['attrs']) From 95c572a3c59fdac82a1633113adeb2d16dba3588 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 9 Nov 2022 12:54:35 -0800 Subject: [PATCH 14/19] fix trufflehog with jhk help --- .trufflehog.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.trufflehog.txt b/.trufflehog.txt index a21adc3..aa7181e 100644 --- a/.trufflehog.txt +++ b/.trufflehog.txt @@ -1,5 +1,6 @@ -.*gitleaks.toml$ .*.ipynb$ .*/templates/.* tests/sample_loc_metadata.json Dockerfile +tests/test_data/sec_manifest.xml +tests/test_data/ref_manifest.xml From 499a927b4d21cfae7d418d85582f3178b523a661 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 10 Nov 2022 09:22:19 -0800 Subject: [PATCH 15/19] add changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3676db4..11ab415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.4] + +### Added +* Provide prototype (internal) for burst analysis thanks to Forrest Williams and Joseph Kennedy (see PR #73) +* CLI (and API) can switch between burst and SLC ifg generation thanks to entry point magic (see PR #73 for details) +* Exposes Ionosphere correction in CLI (and API) +* Exposes ESD and ESD threshold in CLI (and API) + ## [0.1.3] From 710af6636df844a2e7a5ce470f6a208da219e60c Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Tue, 17 Jan 2023 10:26:37 -0800 Subject: [PATCH 16/19] revert packaging template --- .../templates/nc_packaging_template.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/isce2_topsapp/templates/nc_packaging_template.json b/isce2_topsapp/templates/nc_packaging_template.json index a189118..c00538c 100755 --- a/isce2_topsapp/templates/nc_packaging_template.json +++ b/isce2_topsapp/templates/nc_packaging_template.json @@ -255,6 +255,24 @@ { "group" : [ { + "name" : "derived", + "content" : [ + { + "group" : [ + { + "name" : "ionosphere", + "content" : [ + { + "dataset" : [ + { + "name" : "ionosphere", + "type" : "str", + "description" : "placeholder dummy: [2D]" + }] + }] + }] + }] + },{ "name" : "external", "content" : [ { From 0b405a1a4e7f7f0eca02a033e800d1159147af33 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Tue, 17 Jan 2023 10:54:22 -0800 Subject: [PATCH 17/19] ensure default placeholders in packaging; consistency; Buzzanga H5 overwrite --- .../packaging_utils/additional_layers.json | 1 + .../packaging_utils/additional_layers.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/isce2_topsapp/packaging_utils/additional_layers.json b/isce2_topsapp/packaging_utils/additional_layers.json index 0d2cde4..fa2eb1e 100644 --- a/isce2_topsapp/packaging_utils/additional_layers.json +++ b/isce2_topsapp/packaging_utils/additional_layers.json @@ -1,4 +1,5 @@ {"ionosphere": {"dst_group": "/science/grids/corrections/derived/ionosphere", + "dst_variable": "ionosphere", "input_relative_path": "merged/topophase.ion.geo", "attrs": {"_FillValue": 0, "standard_name": "ionospherePhaseCorrection", diff --git a/isce2_topsapp/packaging_utils/additional_layers.py b/isce2_topsapp/packaging_utils/additional_layers.py index 03aea50..01ba817 100644 --- a/isce2_topsapp/packaging_utils/additional_layers.py +++ b/isce2_topsapp/packaging_utils/additional_layers.py @@ -1,6 +1,7 @@ import json from pathlib import Path +import h5py import xarray as xr LAYER_JSON = Path(__file__).parents[0] / 'additional_layers.json' @@ -8,8 +9,21 @@ def add_2d_layer(layer_name: str, netcdf_path: Path) -> Path: + """ + Combines a lot of standard formatting of the netcdf via rioxarray and + deletes the previous placeholder (we assume it exists via the placeholder). + + We also assume any additional processing specific to GUNW is done outside of + this function. + """ layer_data = ADDITIONAL_LAYERS[layer_name] + dst_group = layer_data['dst_group'] + dst_variable = layer_data['dst_variable'] + + # The layers generally already exist within the file + with h5py.File(netcdf_path, mode='a') as file: + del file[dst_group][dst_variable] ds = xr.open_dataset(layer_data['input_relative_path'], engine='rasterio') @@ -17,7 +31,7 @@ def add_2d_layer(layer_name: str, netcdf_path: Path) -> Path: # Renaming ensures correct geo-referencing with spatial_ref grid mapping ds = ds.rename({'x': 'longitude', 'y': 'latitude', - 'band_data': layer_name}) + 'band_data': dst_variable}) ds['latitude'].attrs.update({'long_name': 'latitude', 'standard_name': 'latitude'}) ds['longitude'].attrs.update({'long_name': 'longitude', From 57962c73a08914691b9397f74e2784a7236b7bf6 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 18 Jan 2023 15:24:59 -0800 Subject: [PATCH 18/19] try micromamba --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d64ea9..f9b7513 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,12 +26,12 @@ jobs: username: ${{ secrets.EARTHDATA_USERNAME }} password: ${{ secrets.EARTHDATA_PASSWORD }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: mamba-org/provision-with-micromamba@main with: - mamba-version: "*" - python-version: ${{ matrix.python-version }} - activate-environment: topsapp_env + environment-name: topsapp_env environment-file: environment.yml + extra-specs: | + python=${{ matrix.python-version }} - name: Pytest in conda environment shell: bash -l {0} From b98ecd18f3d7e74a2e242817fa5f8063b80a37f5 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 18 Jan 2023 17:26:12 -0800 Subject: [PATCH 19/19] update mask and 1km reprojection --- isce2_topsapp/packaging.py | 44 ++++++++++++++--- .../packaging_utils/additional_layers.json | 4 +- .../packaging_utils/additional_layers.py | 13 +++-- isce2_topsapp/packaging_utils/ionosphere.py | 47 +++++++++++++++++++ 4 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 isce2_topsapp/packaging_utils/ionosphere.py diff --git a/isce2_topsapp/packaging.py b/isce2_topsapp/packaging.py index a422706..32b8072 100644 --- a/isce2_topsapp/packaging.py +++ b/isce2_topsapp/packaging.py @@ -5,16 +5,21 @@ from pathlib import Path from typing import Union +import h5py import numpy as np from dateparser import parse import isce2_topsapp from isce2_topsapp.packaging_utils.additional_layers import add_2d_layer +from isce2_topsapp.packaging_utils.ionosphere import format_ionosphere_for_gunw from isce2_topsapp.templates import read_netcdf_packaging_template DATASET_VERSION = '2.0.6' +PERMISSIBLE_2D_LAYERS = ['ionosphere'] + + """Warning: the packaging scripts were written as command line scripts and are highly dependent on the current working directory and its structure. @@ -212,6 +217,30 @@ def perform_netcdf_packaging(*, return out_nc_file +def package_additional_layers_into_gunw(gunw_path: Path, + isce_data_directory: Path, + additional_2d_layers: list): + # Current workflow of additional layers + # 1. Do any additional processing/formatting outside of GUNW + # 2. Add layer into GUNW + # 3. Update Version + if not set(additional_2d_layers).issubset(set(PERMISSIBLE_2D_LAYERS)): + raise RuntimeError('Additional 2d layers must be subset of ' + f'{PERMISSIBLE_2D_LAYERS}') + + if 'ionosphere' in additional_2d_layers: + # current working directory is ISCE directory + _ = format_ionosphere_for_gunw(isce_data_directory, gunw_path) + + # Assumes ionosphere raster is written to specific path + [add_2d_layer(layer, gunw_path) for layer in additional_2d_layers] + + # Update + with h5py.File(gunw_path, mode='a') as file: + file.attrs.modify('version', '1c') + return gunw_path + + def package_gunw_product(*, isce_data_directory: Union[str, Path], reference_properties: list, @@ -223,14 +252,16 @@ def package_gunw_product(*, Parameters ---------- - isce_data_directory + isce_data_directory: str | path Where the ISCE outputs are relative to current working directory - reference_properties + reference_properties: list Each item a dictionary per ASF API including ID, starttime, etc. - secondary_properties + secondary_properties: list Each item a dictionary per ASF API including ID, starttime, etc - extent + extent: list List of extents ([xmin, ymin, xmax, ymax]) + additional_2d_layers: list + List of 2d layers to add. Currently, supported is ionosphere. Returns ------- @@ -247,6 +278,7 @@ def package_gunw_product(*, gunw_id=gunw_id) if additional_2d_layers is not None: - [add_2d_layer(layer, out_nc_file) for layer in additional_2d_layers] - + package_additional_layers_into_gunw(out_nc_file, + isce_data_directory, + additional_2d_layers) return out_nc_file diff --git a/isce2_topsapp/packaging_utils/additional_layers.json b/isce2_topsapp/packaging_utils/additional_layers.json index fa2eb1e..cc6b46b 100644 --- a/isce2_topsapp/packaging_utils/additional_layers.json +++ b/isce2_topsapp/packaging_utils/additional_layers.json @@ -1,7 +1,7 @@ {"ionosphere": {"dst_group": "/science/grids/corrections/derived/ionosphere", "dst_variable": "ionosphere", - "input_relative_path": "merged/topophase.ion.geo", - "attrs": {"_FillValue": 0, + "input_relative_path": "merged/ionosphere_for_gunw.geo", + "attrs": { "standard_name": "ionospherePhaseCorrection", "long_name": "ionospherePhaseCorrection", "description": "Estimated ionosphere phase correction" diff --git a/isce2_topsapp/packaging_utils/additional_layers.py b/isce2_topsapp/packaging_utils/additional_layers.py index 01ba817..8dc0cbc 100644 --- a/isce2_topsapp/packaging_utils/additional_layers.py +++ b/isce2_topsapp/packaging_utils/additional_layers.py @@ -8,7 +8,7 @@ ADDITIONAL_LAYERS = json.load(open(LAYER_JSON)) -def add_2d_layer(layer_name: str, netcdf_path: Path) -> Path: +def add_2d_layer(layer_name: str, gunw_netcdf_path: Path) -> Path: """ Combines a lot of standard formatting of the netcdf via rioxarray and deletes the previous placeholder (we assume it exists via the placeholder). @@ -22,8 +22,11 @@ def add_2d_layer(layer_name: str, netcdf_path: Path) -> Path: dst_variable = layer_data['dst_variable'] # The layers generally already exist within the file - with h5py.File(netcdf_path, mode='a') as file: - del file[dst_group][dst_variable] + with h5py.File(gunw_netcdf_path, mode='a') as file: + if dst_group in file: + for key in file[dst_group].keys(): + del file[dst_group][key] + del file[dst_group] ds = xr.open_dataset(layer_data['input_relative_path'], engine='rasterio') @@ -41,8 +44,8 @@ def add_2d_layer(layer_name: str, netcdf_path: Path) -> Path: ds = ds.squeeze(['band'], drop=True) ds[layer_name].attrs.update(layer_data['attrs']) - ds.to_netcdf(netcdf_path, + ds.to_netcdf(gunw_netcdf_path, group=layer_data['dst_group'], mode='a') - return netcdf_path + return gunw_netcdf_path diff --git a/isce2_topsapp/packaging_utils/ionosphere.py b/isce2_topsapp/packaging_utils/ionosphere.py new file mode 100644 index 0000000..c2493fe --- /dev/null +++ b/isce2_topsapp/packaging_utils/ionosphere.py @@ -0,0 +1,47 @@ +from pathlib import Path + +import numpy as np +import rasterio +from dem_stitcher.rio_tools import (reproject_arr_to_match_profile, + update_profile_resolution) + + +def format_ionosphere_for_gunw(isce_directory: Path, + gunw_netcdf_path: Path) -> Path: + with rasterio.open(isce_directory / "merged/topophase.ion.geo") as ds: + X_ion = ds.read(1) + p_ion = ds.profile + + X_ion[X_ion == 0] = np.nan + p_ion['nodata'] = np.nan + + # Get GUNW Mask + nc_path_str = (f'netcdf:{gunw_netcdf_path}:' + '/science/grids/data/connectedComponents') + with rasterio.open(nc_path_str) as ds: + cc = ds.read(1) + p_cc = ds.profile + mask = (cc == -1).astype(np.int32) + + # Lower resolution by factor of 11 to approximatly 990 m at equator + p_ion_low_res = update_profile_resolution(p_ion, 0.0091666666) + X_ion_low_res, _ = reproject_arr_to_match_profile(X_ion, + p_ion, + p_ion_low_res, + resampling='bilinear') + p_cc['nodata'] = None + p_cc['dtype'] = np.int32 + mask_low_res, _ = reproject_arr_to_match_profile(mask, + p_cc, + p_ion_low_res, + resampling='nearest') + + X_ion_low_res = X_ion_low_res[0, ...] + mask_low_res = mask_low_res[0, ...].astype(bool) + X_ion_low_res[mask_low_res] = np.nan + + out_path = isce_directory / 'merged/ionosphere_for_gunw.geo' + with rasterio.open(out_path, 'w', **p_ion_low_res) as ds: + ds.write(X_ion_low_res, 1) + + return out_path