From d537f339bd8e814a1ae4cde6156b4afb1aabcf6a Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Thu, 21 Apr 2022 00:12:43 +0200 Subject: [PATCH 1/6] Use XML names not STAC ones in datasets --- tests/test_10_esa_safe.py | 146 ++++++++++++++++--------------- tests/test_30_xarray_backends.py | 22 +++-- xarray_sentinel/esa_safe.py | 66 ++++++++------ 3 files changed, 122 insertions(+), 112 deletions(-) diff --git a/tests/test_10_esa_safe.py b/tests/test_10_esa_safe.py index 9b7cedb..c9ecd36 100644 --- a/tests/test_10_esa_safe.py +++ b/tests/test_10_esa_safe.py @@ -10,88 +10,88 @@ SENTINEL1_ATTRIBUTES = { "S1B_IW_SLC__1SDV_20210401T052622_20210401T052650_026269_032297_EFA4": { - "constellation": "sentinel-1", - "platform": "sentinel-1b", - "instrument": ["c-sar"], - "sat:orbit_state": "descending", - "sat:absolute_orbit": 26269, - "sat:relative_orbit": 168, - "sat:anx_datetime": "2021-04-01T04:49:55.637823Z", - "sar:frequency_band": "C", - "sar:instrument_mode": "IW", - "sar:polarizations": ["VV", "VH"], - "sar:product_type": "SLC", - "xs:instrument_mode_swaths": ["IW1", "IW2", "IW3"], + "ascending_node_time": "2021-04-01T04:49:55.637823", + "family_name": "SENTINEL-1", + "mode": "IW", + "number": "B", + "orbit_number": 26269, + "pass": "DESCENDING", + "product_type": "SLC", + "relative_orbit_number": 168, + "start_time": "2021-04-01T05:26:22.396989", + "stop_time": "2021-04-01T05:26:50.325833", + "swaths": ["IW1", "IW2", "IW3"], + "transmitter_receiver_polarisations": ["VV", "VH"], }, "S1A_S6_SLC__1SDV_20210402T115512_20210402T115535_037271_046407_39FD": { - "constellation": "sentinel-1", - "platform": "sentinel-1a", - "instrument": ["c-sar"], - "sat:orbit_state": "descending", - "sat:absolute_orbit": 37271, - "sat:relative_orbit": 99, - "sat:anx_datetime": "2021-04-02T11:17:22.132050Z", - "sar:frequency_band": "C", - "sar:instrument_mode": "SM", - "sar:polarizations": ["VV", "VH"], - "sar:product_type": "SLC", - "xs:instrument_mode_swaths": ["S6"], + "ascending_node_time": "2021-04-02T11:17:22.132050", + "family_name": "SENTINEL-1", + "mode": "SM", + "number": "A", + "orbit_number": 37271, + "pass": "DESCENDING", + "product_type": "SLC", + "relative_orbit_number": 99, + "start_time": "2021-04-02T11:55:12.030410", + "stop_time": "2021-04-02T11:55:35.706705", + "swaths": ["S6"], + "transmitter_receiver_polarisations": ["VV", "VH"], }, "S1A_S3_SLC__1SDV_20210401T152855_20210401T152914_037258_04638E_6001": { - "constellation": "sentinel-1", - "platform": "sentinel-1a", - "instrument": ["c-sar"], - "sat:orbit_state": "ascending", - "sat:absolute_orbit": 37258, - "sat:relative_orbit": 86, - "sat:anx_datetime": "2021-04-01T13:53:42.874198Z", - "sar:frequency_band": "C", - "sar:instrument_mode": "SM", - "sar:polarizations": ["VV", "VH"], - "sar:product_type": "SLC", - "xs:instrument_mode_swaths": ["S3"], + "ascending_node_time": "2021-04-01T13:53:42.874198", + "family_name": "SENTINEL-1", + "mode": "SM", + "number": "A", + "orbit_number": 37258, + "pass": "ASCENDING", + "product_type": "SLC", + "relative_orbit_number": 86, + "start_time": "2021-04-01T15:28:55.111501", + "stop_time": "2021-04-01T15:29:14.277650", + "swaths": ["S3"], + "transmitter_receiver_polarisations": ["VV", "VH"], }, "S1A_EW_SLC__1SDH_20210403T122536_20210403T122630_037286_046484_8152": { - "constellation": "sentinel-1", - "platform": "sentinel-1a", - "instrument": ["c-sar"], - "sat:orbit_state": "descending", - "sat:absolute_orbit": 37286, - "sat:relative_orbit": 114, - "sat:anx_datetime": "2021-04-03T11:58:30.792178Z", - "sar:frequency_band": "C", - "sar:instrument_mode": "EW", - "sar:polarizations": ["HH", "HV"], - "sar:product_type": "SLC", - "xs:instrument_mode_swaths": ["EW1", "EW2", "EW3", "EW4", "EW5"], + "ascending_node_time": "2021-04-03T11:58:30.792178", + "family_name": "SENTINEL-1", + "mode": "EW", + "number": "A", + "orbit_number": 37286, + "pass": "DESCENDING", + "product_type": "SLC", + "relative_orbit_number": 114, + "start_time": "2021-04-03T12:25:36.505937", + "stop_time": "2021-04-03T12:26:30.902216", + "swaths": ["EW1", "EW2", "EW3", "EW4", "EW5"], + "transmitter_receiver_polarisations": ["HH", "HV"], }, "S1B_WV_SLC__1SSV_20210403T083025_20210403T084452_026300_032390_D542": { - "constellation": "sentinel-1", - "instrument": ["c-sar"], - "platform": "sentinel-1b", - "sar:frequency_band": "C", - "sar:instrument_mode": "WV", - "sar:polarizations": ["VV"], - "sar:product_type": "SLC", - "sat:absolute_orbit": 26300, - "sat:anx_datetime": "2021-04-03T07:50:57.437371Z", - "sat:orbit_state": "descending", - "sat:relative_orbit": 24, - "xs:instrument_mode_swaths": ["WV1", "WV2"], + "ascending_node_time": "2021-04-03T07:50:57.437371", + "family_name": "SENTINEL-1", + "mode": "WV", + "number": "B", + "orbit_number": 26300, + "pass": "DESCENDING", + "product_type": "SLC", + "relative_orbit_number": 24, + "start_time": "2021-04-03T08:30:25.749829", + "stop_time": "2021-04-03T08:44:52.841818", + "swaths": ["WV1", "WV2"], + "transmitter_receiver_polarisations": ["VV"], }, "S1B_IW_GRDH_1SDV_20210401T052623_20210401T052648_026269_032297_ECC8": { - "constellation": "sentinel-1", - "platform": "sentinel-1b", - "instrument": ["c-sar"], - "sat:orbit_state": "descending", - "sat:absolute_orbit": 26269, - "sat:relative_orbit": 168, - "sat:anx_datetime": "2021-04-01T04:49:55.637823Z", - "sar:frequency_band": "C", - "sar:instrument_mode": "IW", - "sar:polarizations": ["VV", "VH"], - "sar:product_type": "GRD", - "xs:instrument_mode_swaths": ["IW"], + "ascending_node_time": "2021-04-01T04:49:55.637823", + "family_name": "SENTINEL-1", + "mode": "IW", + "number": "B", + "orbit_number": 26269, + "pass": "DESCENDING", + "product_type": "GRD", + "relative_orbit_number": 168, + "start_time": "2021-04-01T05:26:23.794457", + "stop_time": "2021-04-01T05:26:48.793373", + "swaths": ["IW"], + "transmitter_receiver_polarisations": ["VV", "VH"], }, } @@ -173,4 +173,6 @@ def test_parse_manifest_sentinel1( res_attrs, res_files = esa_safe.parse_manifest_sentinel1(manifest_path) + print(res_attrs) + assert res_attrs == expected diff --git a/tests/test_30_xarray_backends.py b/tests/test_30_xarray_backends.py index 7ba8226..33250d5 100644 --- a/tests/test_30_xarray_backends.py +++ b/tests/test_30_xarray_backends.py @@ -9,18 +9,16 @@ COMMON_ATTRIBUTES = { - "constellation": "sentinel-1", - "platform": "sentinel-1b", - "instrument": ["c-sar"], - "sat:orbit_state": "descending", - "sat:absolute_orbit": 26269, - "sat:relative_orbit": 168, - "sat:anx_datetime": "2021-04-01T04:49:55.637823Z", - "sar:frequency_band": "C", - "sar:instrument_mode": "IW", - "sar:polarizations": ["VV", "VH"], - "sar:product_type": "SLC", - "xs:instrument_mode_swaths": ["IW1", "IW2", "IW3"], + "family_name": "SENTINEL-1", + "number": "B", + "mode": "IW", + "swaths": ["IW1", "IW2", "IW3"], + "orbit_number": 26269, + "relative_orbit_number": 168, + "pass": "DESCENDING", + "ascending_node_time": "2021-04-01T04:49:55.637823", + "transmitter_receiver_polarisations": ["VV", "VH"], + "product_type": "SLC", } diff --git a/xarray_sentinel/esa_safe.py b/xarray_sentinel/esa_safe.py index a6ad15b..9aba9f0 100644 --- a/xarray_sentinel/esa_safe.py +++ b/xarray_sentinel/esa_safe.py @@ -110,42 +110,52 @@ def parse_manifest_sentinel1( # We use ElementTree because we didn't find a XSD definition for the manifest manifest = ElementTree.parse(manifest_path) - familyName = findtext(manifest, ".//safe:platform/safe:familyName") - if familyName != "SENTINEL-1": - raise ValueError(f"familyName={familyName} not supported") + family_name = findtext(manifest, ".//safe:platform/safe:familyName") + if family_name != "SENTINEL-1": + raise ValueError(f"familyName={family_name} not supported") number = findtext(manifest, ".//safe:platform/safe:number") - instrumentMode = findtext(manifest, ".//s1sarl1:instrumentMode/s1sarl1:mode") + mode = findtext(manifest, ".//s1sarl1:instrumentMode/s1sarl1:mode") swaths = findall(manifest, ".//s1sarl1:instrumentMode/s1sarl1:swath") - polarizations = findall(manifest, ".//s1sarl1:transmitterReceiverPolarisation") - productType = findtext(manifest, ".//s1sarl1:productType") - ascendingNodeTime = findtext(manifest, ".//s1:ascendingNodeTime") - orbitProperties_pass = findtext(manifest, ".//s1:pass") - if orbitProperties_pass not in {"ASCENDING", "DESCENDING"}: - raise ValueError(f"pass={orbitProperties_pass} not supported") + orbit_number = findall(manifest, ".//safe:orbitNumber") + if len(orbit_number) != 2 or orbit_number[0] != orbit_number[1]: + raise ValueError(f"orbitNumber={orbit_number} not supported") - orbitNumber = findall(manifest, ".//safe:orbitNumber") - if len(orbitNumber) != 2 or orbitNumber[0] != orbitNumber[1]: - raise ValueError(f"orbitNumber={orbitNumber} not supported") + relative_orbit_number = findall(manifest, ".//safe:relativeOrbitNumber") + if ( + len(relative_orbit_number) != 2 + or relative_orbit_number[0] != relative_orbit_number[1] + ): + raise ValueError(f"relativeOrbitNumber={relative_orbit_number} not supported") - relative_orbit = findall(manifest, ".//safe:relativeOrbitNumber") - if len(relative_orbit) != 2 or relative_orbit[0] != relative_orbit[1]: - raise ValueError(f"relativeOrbitNumber={relative_orbit} not supported") + orbit_pass = findtext(manifest, ".//s1:pass") + if orbit_pass not in {"ASCENDING", "DESCENDING"}: + raise ValueError(f"pass={orbit_pass} not supported") + + ascending_node_time = findtext(manifest, ".//s1:ascendingNodeTime") + + transmitter_receiver_polarisations = findall( + manifest, ".//s1sarl1:transmitterReceiverPolarisation" + ) + product_type = findtext(manifest, ".//s1sarl1:productType") + + start_time = findtext(manifest, ".//safe:startTime") + stop_time = findtext(manifest, ".//safe:stopTime") attributes = { - "constellation": "sentinel-1", - "platform": "sentinel-1" + number.lower(), - "instrument": ["c-sar"], - "sat:orbit_state": orbitProperties_pass.lower(), - "sat:absolute_orbit": int(orbitNumber[0]), - "sat:relative_orbit": int(relative_orbit[0]), - "sat:anx_datetime": ascendingNodeTime + "Z", - "sar:frequency_band": "C", - "sar:instrument_mode": instrumentMode, - "sar:polarizations": polarizations, - "sar:product_type": productType, - "xs:instrument_mode_swaths": swaths, + "family_name": family_name, + "number": number, + "mode": mode, + "swaths": swaths, + "orbit_number": int(orbit_number[0]), + "relative_orbit_number": int(relative_orbit_number[0]), + "pass": orbit_pass, + "ascending_node_time": ascending_node_time, + "transmitter_receiver_polarisations": transmitter_receiver_polarisations, + "product_type": product_type, + "start_time": start_time, + "stop_time": stop_time, } files = {} From 32f86f79ac5af29dea8b260e6725315603bca0c8 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Thu, 21 Apr 2022 00:14:42 +0200 Subject: [PATCH 2/6] Check existing product --- tests/slow_test_50_cfchecker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/slow_test_50_cfchecker.py b/tests/slow_test_50_cfchecker.py index 36b4e06..1b99134 100644 --- a/tests/slow_test_50_cfchecker.py +++ b/tests/slow_test_50_cfchecker.py @@ -52,10 +52,10 @@ def cfcheck(path: str) -> T.Dict[str, int]: return totals -def test_cfcheck(tmpdir: T.Any) -> None: +def test_cfcheck_grd(tmpdir: T.Any) -> None: product_path = ( DATA_FOLDER - / "S1B_IW_GRDH_1SDV_20211223T051122_20211223T051147_030148_039993_5371.SAFE" + / "S1B_IW_GRDH_1SDV_20210401T052623_20210401T052648_026269_032297_ECC8.SAFE" ) groups = [""] From 6ae2df04c24fddaf00c9cdbd2b72d2022ab0fbea Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Thu, 21 Apr 2022 09:11:14 +0200 Subject: [PATCH 3/6] Drop more STAC like naming --- xarray_sentinel/sentinel1.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/xarray_sentinel/sentinel1.py b/xarray_sentinel/sentinel1.py index 6501e6b..e903ea9 100644 --- a/xarray_sentinel/sentinel1.py +++ b/xarray_sentinel/sentinel1.py @@ -404,7 +404,7 @@ def open_pol_dataset( swath_timing = esa_safe.parse_tag(annotation, ".//swathTiming") number_of_samples = image_information["numberOfSamples"] - slant_range_time_interval = 1 / product_information["rangeSamplingRate"] + range_sampling_rate = product_information["rangeSamplingRate"] number_of_lines = image_information["numberOfLines"] azimuth_time_interval = image_information["azimuthTimeInterval"] @@ -412,13 +412,13 @@ def open_pol_dataset( range_pixel_spacing = image_information["rangePixelSpacing"] attrs = { - "sar:center_frequency": product_information["radarFrequency"] / 10**9, - "sar:pixel_spacing_azimuth": image_information["azimuthPixelSpacing"], - "sar:pixel_spacing_range": range_pixel_spacing, + "radar_frequency": product_information["radarFrequency"] / 10**9, + "azimuth_pixel_spacing": image_information["azimuthPixelSpacing"], + "range_pixel_spacing": range_pixel_spacing, "azimuth_time_interval": azimuth_time_interval, - "slant_range_time_interval": slant_range_time_interval, + "range_sampling_rate": range_sampling_rate, "incidence_angle_mid_swath": image_information["incidenceAngleMidSwath"], - "sat:anx_datetime": image_information["ascendingNodeTime"] + "Z", + "ascending_node_time": image_information["ascendingNodeTime"], } encoding = {} swap_dims = {} @@ -468,14 +468,21 @@ def open_pol_dataset( coords = { "pixel": np.arange(0, number_of_samples, dtype=int), "line": np.arange(0, number_of_lines, dtype=int), - "azimuth_time": ("line", azimuth_time), + # set "units" explicitly as CF conventions don't support "nanoseconds". + # See: https://github.com/pydata/xarray/issues/4183#issuecomment-685200043 + "azimuth_time": ( + "line", + azimuth_time, + {}, + {"units": f"microseconds since {azimuth_time[0]}"}, + ), } if product_information["projection"] == "Slant Range": slant_range_time = np.linspace( image_information["slantRangeTime"], image_information["slantRangeTime"] - + slant_range_time_interval * (number_of_samples - 1), + + (number_of_samples - 1) / range_sampling_rate, number_of_samples, ) coords["slant_range_time"] = ("pixel", slant_range_time) @@ -500,6 +507,9 @@ def open_pol_dataset( except AttributeError: arr = xr.open_dataarray(measurement, engine="rasterio", chunks=chunks) # type: ignore + # clear the encoding as many GeoTIFF details are inconpatible with the CF conventions + arr.encoding.clear() + arr = arr.squeeze("band").drop_vars(["band", "spatial_ref"]) arr = arr.rename({"y": "line", "x": "pixel"}) arr = arr.assign_coords(coords) @@ -517,7 +527,7 @@ def find_bursts_index( use_center: bool = False, ) -> int: lines_per_burst = pol_dataset.attrs["lines_per_burst"] - anx_datetime = np.datetime64(pol_dataset.attrs["sat:anx_datetime"].replace("Z", "")) + anx_datetime = np.datetime64(pol_dataset.attrs["ascending_node_time"]) azimuth_anx_time = pd.Timedelta(azimuth_anx_time, unit="s") if use_center: azimuth_anx_time_center = ( @@ -583,7 +593,7 @@ def crop_burst_dataset( ) ) - anx_datetime = np.datetime64(pol_dataset.attrs["sat:anx_datetime"].replace("Z", "")) + anx_datetime = np.datetime64(pol_dataset.attrs["ascending_node_time"]) burst_azimuth_anx_times = ds.azimuth_time - anx_datetime ds.attrs["azimuth_anx_time"] = burst_azimuth_anx_times.values[0] / ONE_SECOND ds = ds.swap_dims({"line": "azimuth_time", "pixel": "slant_range_time"}) From 6686ce9d5ef780dd6d7b5daa7c95d7819b276b51 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Thu, 21 Apr 2022 10:44:59 +0200 Subject: [PATCH 4/6] Duplicate attributes to subgroups and arrays --- README.md | 224 +++++++++++++++++------------------ xarray_sentinel/sentinel1.py | 79 +++++++----- 2 files changed, 161 insertions(+), 142 deletions(-) diff --git a/README.md b/README.md index 66ca6f6..218c444 100644 --- a/README.md +++ b/README.md @@ -74,19 +74,19 @@ Dimensions: () Data variables: *empty* Attributes: ... - constellation: sentinel-1 - platform: sentinel-1a - instrument: ['c-sar'] - sat:orbit_state: ascending - sat:absolute_orbit: 37258 - sat:relative_orbit: 86 - ... ... - sar:product_type: SLC - xs:instrument_mode_swaths: ['S3'] - group: / - subgroups: ['S3', 'S3/VH', 'S3/VH/orbit', 'S3/VH/attitud... - Conventions: CF-1.8 - history: created by xarray_sentinel-... + family_name: SENTINEL-1 + number: A + mode: SM + swaths: ['S3'] + orbit_number: 37258 + relative_orbit_number: 86 + ... ... + start_time: 2021-04-01T15:28:55.111501 + stop_time: 2021-04-01T15:29:14.277650 + group: / + subgroups: ['S3', 'S3/VH', 'S3/VH/orbit', 'S3/V... + Conventions: CF-1.8 + history: created by xarray_sentinel-... ``` @@ -114,19 +114,19 @@ Coordinates: Data variables: measurement (azimuth_time, slant_range_time) complex64 ... Attributes: ... - sar:center_frequency: 5.40500045433435 - sar:pixel_spacing_azimuth: 3.55338 - sar:pixel_spacing_range: 2.246363 - azimuth_time_interval: 0.0005194923129469381 - slant_range_time_interval: 1.498612395219899e-08 - incidence_angle_mid_swath: 32.03479766845703 - ... ... - sar:product_type: SLC - xs:instrument_mode_swaths: ['S3'] - group: /S3/VH - subgroups: ['orbit', 'attitude', 'azimuth_fm_rate', 'dc_... - Conventions: CF-1.8 - history: created by xarray_sentinel-... + family_name: SENTINEL-1 + number: A + mode: SM + swaths: ['S3'] + orbit_number: 37258 + relative_orbit_number: 86 + ... ... + range_sampling_rate: 66728395.09333333 + incidence_angle_mid_swath: 32.03479766845703 + group: /S3/VH + subgroups: ['orbit', 'attitude', 'azimuth_fm_ra... + Conventions: CF-1.8 + history: created by xarray_sentinel-... ``` @@ -173,19 +173,19 @@ Data variables: gamma (line, pixel) float32 ... dn (line, pixel) float32 ... Attributes: ... - constellation: sentinel-1 - platform: sentinel-1a - instrument: ['c-sar'] - sat:orbit_state: ascending - sat:absolute_orbit: 37258 - sat:relative_orbit: 86 - ... ... - xs:instrument_mode_swaths: ['S3'] - group: /S3/VH/calibration - Conventions: CF-1.8 - title: Calibration coefficients - comment: The dataset contains calibration information ... - history: created by xarray_sentinel-... + family_name: SENTINEL-1 + number: A + mode: SM + swaths: ['S3'] + orbit_number: 37258 + relative_orbit_number: 86 + ... ... + stop_time: 2021-04-01T15:29:14.277650 + group: /S3/VH/calibration + Conventions: CF-1.8 + title: Calibration coefficients + comment: The dataset contains calibration inf... + history: created by xarray_sentinel-... ``` @@ -247,20 +247,20 @@ Coordinates: slant_range_time (pixel) float64 ... Data variables: measurement (line, pixel) complex64 ... -Attributes: (12/26) - sar:center_frequency: 5.40500045433435 - sar:pixel_spacing_azimuth: 13.9283 - sar:pixel_spacing_range: 2.329562 - azimuth_time_interval: 0.002055556299999998 - slant_range_time_interval: 1.554116558005821e-08 - incidence_angle_mid_swath: 33.63858785673874 - ... ... - sar:product_type: SLC - xs:instrument_mode_swaths: ['IW1', 'IW2', 'IW3'] - group: /IW1/HH - subgroups: ['orbit', 'attitude', 'azimuth_fm_rate', 'dc_... - Conventions: CF-1.8 - history: created by xarray_sentinel-... +Attributes: ... + family_name: SENTINEL-1 + number: A + mode: IW + swaths: ['IW1', 'IW2', 'IW3'] + orbit_number: 42768 + relative_orbit_number: 171 + ... ... + number_of_bursts: 9 + lines_per_burst: 1500 + group: /IW1/HH + subgroups: ['orbit', 'attitude', 'azimuth_fm_ra... + Conventions: CF-1.8 + history: created by xarray_sentinel-... ``` @@ -281,20 +281,20 @@ Coordinates: * slant_range_time (slant_range_time) float64 0.005348 0.005349 ... 0.005677 Data variables: measurement (azimuth_time, slant_range_time) complex64 ... -Attributes: (12/27) - sar:center_frequency: 5.40500045433435 - sar:pixel_spacing_azimuth: 13.9283 - sar:pixel_spacing_range: 2.329562 - azimuth_time_interval: 0.002055556299999998 - slant_range_time_interval: 1.554116558005821e-08 - incidence_angle_mid_swath: 33.63858785673874 +Attributes: ... + family_name: SENTINEL-1 + number: A + mode: IW + swaths: ['IW1', 'IW2', 'IW3'] + orbit_number: 42768 + relative_orbit_number: 171 ... ... - group: /IW1/HH - Conventions: CF-1.8 - history: created by xarray_sentinel-... - azimuth_anx_time: 2136.774327 - burst_index: 8 - burst_id: 365923 + group: /IW1/HH + Conventions: CF-1.8 + history: created by xarray_sentinel-... + azimuth_anx_time: 2136.774327 + burst_index: 8 + burst_id: 365923 ``` @@ -312,20 +312,20 @@ Coordinates: * slant_range_time (slant_range_time) float64 0.005348 0.005349 ... 0.005677 Data variables: measurement (azimuth_time, slant_range_time) complex64 ... -Attributes: (12/27) - sar:center_frequency: 5.40500045433435 - sar:pixel_spacing_azimuth: 13.9283 - sar:pixel_spacing_range: 2.329562 - azimuth_time_interval: 0.002055556299999998 - slant_range_time_interval: 1.554116558005821e-08 - incidence_angle_mid_swath: 33.63858785673874 +Attributes: ... + family_name: SENTINEL-1 + number: A + mode: IW + swaths: ['IW1', 'IW2', 'IW3'] + orbit_number: 42768 + relative_orbit_number: 171 ... ... - group: /IW1/HH - Conventions: CF-1.8 - history: created by xarray_sentinel-... - azimuth_anx_time: 2136.774327 - burst_index: 8 - burst_id: 365923 + group: /IW1/HH + Conventions: CF-1.8 + history: created by xarray_sentinel-... + azimuth_anx_time: 2136.774327 + burst_index: 8 + burst_id: 365923 ``` @@ -349,19 +349,19 @@ Coordinates: Data variables: measurement (azimuth_time, slant_range_time) complex64 ... Attributes: ... - sar:center_frequency: 5.40500045433435 - sar:pixel_spacing_azimuth: 13.94053 - sar:pixel_spacing_range: 2.329562 - azimuth_time_interval: 0.002055556299999998 - slant_range_time_interval: 1.554116558005821e-08 - incidence_angle_mid_swath: 33.87494380774521 - ... ... - xs:instrument_mode_swaths: ['IW1', 'IW2', 'IW3'] - group: /IW1/VH - azimuth_anx_time: 2210.634453 - burst_index: 8 - Conventions: CF-1.8 - history: created by xarray_sentinel-... + family_name: SENTINEL-1 + number: B + mode: IW + swaths: ['IW1', 'IW2', 'IW3'] + orbit_number: 26269 + relative_orbit_number: 168 + ... ... + lines_per_burst: 1501 + group: /IW1/VH + azimuth_anx_time: 2210.634453 + burst_index: 8 + Conventions: CF-1.8 + history: created by xarray_sentinel-... ``` @@ -380,26 +380,26 @@ Coordinates: * azimuth_time (azimuth_time) datetime64[ns] 2021-04-01T15:28:55.11150... * slant_range_time (slant_range_time) float64 0.005273 0.005273 ... 0.005303 Attributes: - sar:center_frequency: 5.40500045433435 - sar:pixel_spacing_azimuth: 3.55338 - sar:pixel_spacing_range: 2.246363 - azimuth_time_interval: 0.0005194923129469381 - slant_range_time_interval: 1.498612395219899e-08 - incidence_angle_mid_swath: 32.03479766845703 - sat:anx_datetime: 2021-04-01T13:53:42.874198Z - constellation: sentinel-1 - platform: sentinel-1a - instrument: ['c-sar'] - sat:orbit_state: ascending - sat:absolute_orbit: 37258 - sat:relative_orbit: 86 - sar:frequency_band: C - sar:instrument_mode: SM - sar:polarizations: ['VV', 'VH'] - sar:product_type: SLC - xs:instrument_mode_swaths: ['S3'] - units: m2 m-2 - long_name: gamma + family_name: SENTINEL-1 + number: A + mode: SM + swaths: ['S3'] + orbit_number: 37258 + relative_orbit_number: 86 + pass: ASCENDING + ascending_node_time: 2021-04-01T13:53:42.874198 + transmitter_receiver_polarisations: ['VV', 'VH'] + product_type: SLC + start_time: 2021-04-01T15:28:55.111501 + stop_time: 2021-04-01T15:29:14.277650 + radar_frequency: 5.40500045433435 + azimuth_pixel_spacing: 3.55338 + range_pixel_spacing: 2.246363 + azimuth_time_interval: 0.0005194923129469381 + range_sampling_rate: 66728395.09333333 + incidence_angle_mid_swath: 32.03479766845703 + units: m2 m-2 + long_name: gamma ``` diff --git a/xarray_sentinel/sentinel1.py b/xarray_sentinel/sentinel1.py index e903ea9..b3e5e16 100644 --- a/xarray_sentinel/sentinel1.py +++ b/xarray_sentinel/sentinel1.py @@ -63,7 +63,9 @@ def normalise_group(group: T.Optional[str]) -> T.Tuple[str, T.Optional[int]]: return group, burst_index -def open_calibration_dataset(calibration: esa_safe.PathType) -> xr.Dataset: +def open_calibration_dataset( + calibration: esa_safe.PathType, attrs: T.Dict[str, T.Any] = {} +) -> xr.Dataset: calibration_vectors = esa_safe.parse_tag_as_list( calibration, ".//calibrationVector", "calibration" ) @@ -103,10 +105,12 @@ def open_calibration_dataset(calibration: esa_safe.PathType) -> xr.Dataset: } coords = {"line": line_list, "pixel": pixel_list[0]} - return xr.Dataset(data_vars=data_vars, coords=coords) + return xr.Dataset(data_vars=data_vars, coords=coords, attrs=attrs) -def open_noise_range_dataset(noise: esa_safe.PathType) -> xr.Dataset: +def open_noise_range_dataset( + noise: esa_safe.PathType, attrs: T.Dict[str, T.Any] = {} +) -> xr.Dataset: noise_vectors = esa_safe.parse_tag_as_list(noise, ".//noiseRangeVector", "noise") azimuth_time_list = [] @@ -132,10 +136,12 @@ def open_noise_range_dataset(noise: esa_safe.PathType) -> xr.Dataset: } coords = {"line": line_list, "pixel": pixel_list[0]} - return xr.Dataset(data_vars=data_vars, coords=coords) + return xr.Dataset(data_vars=data_vars, coords=coords, attrs=attrs) -def open_noise_azimuth_dataset(noise: esa_safe.PathType) -> xr.Dataset: +def open_noise_azimuth_dataset( + noise: esa_safe.PathType, attrs: T.Dict[str, T.Any] = {} +) -> xr.Dataset: noise_vectors = esa_safe.parse_tag_as_list(noise, ".//noiseAzimuthVector", "noise") first_range_sample = [] @@ -155,11 +161,11 @@ def open_noise_azimuth_dataset(noise: esa_safe.PathType) -> xr.Dataset: data_vars["noiseAzimuthLut"] = ("line", noiseAzimuthLut_list[0]) coords["line"] = line_list[0] - return xr.Dataset(data_vars=data_vars, coords=coords) + return xr.Dataset(data_vars=data_vars, coords=coords, attrs=attrs) def open_coordinate_conversion_dataset( - annotation_path: esa_safe.PathType, + annotation_path: esa_safe.PathType, attrs: T.Dict[str, T.Any] = {} ) -> xr.Dataset: coordinate_conversion = esa_safe.parse_tag_as_list( annotation_path, ".//coordinateConversionList/coordinateConversion" @@ -195,10 +201,12 @@ def open_coordinate_conversion_dataset( data_vars["srgrCoefficients"] = (("azimuth_time", "degree"), srgrCoefficients) data_vars["grsrCoefficients"] = (("azimuth_time", "degree"), grsrCoefficients) - return xr.Dataset(data_vars=data_vars, coords=coords) + return xr.Dataset(data_vars=data_vars, coords=coords, attrs=attrs) -def open_gcp_dataset(annotation: esa_safe.PathOrFileType) -> xr.Dataset: +def open_gcp_dataset( + annotation: esa_safe.PathOrFileType, attrs: T.Dict[str, T.Any] = {} +) -> xr.Dataset: geolocation_grid_points = esa_safe.parse_tag_as_list( annotation, ".//geolocationGridPoint" ) @@ -217,11 +225,11 @@ def open_gcp_dataset(annotation: esa_safe.PathOrFileType) -> xr.Dataset: shape = (len(azimuth_time), len(slant_range_time)) dims = ("azimuth_time", "slant_range_time") data_vars = { - "latitude": (dims, np.full(shape, np.nan)), - "longitude": (dims, np.full(shape, np.nan)), - "height": (dims, np.full(shape, np.nan)), - "incidenceAngle": (dims, np.full(shape, np.nan)), - "elevationAngle": (dims, np.full(shape, np.nan)), + "latitude": (dims, np.full(shape, np.nan), attrs), + "longitude": (dims, np.full(shape, np.nan), attrs), + "height": (dims, np.full(shape, np.nan), attrs), + "incidenceAngle": (dims, np.full(shape, np.nan), attrs), + "elevationAngle": (dims, np.full(shape, np.nan), attrs), } line = sorted(line_set) pixel = sorted(pixel_set) @@ -239,11 +247,14 @@ def open_gcp_dataset(annotation: esa_safe.PathOrFileType) -> xr.Dataset: "line": ("azimuth_time", line), "pixel": ("slant_range_time", pixel), }, + attrs=attrs, ) return ds -def open_attitude_dataset(annotation: esa_safe.PathOrFileType) -> xr.Dataset: +def open_attitude_dataset( + annotation: esa_safe.PathOrFileType, attrs: T.Dict[str, T.Any] = {} +) -> xr.Dataset: attitudes = esa_safe.parse_tag_as_list(annotation, ".//attitude") variables = ["q0", "q1", "q2", "q3", "wx", "wy", "wz", "pitch", "roll", "yaw"] @@ -257,12 +268,15 @@ def open_attitude_dataset(annotation: esa_safe.PathOrFileType) -> xr.Dataset: ds = xr.Dataset( data_vars=data_vars, coords={"azimuth_time": [np.datetime64(dt) for dt in azimuth_time]}, + attrs=attrs, ) return ds -def open_orbit_dataset(annotation: esa_safe.PathOrFileType) -> xr.Dataset: +def open_orbit_dataset( + annotation: esa_safe.PathOrFileType, attrs: T.Dict[str, T.Any] = {} +) -> xr.Dataset: orbits = esa_safe.parse_tag_as_list(annotation, ".//orbit") reference_system = orbits[0]["frame"] @@ -286,7 +300,7 @@ def open_orbit_dataset(annotation: esa_safe.PathOrFileType) -> xr.Dataset: position = xr.Variable(data=data["position"], dims=("axis", "azimuth_time")) # type: ignore velocity = xr.Variable(data=data["velocity"], dims=("axis", "azimuth_time")) # type: ignore - attrs = {} + attrs = attrs | {} if reference_system is not None: attrs.update({"reference_system": reference_system}) @@ -302,7 +316,9 @@ def open_orbit_dataset(annotation: esa_safe.PathOrFileType) -> xr.Dataset: return ds -def open_dc_estimate_dataset(annotation: esa_safe.PathOrFileType) -> xr.Dataset: +def open_dc_estimate_dataset( + annotation: esa_safe.PathOrFileType, attrs: T.Dict[str, T.Any] = {} +) -> xr.Dataset: dc_estimates = esa_safe.parse_tag_as_list(annotation, ".//dcEstimate") azimuth_time = [] @@ -324,11 +340,14 @@ def open_dc_estimate_dataset(annotation: esa_safe.PathOrFileType) -> xr.Dataset: "azimuth_time": [np.datetime64(at) for at in azimuth_time], "degree": list(range(len(data_dc_poly[0]))), }, + attrs=attrs, ) return ds -def open_azimuth_fm_rate_dataset(annotation: esa_safe.PathOrFileType) -> xr.Dataset: +def open_azimuth_fm_rate_dataset( + annotation: esa_safe.PathOrFileType, attrs: T.Dict[str, T.Any] = {} +) -> xr.Dataset: azimuth_fm_rates = esa_safe.parse_tag_as_list(annotation, ".//azimuthFmRate") azimuth_time = [] @@ -353,6 +372,7 @@ def open_azimuth_fm_rate_dataset(annotation: esa_safe.PathOrFileType) -> xr.Data "azimuth_time": [np.datetime64(at) for at in azimuth_time], "degree": list(range(len(azimuth_fm_rate_poly[0]))), }, + attrs=attrs, ) return ds @@ -397,6 +417,7 @@ def open_pol_dataset( measurement: esa_safe.PathOrFileType, annotation: esa_safe.PathOrFileType, fs: T.Optional[fsspec.AbstractFileSystem] = None, + attrs: T.Dict[str, T.Any] = {}, ) -> xr.Dataset: product_information = esa_safe.parse_tag(annotation, ".//productInformation") @@ -411,7 +432,7 @@ def open_pol_dataset( number_of_bursts = swath_timing["burstList"]["@count"] range_pixel_spacing = image_information["rangePixelSpacing"] - attrs = { + attrs = attrs | { "radar_frequency": product_information["radarFrequency"] / 10**9, "azimuth_pixel_spacing": image_information["azimuthPixelSpacing"], "range_pixel_spacing": range_pixel_spacing, @@ -748,7 +769,7 @@ def open_sentinel1_dataset( product_path = os.path.dirname(manifest_path) with fs.open(manifest_path) as file: - product_attrs, product_files = esa_safe.parse_manifest_sentinel1(file) + common_attrs, product_files = esa_safe.parse_manifest_sentinel1(file) if override_product_files: product_files = do_override_product_files(override_product_files, product_files) @@ -767,7 +788,7 @@ def open_sentinel1_dataset( metadata = "" - ds = xr.Dataset() + ds = xr.Dataset(attrs=common_attrs) if group == "": subgroups = list(groups) else: @@ -777,19 +798,17 @@ def open_sentinel1_dataset( if group.count("/") == 1: with fs.open(groups[group][1]) as annotation: - ds = open_pol_dataset(groups[group][0], annotation, fs=fs) + ds = open_pol_dataset( + groups[group][0], annotation, fs=fs, attrs=common_attrs + ) elif group.count("/") == 2: _, _, metadata = group.split("/", 2) with fs.open(groups[group][0]) as file: - ds = METADATA_OPENERS[metadata](file) - - for data_var in ds.data_vars: - ds.data_vars[data_var].attrs.update(product_attrs) + ds = METADATA_OPENERS[metadata](file, attrs=common_attrs) - product_attrs["group"] = absgroup + ds.attrs["group"] = absgroup if len(subgroups): - product_attrs["subgroups"] = subgroups - ds.attrs.update(product_attrs) # type: ignore + ds.attrs["subgroups"] = subgroups if group.count("/") == 1 and burst_index is not None: ds = crop_burst_dataset(ds, burst_index=burst_index) From 4c42bc4b410b44f89abc709dca3dcc03499c0010 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Thu, 21 Apr 2022 10:54:41 +0200 Subject: [PATCH 5/6] Add more attrs to data variables --- xarray_sentinel/sentinel1.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/xarray_sentinel/sentinel1.py b/xarray_sentinel/sentinel1.py index b3e5e16..46d51c3 100644 --- a/xarray_sentinel/sentinel1.py +++ b/xarray_sentinel/sentinel1.py @@ -259,7 +259,8 @@ def open_attitude_dataset( variables = ["q0", "q1", "q2", "q3", "wx", "wy", "wz", "pitch", "roll", "yaw"] azimuth_time: T.List[T.Any] = [] - data_vars: T.Dict[str, T.Any] = {var: ("azimuth_time", []) for var in variables} + data_vars: T.Dict[str, T.Any] + data_vars = {var: ("azimuth_time", [], attrs) for var in variables} for attitude in attitudes: azimuth_time.append(attitude["time"]) for var in variables: @@ -312,6 +313,8 @@ def open_orbit_dataset( "axis": [0, 1, 2], }, ) + for data_var in ds.data_vars: + ds[data_var].attrs = attrs return ds @@ -333,8 +336,8 @@ def open_dc_estimate_dataset( ds = xr.Dataset( data_vars={ - "t0": ("azimuth_time", t0), - "data_dc_polynomial": (("azimuth_time", "degree"), data_dc_poly), + "t0": ("azimuth_time", t0, attrs), + "data_dc_polynomial": (("azimuth_time", "degree"), data_dc_poly, attrs), }, coords={ "azimuth_time": [np.datetime64(at) for at in azimuth_time], @@ -362,10 +365,15 @@ def open_azimuth_fm_rate_dataset( ds = xr.Dataset( data_vars={ - "t0": ("azimuth_time", t0), + "t0": ( + "azimuth_time", + t0, + attrs, + ), "azimuth_fm_rate_polynomial": ( ("azimuth_time", "degree"), azimuth_fm_rate_poly, + attrs, ), }, coords={ From 90ef68ef76e1366ad94054e2ac31370937169aa3 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Thu, 21 Apr 2022 10:55:12 +0200 Subject: [PATCH 6/6] Support writing partial products --- xarray_sentinel/reformat.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/xarray_sentinel/reformat.py b/xarray_sentinel/reformat.py index 5cf03fc..094dd75 100644 --- a/xarray_sentinel/reformat.py +++ b/xarray_sentinel/reformat.py @@ -17,8 +17,11 @@ def to_group_zarr( groups = {g: g for g in root.attrs["subgroups"]} for group_out, group_in in groups.items(): - group_ds = xr.open_dataset(product_path, engine="sentinel-1", group=group_in) # type: ignore - group_ds.to_zarr(output_store, mode="a", group=group_out) + try: + group_ds = xr.open_dataset(product_path, engine="sentinel-1", group=group_in) # type: ignore + group_ds.to_zarr(output_store, mode="a", group=group_out) + except FileNotFoundError: + pass # Apparently there is no way to save SLC images because "netcdf4" doesn't support complex data @@ -36,5 +39,8 @@ def to_group_netcdf( groups = {g: g for g in root.attrs["subgroups"]} for group_out, group_in in groups.items(): - group_ds = xr.open_dataset(product_path, engine="sentinel-1", group=group_in) # type: ignore - group_ds.to_netcdf(output_store, mode="a", group=group_out, engine=engine) + try: + group_ds = xr.open_dataset(product_path, engine="sentinel-1", group=group_in) # type: ignore + group_ds.to_netcdf(output_store, mode="a", group=group_out, engine=engine) + except FileNotFoundError: + pass