Merge pull request #157 from ACCESS-Cloud-Based-InSAR/dev

Release v0.3.0 -- ESA CDSE cutover and timstamp fixes

CHANGELOG.md

@@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.0]
+
+### Added
+* `check_esa_credentials` function to `__main__.py` to check for the existence of Dataspace credentials before processing begins.
+
+### Changed
+* Updated `hyp3lib` to  v2.0.2+, which uses the new Copernicus Dataspace Ecosystem API to download orbit files.
+* Calls to `downloadSentinelOrbitFile` now specify the `esa_credentials` argument.
+
 ## [0.2.5]
 
 ### Fixed

README.md

@@ -30,9 +30,12 @@ We note all the input datasets are publicly available using a NASA Earthdata acc
     machine urs.earthdata.nasa.gov
         login <username>
         password <password>
-    ```
-    The `username`/`password` are the appropriate Earthdata Login credentials that are used to access NASA data. This file is necessary for downloading the Sentinel-1 orbit files from the ASF DAAC. Additionally, the [`requests`](https://docs.python-requests.org/en/latest/) library automatically uses credentials stored in the `~/.netrc` for authentification when none are supplied.
 
+    machine dataspace.copernicus.eu
+        login <username>
+        password <password>
+    ```
+    The first `username`/`password` pair are the appropriate Earthdata Login credentials that are used to access NASA data. The second pair are your credentials for the [Copernicus Data Space Ecosystem](https://dataspace.copernicus.eu). This file is necessary for downloading the Sentinel-1 files, and auxiliary data. Additionally, the [`requests`](https://docs.python-requests.org/en/latest/) library automatically uses credentials stored in the `~/.netrc` for authentification when none are supplied.
 
 ## Generate a GUNW
 
@@ -112,6 +115,8 @@ isce2_topsapp --reference-scenes S1A_IW_SLC__1SDV_20230125T140019_20230125T14004
     ```
     isce2_topsapp --username <username> \
                   --password <password> \
+                  --esa-username <esa-username> \
+                  --esa-password <esa-password> \
                   --reference-scenes S1B_IW_SLC__1SDV_20210723T014947_20210723T015014_027915_0354B4_B3A9 \
                   --secondary-scenes S1B_IW_SLC__1SDV_20210711T014922_20210711T014949_027740_034F80_859D \
                                      S1B_IW_SLC__1SDV_20210711T014947_20210711T015013_027740_034F80_D404 \
@@ -141,6 +146,8 @@ docker run -ti -v $PWD:/home/ops/topsapp_data topsapp_img \
                                     S1B_IW_SLC__1SDV_20210711T015011_20210711T015038_027740_034F80_376C \
                --username <username>
                --password <password>
+               --esa-username <esa-username> \
+               --esa-password <esa-password> \
 ```
 where the `username`/`password` are the Earthdata credentials for accessing NASA data. We note the command line magic of the above is taken care of the `isce2_topsapp/etc/entrypoint.sh` (written by Joe Kennedy) which automatically runs certain bash commands on startup of the container, i.e. the run commands also calls the `isce2_topsapp` command line function as can be seen [here](isce2_topsapp/etc/entrypoint.sh).
 

environment.yml

@@ -15,7 +15,7 @@ dependencies:
  - fsspec
  - gdal
  - geopandas
- - hyp3lib>=1.7
+ - hyp3lib>=2,<3
  - ipykernel
  - isce2==2.6.1
  - jinja2

isce2_topsapp/__main__.py

@@ -6,6 +6,7 @@
 from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
 from importlib.metadata import entry_points
 from pathlib import Path
+from platform import system
 from typing import Optional
 
 from isce2_topsapp import (
@@ -29,6 +30,9 @@
 from isce2_topsapp.solid_earth_tides import update_gunw_with_solid_earth_tide
 
 
+ESA_HOST = 'dataspace.copernicus.eu'
+
+
 def localize_data(
     reference_scenes: list,
     secondary_scenes: list,
@@ -58,13 +62,13 @@ def localize_data(
         # For ionospheric correction computation
         if water_mask_flag:
             out_water_mask = download_water_mask(
-                out_slc["extent"],  water_name='SWBD')
+                out_slc["extent"],  water_name="SWBD")
 
         out_aux_cal = download_aux_cal()
 
-    out = {'reference_scenes': reference_scenes,
-           'secondary_scenes': secondary_scenes,
-           'frame_id': frame_id,
+    out = {"reference_scenes": reference_scenes,
+           "secondary_scenes": secondary_scenes,
+           "frame_id": frame_id,
            **out_slc,
            **out_dem,
            **out_water_mask,
@@ -109,6 +113,35 @@ def ensure_earthdata_credentials(
         )
 
 
+def check_esa_credentials(username: Optional[str], password: Optional[str]) -> None:
+    netrc_name = '_netrc' if system().lower() == 'windows' else '.netrc'
+    netrc_file = Path.home() / netrc_name
+
+    if (username is not None) != (password is not None):
+        raise ValueError('Both username and password arguments must be provided')
+
+    if username is not None:
+        os.environ["ESA_USERNAME"] = username
+        os.environ["ESA_PASSWORD"] = password
+        return
+
+    if "ESA_USERNAME" in os.environ and "ESA_PASSWORD" in os.environ:
+        return
+
+    if netrc_file.exists():
+        netrc_credentials = netrc.netrc(netrc_file)
+        if ESA_HOST in netrc_credentials.hosts:
+            os.environ["ESA_USERNAME"] = netrc_credentials.hosts[ESA_HOST][0]
+            os.environ["ESA_PASSWORD"] = netrc_credentials.hosts[ESA_HOST][2]
+            return
+
+    raise ValueError(
+        "Please provide Copernicus Data Space Ecosystem (CDSE) credentials via the "
+        "--esa-username and --esa-password options, "
+        "the ESA_USERNAME and ESA_PASSWORD environment variables, or your netrc file."
+    )
+
+
 def true_false_string_argument(s: str) -> bool:
     s = s.lower()
     if s not in ("true", "false"):
@@ -136,6 +169,8 @@ def gunw_slc():
     parser = ArgumentParser()
     parser.add_argument("--username")
     parser.add_argument("--password")
+    parser.add_argument("--esa-username")
+    parser.add_argument("--esa-password")
     parser.add_argument("--bucket")
     parser.add_argument("--bucket-prefix", default="")
     parser.add_argument("--dry-run", action="store_true")
@@ -154,6 +189,7 @@ def gunw_slc():
     args = parser.parse_args()
 
     ensure_earthdata_credentials(args.username, args.password)
+    check_esa_credentials(args.esa_username, args.esa_password)
 
     args.reference_scenes = [
         item for sublist in args.reference_scenes for item in sublist
@@ -247,6 +283,8 @@ def gunw_burst():
     parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
     parser.add_argument("--username")
     parser.add_argument("--password")
+    parser.add_argument("--esa-username")
+    parser.add_argument("--esa-password")
     parser.add_argument("--bucket")
     parser.add_argument("--bucket-prefix", default="")
     parser.add_argument("--dry-run", action="store_true")
@@ -262,6 +300,7 @@ def gunw_burst():
     args = parser.parse_args()
 
     ensure_earthdata_credentials(args.username, args.password)
+    check_esa_credentials(args.esa_username, args.esa_password)
 
     ref_obj, sec_obj = get_asf_slc_objects(
         [args.reference_scene, args.secondary_scene])

isce2_topsapp/delivery_prep.py

@@ -117,7 +117,6 @@ def gen_browse_imagery(nc_path: Path,
 def format_metadata(nc_path: Path,
                     all_metadata: dict) -> dict:
 
-    now = datetime.datetime.now()
     label = nc_path.name[:-3]  # removes suffix .nc
     geojson = all_metadata['gunw_geo'].__geo_interface__
 
@@ -140,6 +139,7 @@ def format_metadata(nc_path: Path,
     # The %f is miliseconds zero-padded with 6 decimals - just as we need!
     ref_start_time_formatted = ref_start_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
     ref_stop_time_formatted = ref_stop_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+    creation_timestamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%fZ')
 
     # We want the nearest day (dt.days takes a floor) so we use total seconds and then round
     temporal_baseline_seconds = (ref_start_time - sec_start_time).total_seconds()
@@ -171,7 +171,7 @@ def format_metadata(nc_path: Path,
 
     data = {"label": label,
             "location": geojson,
-            "creation_timestamp": f'{now.isoformat()}Z',
+            "creation_timestamp": creation_timestamp,
             "version": DATASET_VERSION,
             "metadata": metadata}
 

isce2_topsapp/localize_orbits.py

@@ -1,10 +1,13 @@
+import os
 from pathlib import Path
 
 import requests
 from hyp3lib import get_orb
 
 
-def _spoof_orbit_download(scene, _, providers=('ESA', 'ASF'), orbit_types=('AUX_POEORB', 'AUX_RESORB')):
+def _spoof_orbit_download(
+    scene, _, providers=('ESA', 'ASF'), orbit_types=('AUX_POEORB', 'AUX_RESORB'), esa_credentials=('name', 'password')
+):
     for orbit_type in orbit_types:
         for provider in providers:
             try:
@@ -17,10 +20,11 @@ def _spoof_orbit_download(scene, _, providers=('ESA', 'ASF'), orbit_types=('AUX_
     return None, None
 
 
-def download_orbits(reference_scenes: list,
-                    secondary_scenes: list,
-                    orbit_directory: str = None,
-                    dry_run: bool = False) -> dict:
+def download_orbits(
+    reference_scenes: list, secondary_scenes: list, orbit_directory: str = None, dry_run: bool = False
+) -> dict:
+    esa_credentials = (os.environ['ESA_USERNAME'], os.environ['ESA_PASSWORD'])
+
     orbit_directory = orbit_directory or 'orbits'
     orbit_dir = Path(orbit_directory)
     orbit_dir.mkdir(exist_ok=True)
@@ -29,12 +33,12 @@ def download_orbits(reference_scenes: list,
 
     reference_orbits = []
     for scene in reference_scenes:
-        orbit_file, _ = orbit_fetcher(scene, str(orbit_dir))
+        orbit_file, _ = orbit_fetcher(scene, str(orbit_dir), esa_credentials=esa_credentials)
         reference_orbits.append(orbit_file)
 
     secondary_orbits = []
     for scene in secondary_scenes:
-        orbit_file, _ = orbit_fetcher(scene, str(orbit_dir))
+        orbit_file, _ = orbit_fetcher(scene, str(orbit_dir), esa_credentials=esa_credentials)
         secondary_orbits.append(orbit_file)
 
     reference_orbits = list(set(reference_orbits))

sample_run.sh

@@ -1,7 +1,9 @@
 isce2_topsapp --username <username> \
               --password <password> \
+              --esa-username <esa-username> \
+              --esa-password <esa-password> \
               --reference-scenes S1B_IW_SLC__1SDV_20210723T014947_20210723T015014_027915_0354B4_B3A9 \
               --secondary-scenes S1B_IW_SLC__1SDV_20210711T014922_20210711T014949_027740_034F80_859D \
                                  S1B_IW_SLC__1SDV_20210711T014947_20210711T015013_027740_034F80_D404 \
                                  S1B_IW_SLC__1SDV_20210711T015011_20210711T015038_027740_034F80_376C \
-             > topsapp_img.out 2> topsapp_img.err
+             > topsapp_img.out 2> topsapp_img.err

setup.py

@@ -34,7 +34,7 @@
         'dateparser',
         'dem_stitcher>=2.1',
         'geopandas',
-        'hyp3lib>=1.7',
+        'hyp3lib>=2,<3',
         'jinja2',
         'lxml',
         'matplotlib',

tests/localize-data.ipynb

@@ -116,7 +116,8 @@
    "id": "mounted-title",
    "metadata": {},
    "source": [
-    "# Orbits"
+    "# Orbits\n",
+    "Make sure `ESA_USERNAME` and `ESA_PASSWORD` environment variables are set before running this."
    ]
   },
   {
@@ -249,9 +250,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "topsapp_env",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "topsapp_env"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -263,7 +264,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.11.5"
   }
  },
  "nbformat": 4,

tests/prepare-for-delivery.ipynb

@@ -485,9 +485,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "topsapp_env",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "topsapp_env"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -499,7 +499,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.11.5"
   }
  },
  "nbformat": 4,

tests/solid_earth_tides.ipynb

@@ -24,6 +24,7 @@
    "outputs": [],
    "source": [
     "from pathlib import Path\n",
+    "import os\n",
     "import xarray as xr\n",
     "import requests\n",
     "from hyp3lib import get_orb\n",
@@ -89,6 +90,14 @@
     "slc_id"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "44fe249b-5691-4d96-9e11-1418ea9c2f4b",
+   "metadata": {},
+   "source": [
+    "**Make sure the `ESA_USERNAME` and `ESA_PASSWORD` environment variables are set before running this.**"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 7,
@@ -114,7 +123,8 @@
     }
    ],
    "source": [
-    "orb_file, _ = get_orb.downloadSentinelOrbitFile(slc_id)\n",
+    "esa_credentials = (os.environ['ESA_USERNAME'], os.environ['ESA_PASSWORD'])\n",
+    "orb_file, _ = get_orb.downloadSentinelOrbitFile(slc_id, esa_credentials=esa_credentials)\n",
     "orb_file"
    ]
   },
@@ -7727,9 +7737,9 @@
    "hash": "83c8da8660bbe18150fdc09aeaa55e9731d119a6641346a233b76fde249768bb"
   },
   "kernelspec": {
-   "display_name": "topsapp_env",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "topsapp_env"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -7741,7 +7751,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.11.5"
   }
  },
  "nbformat": 4,