Merge pull request #2192 from ASFHyP3/raider-job

add a raider job for the ARIA project

.github/workflows/deploy-enterprise-test.yml

@@ -46,7 +46,9 @@ jobs:
             default_credits_per_user: 0
             reset_credits_monthly: true
             cost_profile: DEFAULT
-            job_files: job_spec/INSAR_ISCE_TEST.yml
+            job_files: >-
+              job_spec/ARIA_RAIDER.yml
+              job_spec/INSAR_ISCE_TEST.yml
             instance_types: m6id.xlarge,m6id.2xlarge,m6id.4xlarge,m6id.8xlarge,m6idn.xlarge,m6idn.2xlarge,m6idn.4xlarge,m6idn.8xlarge
             default_max_vcpus: 640
             expanded_max_vcpus: 640

.github/workflows/deploy-enterprise.yml

@@ -41,7 +41,9 @@ jobs:
             default_credits_per_user: 0
             reset_credits_monthly: true
             cost_profile: DEFAULT
-            job_files: job_spec/INSAR_ISCE.yml
+            job_files: >-
+              job_spec/ARIA_RAIDER.yml
+              job_spec/INSAR_ISCE.yml
             instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
             default_max_vcpus: 4000
             expanded_max_vcpus: 4000

CHANGELOG.md

@@ -8,9 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - A `hyp3-its-live-test` deployment to [`deploy-enterprise-test.yml`](.github/workflows/deploy-enterprise-test.yml) for ITS_LIVE testing in preparation for some significant ITS_LIVE project development
 - A `hyp3-a19-jpl-test` deployment to [`deploy-enterprise-test.yml`](.github/workflows/deploy-enterprise-test.yml) for ARIA testing of the `m6id[n]` instance families
+- An `ARIA_RAIDER` job spec that allows RAIDER processing of previous INSAR_ISCE job that either did not include a weather model or failed on the RAiDER step.
+- `ARIA_RAIDER` jobs are now available in the `hyp3-a19-jpl` and `hyp3-a19-jpl-test` deployments.
 
 ### Changed
 - The `INSAR_ISCE_TEST.yml` job spec now only differs from the `INSAR_ISCE.yml` with respect to the `++omp-num-threads` parameter, because the value is specific to a particular instance family
+- Job specs are no longer required to include the `granules` parameter.
 
 ### Removed
 - The `AUTORIFT_ITS_LIVE_TEST.yml` job spec which supported running test versions of the AUTORIFT jobs in the production hyp3-its-live deployment

apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2

@@ -23,7 +23,7 @@ components:
 
     {% for job_type, job_spec in job_types.items() %}
     {{ job_type }}Parameters:
-      description: Parameters for running {{ job_type }} jobs, including specific granules
+      description: Parameters for running {{ job_type }} jobs
       type: object
       additionalProperties: false
       {% for parameter, parameter_spec in job_spec['parameters'].items() if 'api_schema' in parameter_spec and parameter in job_spec.get('required_parameters', []) %}

apps/api/src/hyp3_api/handlers.py

@@ -1,9 +1,7 @@
 from http.client import responses
-from uuid import UUID
 
 import requests
 from flask import abort, jsonify, request
-from jsonschema import draft4_format_checker
 
 import dynamo
 from hyp3_api import util
@@ -22,15 +20,6 @@ def problem_format(status, message):
     return response
 
 
-@draft4_format_checker.checks('uuid')
-def is_uuid(val):
-    try:
-        UUID(val, version=4)
-    except ValueError:
-        return False
-    return True
-
-
 def post_jobs(body, user):
     print(body)
 

apps/api/src/hyp3_api/util.py

@@ -8,12 +8,8 @@ class TokenDeserializeError(Exception):
     """Raised when paging results and `start_token` fails to deserialize"""
 
 
-def get_granules(jobs):
-    granules = set()
-    for job in jobs:
-        for granule in job['job_parameters']['granules']:
-            granules.add(granule)
-    return granules
+def get_granules(jobs: list[dict]) -> set[str]:
+    return {granule for job in jobs for granule in job['job_parameters'].get('granules', [])}
 
 
 def serialize(payload: dict):

job_spec/ARIA_RAIDER.yml

@@ -0,0 +1,55 @@
+ARIA_RAIDER:
+  required_parameters:
+    - job_id
+  parameters:
+    job_id:
+      api_schema:
+        description: HyP3 job ID for an INSAR_ISCE job that has successfully completed the DockerizedTopsApp step
+        type: string
+        format: uuid
+        example: 27836b79-e5b2-4d8f-932f-659724ea02c3
+    weather_model:
+      api_schema:
+        description: Weather model used to generate tropospheric delay estimations.
+        default: None
+        type: string
+        enum:
+          - None
+          - ERA5
+          - ERA5T
+          - GMAO
+          - HRES
+          - HRRR
+    bucket_prefix:
+      default:  '""'
+  cost_profiles:
+    DEFAULT:
+      cost: 1.0
+  validators: []
+  tasks:
+    - name: ''
+      image: ghcr.io/dbekaert/raider
+      command:
+        - ++process
+        - calcDelaysGUNW
+        - --bucket
+        - '!Ref Bucket'
+        - --bucket-prefix
+        - Ref::bucket_prefix
+        - --weather-model
+        - Ref::weather_model
+        - --input-bucket-prefix
+        - Ref::job_id
+      timeout: 10800
+      vcpu: 1
+      memory: 7500
+      secrets:
+        - EARTHDATA_USERNAME
+        - EARTHDATA_PASSWORD
+        - ESA_USERNAME
+        - ESA_PASSWORD
+        - RAIDER_ECMWF_ERA5_UID
+        - RAIDER_ECMWF_ERA5_API_KEY
+        - RAIDER_HRES_EMAIL
+        - RAIDER_HRES_API_KEY
+        - RAIDER_HRES_URL

tests/test_api/test_util.py

@@ -6,6 +6,20 @@
 from hyp3_api import util
 
 
+def test_get_granules():
+    assert util.get_granules(
+        [
+            {'job_parameters': {'granules': []}},
+            {'job_parameters': {'granules': ['A']}},
+            {'job_parameters': {'granules': ['B']}},
+            {'job_parameters': {'granules': ['C', 'D']}},
+            {'job_parameters': {'granules': ['C', 'D', 'E']}},
+            {'job_parameters': {'granules': ['F', 'F']}},
+            {'job_parameters': {}},
+        ]
+    ) == {'A', 'B', 'C', 'D', 'E', 'F'}
+
+
 def test_serialize_token():
     token = {'foo': 1, 'bar': 2}
     assert util.serialize(token) == 'eyJmb28iOiAxLCAiYmFyIjogMn0='

tests/test_api/test_validation.py

@@ -266,6 +266,10 @@ def test_validate_jobs():
                 'granules': [granule_with_dem_coverage, granule_without_dem_coverage],
             }
         },
+        {
+            'job_type': 'ARIA_RAIDER',
+            'job_parameters': {}
+        },
     ]
     validation.validate_jobs(jobs)