Merge pull request #62 from ASFHyP3/develop

Release 0.0.10

CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) 
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.0.10]
+### Fixed
+- Replaced deprecated `hyp3_sdk.asf_search.get_nearest_neighbors` function with `find_new.get_neighbors`.
+
 ## [0.0.9]
 ### Security
 - Set `NoEcho` for EDL password in CloudFormation stacks.

README.md

@@ -89,15 +89,15 @@ The HyP3 Event Monitoring source contains test files in `tests/`. To run them yo
 
 - Add components to python path
 ```sh
-export PYTHONPATH="${PYTHONPATH}:`pwd`find_new/src:`pwd`api/src:`pwd`harvest_products/src"
+export PYTHONPATH="${PYTHONPATH}:${PWD}/find_new/src:${PWD}/api/src:${PWD}/harvest_products/src"
 ```
 - Setup environment variables
 ```sh
 export $(cat tests/cfg.env | xargs)
 ```
 - Install test requirements
 ```sh
-pip install -r apps/api/requirements-all.txt
+python -m pip install -r requirements-all.txt
 ```
 
 - Run tests

find_new/requirements.txt

@@ -1,3 +1,4 @@
+asf-search==3.0.6
 hyp3-sdk>=1.3.2
 python-dateutil
 requests

find_new/src/find_new.py

@@ -1,11 +1,13 @@
 from datetime import datetime, timezone
 from os import environ
+from typing import List
 from uuid import uuid4
 
+import asf_search
 import requests
 from dateutil import parser
-from hyp3_sdk import HyP3, asf_search
-from hyp3_sdk.exceptions import ASFSearchError, HyP3Error, ServerError
+from hyp3_sdk import HyP3
+from hyp3_sdk.exceptions import HyP3Error, ServerError
 
 from database import database
 
@@ -72,6 +74,32 @@ def add_invalid_product_record(event_id, granule, message):
     database.put_product(product)
 
 
+def get_neighbors(product_name: str, max_neighbors: int = 2) -> List[dict]:
+    results = asf_search.product_search([product_name])
+    assert len(results) == 1
+    granule: asf_search.ASFProduct = results[0]
+
+    stack = asf_search.baseline_search.stack_from_product(granule)
+    stack = [item for item in stack if item.properties['temporalBaseline'] < 0]
+    neighbors = [item.properties['fileID'] for item in stack[-max_neighbors:]]
+
+    response = requests.post(
+        SEARCH_URL,
+        params={
+            'product_list': ','.join(neighbors),
+            'output': 'jsonlite'
+        }
+    )
+
+    status_code = str(response.status_code)
+    if status_code[0] == '4':
+        raise asf_search.ASFSearch4xxError()
+    elif status_code[0] == '5':
+        raise asf_search.ASFSearch5xxError()
+
+    return response.json()['results']
+
+
 def submit_jobs_for_granule(hyp3, event_id, granule):
     print(f'submitting jobs for granule {granule["granuleName"]}')
 
@@ -82,10 +110,10 @@ def submit_jobs_for_granule(hyp3, event_id, granule):
     granule_lists.append([granule])
 
     try:
-        neighbors = asf_search.get_nearest_neighbors(granule['granuleName'])
-    except ASFSearchError:
+        neighbors = get_neighbors(granule['productID'])
+    except asf_search.ASFSearch4xxError:
         raise GranuleError()
-    except ServerError as e:
+    except asf_search.ASFSearchError as e:
         print(e)
         print(f'Server error finding neighbors for {granule["granuleName"]}, skipping...')
         return

tests/test_find_new.py

@@ -3,11 +3,12 @@
 from unittest.mock import patch
 from uuid import uuid4
 
+import asf_search
 import pytest
 import responses
 from dateutil import parser
 from hyp3_sdk import HyP3
-from hyp3_sdk.exceptions import ASFSearchError, HyP3Error, ServerError
+from hyp3_sdk.exceptions import HyP3Error, ServerError
 from hyp3_sdk.util import AUTH_URL
 
 import find_new
@@ -252,6 +253,7 @@ def test_submit_jobs_for_granule(tables):
 
     granule = {
         'granuleName': 'reference',
+        'productID': 'reference-SLC',
         'startTime': '2020-01-01T00:00:00+00:00',
         'path': 123,
         'frame': 456,
@@ -260,7 +262,7 @@ def test_submit_jobs_for_granule(tables):
     event_id = 'event_id1'
 
     hyp3 = HyP3(environ['HYP3_URL'], username=environ['EDL_USERNAME'], password=environ['EDL_PASSWORD'])
-    with patch('hyp3_sdk.asf_search.get_nearest_neighbors', lambda x: mock_neighbors):
+    with patch('find_new.get_neighbors', lambda x: mock_neighbors):
         find_new.submit_jobs_for_granule(hyp3, event_id, granule)
 
     products = tables.product_table.scan()['Items']
@@ -283,6 +285,7 @@ def test_submit_jobs_for_granule(tables):
 def test_submit_jobs_for_granule_submit_error(tables):
     granule = {
         'granuleName': 'reference',
+        'productID': 'reference-SLC',
         'startTime': '2020-01-01T00:00:00+00:00',
         'path': 123,
         'frame': 456,
@@ -293,12 +296,12 @@ def test_submit_jobs_for_granule_submit_error(tables):
     responses.add(responses.GET, AUTH_URL)
     hyp3 = HyP3(environ['HYP3_URL'], username=environ['EDL_USERNAME'], password=environ['EDL_PASSWORD'])
 
-    with patch('hyp3_sdk.asf_search.get_nearest_neighbors', lambda x: []):
+    with patch('find_new.get_neighbors', lambda x: []):
         with patch('hyp3_sdk.HyP3.submit_prepared_jobs', side_effect=HyP3Error):
             with pytest.raises(find_new.GranuleError):
                 find_new.submit_jobs_for_granule(hyp3, event_id, granule)
 
-    with patch('hyp3_sdk.asf_search.get_nearest_neighbors', lambda x: []):
+    with patch('find_new.get_neighbors', lambda x: []):
         with patch('hyp3_sdk.HyP3.submit_prepared_jobs', side_effect=ServerError):
             find_new.submit_jobs_for_granule(hyp3, event_id, granule)
     assert tables.product_table.scan()['Items'] == []
@@ -308,6 +311,7 @@ def test_submit_jobs_for_granule_submit_error(tables):
 def test_submit_jobs_for_granule_neighbor_error(tables):
     granule = {
         'granuleName': 'reference',
+        'productID': 'reference-SLC',
         'startTime': '2020-01-01T00:00:00+00:00',
         'path': 123,
         'frame': 456,
@@ -318,11 +322,11 @@ def test_submit_jobs_for_granule_neighbor_error(tables):
     responses.add(responses.GET, AUTH_URL)
     hyp3 = HyP3(environ['HYP3_URL'], username=environ['EDL_USERNAME'], password=environ['EDL_PASSWORD'])
 
-    with patch('hyp3_sdk.asf_search.get_nearest_neighbors', side_effect=ASFSearchError):
+    with patch('find_new.get_neighbors', side_effect=asf_search.ASFSearch4xxError):
         with pytest.raises(find_new.GranuleError):
             find_new.submit_jobs_for_granule(hyp3, event_id, granule)
 
-    with patch('hyp3_sdk.asf_search.get_nearest_neighbors', side_effect=ServerError):
+    with patch('find_new.get_neighbors', side_effect=asf_search.ASFSearchError):
         find_new.submit_jobs_for_granule(hyp3, event_id, granule)
     assert tables.product_table.scan()['Items'] == []
 
@@ -376,20 +380,23 @@ def test_lambda_handler(tables):
         'results': [
             {
                 'granuleName': 'granule1',
+                'productID': 'granule1-SLC',
                 'startTime': '2020-01-01T00:00:00+00:00',
                 'path': 123,
                 'frame': 456,
                 'wkt': 'someWKT',
             },
             {
                 'granuleName': 'granule2',
+                'productID': 'granule2-SLC',
                 'startTime': '2020-01-01T00:00:00+00:00',
                 'path': 456,
                 'frame': 789,
                 'wkt': 'someWKT',
             },
             {
                 'granuleName': 'granule3',
+                'productID': 'granule3-SLC',
                 'startTime': '2020-01-01T00:00:00+00:00',
                 'path': 123,
                 'frame': 456,
@@ -444,7 +451,7 @@ def test_lambda_handler(tables):
     }
     responses.add(responses.POST, environ['HYP3_URL'] + '/jobs', json.dumps(mock_hyp3_response))
 
-    with patch('hyp3_sdk.asf_search.get_nearest_neighbors', lambda x: mock_neighbors):
+    with patch('find_new.get_neighbors', lambda x: mock_neighbors):
         find_new.lambda_handler(None, None)
 
     products = tables.product_table.scan()['Items']