diff --git a/CHANGELOG.md b/CHANGELOG.md index e3d2351..b79c0e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.0.10] +### Fixed +- Replaced deprecated `hyp3_sdk.asf_search.get_nearest_neighbors` function with `find_new.get_neighbors`. + ## [0.0.9] ### Security - Set `NoEcho` for EDL password in CloudFormation stacks. diff --git a/README.md b/README.md index 0ffbbe1..6e0e0f5 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ The HyP3 Event Monitoring source contains test files in `tests/`. To run them yo - Add components to python path ```sh -export PYTHONPATH="${PYTHONPATH}:`pwd`find_new/src:`pwd`api/src:`pwd`harvest_products/src" +export PYTHONPATH="${PYTHONPATH}:${PWD}/find_new/src:${PWD}/api/src:${PWD}/harvest_products/src" ``` - Setup environment variables ```sh @@ -97,7 +97,7 @@ export $(cat tests/cfg.env | xargs) ``` - Install test requirements ```sh -pip install -r apps/api/requirements-all.txt +python -m pip install -r requirements-all.txt ``` - Run tests diff --git a/find_new/requirements.txt b/find_new/requirements.txt index 2867f60..c7aee93 100644 --- a/find_new/requirements.txt +++ b/find_new/requirements.txt @@ -1,3 +1,4 @@ +asf-search==3.0.6 hyp3-sdk>=1.3.2 python-dateutil requests diff --git a/find_new/src/find_new.py b/find_new/src/find_new.py index 7d0c451..c54eca4 100644 --- a/find_new/src/find_new.py +++ b/find_new/src/find_new.py @@ -1,11 +1,13 @@ from datetime import datetime, timezone from os import environ +from typing import List from uuid import uuid4 +import asf_search import requests from dateutil import parser -from hyp3_sdk import HyP3, asf_search -from hyp3_sdk.exceptions import ASFSearchError, HyP3Error, ServerError +from hyp3_sdk import HyP3 +from hyp3_sdk.exceptions import HyP3Error, ServerError from database import database @@ -72,6 +74,32 @@ def add_invalid_product_record(event_id, granule, message): database.put_product(product) +def get_neighbors(product_name: str, max_neighbors: int = 2) -> List[dict]: + results = asf_search.product_search([product_name]) + assert len(results) == 1 + granule: asf_search.ASFProduct = results[0] + + stack = asf_search.baseline_search.stack_from_product(granule) + stack = [item for item in stack if item.properties['temporalBaseline'] < 0] + neighbors = [item.properties['fileID'] for item in stack[-max_neighbors:]] + + response = requests.post( + SEARCH_URL, + params={ + 'product_list': ','.join(neighbors), + 'output': 'jsonlite' + } + ) + + status_code = str(response.status_code) + if status_code[0] == '4': + raise asf_search.ASFSearch4xxError() + elif status_code[0] == '5': + raise asf_search.ASFSearch5xxError() + + return response.json()['results'] + + def submit_jobs_for_granule(hyp3, event_id, granule): print(f'submitting jobs for granule {granule["granuleName"]}') @@ -82,10 +110,10 @@ def submit_jobs_for_granule(hyp3, event_id, granule): granule_lists.append([granule]) try: - neighbors = asf_search.get_nearest_neighbors(granule['granuleName']) - except ASFSearchError: + neighbors = get_neighbors(granule['productID']) + except asf_search.ASFSearch4xxError: raise GranuleError() - except ServerError as e: + except asf_search.ASFSearchError as e: print(e) print(f'Server error finding neighbors for {granule["granuleName"]}, skipping...') return diff --git a/tests/test_find_new.py b/tests/test_find_new.py index f2a2bbd..6422edb 100644 --- a/tests/test_find_new.py +++ b/tests/test_find_new.py @@ -3,11 +3,12 @@ from unittest.mock import patch from uuid import uuid4 +import asf_search import pytest import responses from dateutil import parser from hyp3_sdk import HyP3 -from hyp3_sdk.exceptions import ASFSearchError, HyP3Error, ServerError +from hyp3_sdk.exceptions import HyP3Error, ServerError from hyp3_sdk.util import AUTH_URL import find_new @@ -252,6 +253,7 @@ def test_submit_jobs_for_granule(tables): granule = { 'granuleName': 'reference', + 'productID': 'reference-SLC', 'startTime': '2020-01-01T00:00:00+00:00', 'path': 123, 'frame': 456, @@ -260,7 +262,7 @@ def test_submit_jobs_for_granule(tables): event_id = 'event_id1' hyp3 = HyP3(environ['HYP3_URL'], username=environ['EDL_USERNAME'], password=environ['EDL_PASSWORD']) - with patch('hyp3_sdk.asf_search.get_nearest_neighbors', lambda x: mock_neighbors): + with patch('find_new.get_neighbors', lambda x: mock_neighbors): find_new.submit_jobs_for_granule(hyp3, event_id, granule) products = tables.product_table.scan()['Items'] @@ -283,6 +285,7 @@ def test_submit_jobs_for_granule(tables): def test_submit_jobs_for_granule_submit_error(tables): granule = { 'granuleName': 'reference', + 'productID': 'reference-SLC', 'startTime': '2020-01-01T00:00:00+00:00', 'path': 123, 'frame': 456, @@ -293,12 +296,12 @@ def test_submit_jobs_for_granule_submit_error(tables): responses.add(responses.GET, AUTH_URL) hyp3 = HyP3(environ['HYP3_URL'], username=environ['EDL_USERNAME'], password=environ['EDL_PASSWORD']) - with patch('hyp3_sdk.asf_search.get_nearest_neighbors', lambda x: []): + with patch('find_new.get_neighbors', lambda x: []): with patch('hyp3_sdk.HyP3.submit_prepared_jobs', side_effect=HyP3Error): with pytest.raises(find_new.GranuleError): find_new.submit_jobs_for_granule(hyp3, event_id, granule) - with patch('hyp3_sdk.asf_search.get_nearest_neighbors', lambda x: []): + with patch('find_new.get_neighbors', lambda x: []): with patch('hyp3_sdk.HyP3.submit_prepared_jobs', side_effect=ServerError): find_new.submit_jobs_for_granule(hyp3, event_id, granule) assert tables.product_table.scan()['Items'] == [] @@ -308,6 +311,7 @@ def test_submit_jobs_for_granule_submit_error(tables): def test_submit_jobs_for_granule_neighbor_error(tables): granule = { 'granuleName': 'reference', + 'productID': 'reference-SLC', 'startTime': '2020-01-01T00:00:00+00:00', 'path': 123, 'frame': 456, @@ -318,11 +322,11 @@ def test_submit_jobs_for_granule_neighbor_error(tables): responses.add(responses.GET, AUTH_URL) hyp3 = HyP3(environ['HYP3_URL'], username=environ['EDL_USERNAME'], password=environ['EDL_PASSWORD']) - with patch('hyp3_sdk.asf_search.get_nearest_neighbors', side_effect=ASFSearchError): + with patch('find_new.get_neighbors', side_effect=asf_search.ASFSearch4xxError): with pytest.raises(find_new.GranuleError): find_new.submit_jobs_for_granule(hyp3, event_id, granule) - with patch('hyp3_sdk.asf_search.get_nearest_neighbors', side_effect=ServerError): + with patch('find_new.get_neighbors', side_effect=asf_search.ASFSearchError): find_new.submit_jobs_for_granule(hyp3, event_id, granule) assert tables.product_table.scan()['Items'] == [] @@ -376,6 +380,7 @@ def test_lambda_handler(tables): 'results': [ { 'granuleName': 'granule1', + 'productID': 'granule1-SLC', 'startTime': '2020-01-01T00:00:00+00:00', 'path': 123, 'frame': 456, @@ -383,6 +388,7 @@ def test_lambda_handler(tables): }, { 'granuleName': 'granule2', + 'productID': 'granule2-SLC', 'startTime': '2020-01-01T00:00:00+00:00', 'path': 456, 'frame': 789, @@ -390,6 +396,7 @@ def test_lambda_handler(tables): }, { 'granuleName': 'granule3', + 'productID': 'granule3-SLC', 'startTime': '2020-01-01T00:00:00+00:00', 'path': 123, 'frame': 456, @@ -444,7 +451,7 @@ def test_lambda_handler(tables): } responses.add(responses.POST, environ['HYP3_URL'] + '/jobs', json.dumps(mock_hyp3_response)) - with patch('hyp3_sdk.asf_search.get_nearest_neighbors', lambda x: mock_neighbors): + with patch('find_new.get_neighbors', lambda x: mock_neighbors): find_new.lambda_handler(None, None) products = tables.product_table.scan()['Items']