Merge pull request #172 from ASFHyP3/develop

Release v0.1.3

.github/workflows/changelog.yml

@@ -13,4 +13,4 @@ on:
 
 jobs:
   call-changelog-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.12.0

.github/workflows/create-jira-issue.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   call-create-jira-issue-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.12.0
     secrets:
       JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
       JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}

.github/workflows/deploy.yml

@@ -74,6 +74,6 @@ jobs:
   call-bump-version-workflow:
     if: github.ref == 'refs/heads/main'
     needs: deploy
-    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.12.0
     secrets:
       USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}

.github/workflows/labeled-pr.yml

@@ -12,4 +12,4 @@ on:
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.12.0

.github/workflows/release-template-comment.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-checklist-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.12.0
     permissions:
       pull-requests: write
     secrets:

.github/workflows/release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.2
+    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.12.0
     with:
       release_prefix: HyP3 Event Monitoring
     secrets:

.github/workflows/static-analysis.yml

@@ -3,10 +3,9 @@ name: Static code analysis
 on: push
 
 jobs:
-  call-flake8-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected]
-    with:
-      local_package_names: api,database,find_new,harvest_products
+  call-ruff-workflow:
+    # Docs: https://github.com/ASFHyP3/actions
+    uses: ASFHyP3/actions/.github/workflows/[email protected]
 
   cfn-lint:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) 
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.3]
+### Changed
+- The [`static-analysis`](.github/workflows/static-analysis.yml) Github Actions workflow now uses `ruff` rather than `flake8` for linting.
+
 ## [0.1.2]
 ### Fixed
 - Upgraded to flask-cors v5.0.0 from v4.0.1. Resolves [CVE-2024-6221](https://www.cve.org/CVERecord?id=CVE-2024-6221).

database/database/database.py

@@ -5,6 +5,7 @@
 import boto3
 from boto3.dynamodb.conditions import Attr, Key
 
+
 DB = boto3.resource('dynamodb')
 EVENT_TABLE = environ.get('EVENT_TABLE', None)
 PRODUCT_TABLE = environ.get('PRODUCT_TABLE', None)
@@ -24,10 +25,7 @@ def query_table(table_name, key_expression, filter_expression=None, index_name=N
     items = response['Items']
 
     while 'LastEvaluatedKey' in response:
-        response = table.query(
-            ExclusiveStartKey=response['LastEvaluatedKey'],
-            **query_params
-        )
+        response = table.query(ExclusiveStartKey=response['LastEvaluatedKey'], **query_params)
         items.extend(response['Items'])
     return items
 

database/setup.py

@@ -1,12 +1,10 @@
 from setuptools import find_packages, setup
 
+
 setup(
     name='database',
-
     license='BSD',
     include_package_data=True,
-
     python_requires='~=3.12',
-
     packages=find_packages(),
 )

find_new/src/find_new.py

@@ -11,6 +11,7 @@
 
 from database import database
 
+
 SEARCH_URL = 'https://api.daac.asf.alaska.edu/services/search/param'
 
 
@@ -76,7 +77,7 @@ def add_invalid_product_record(event_id, granule, message):
 
 def get_neighbors(product_name: str, max_neighbors: int = 2) -> List[dict]:
     if max_neighbors < 1:
-        raise ValueError(f"max_neighbors must be >= 1 but got {max_neighbors}")
+        raise ValueError(f'max_neighbors must be >= 1 but got {max_neighbors}')
 
     results = asf_search.product_search([product_name])
     assert len(results) == 1
@@ -89,13 +90,7 @@ def get_neighbors(product_name: str, max_neighbors: int = 2) -> List[dict]:
     if len(neighbor_names) == 0:
         neighbors = []
     else:
-        response = requests.post(
-            SEARCH_URL,
-            params={
-                'product_list': ','.join(neighbor_names),
-                'output': 'jsonlite'
-            }
-        )
+        response = requests.post(SEARCH_URL, params={'product_list': ','.join(neighbor_names), 'output': 'jsonlite'})
 
         status_code = str(response.status_code)
         if status_code[0] == '4':

harvest_products/src/harvest_products.py

@@ -10,6 +10,7 @@
 
 from database import database
 
+
 S3 = boto3.resource('s3')
 
 

pyproject.toml

@@ -0,0 +1,34 @@
+[project]
+name = "hyp3_event_monitoring"
+requires-python = ">=3.10"
+
+[tool.ruff]
+line-length = 120
+# The directories to consider when resolving first- vs. third-party imports.
+# See: https://docs.astral.sh/ruff/settings/#src
+src = [
+    "**/src",
+    "database",
+    "tests",
+]
+
+[tool.ruff.format]
+indent-style = "space"
+quote-style = "single"
+
+[tool.ruff.lint]
+extend-select = [
+    "I",   # isort: https://docs.astral.sh/ruff/rules/#isort-i
+    # TODO: Uncomment the following extensions and address their warnings:
+    # "UP",  # pyupgrade: https://docs.astral.sh/ruff/rules/#pyupgrade-up
+    # "D",   # pydocstyle: https://docs.astral.sh/ruff/rules/#pydocstyle-d
+    # "ANN", # annotations: https://docs.astral.sh/ruff/rules/#flake8-annotations-ann
+    # "PTH", # use-pathlib-pth: https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.isort]
+case-sensitive = true
+lines-after-imports = 2

requirements-all.txt

@@ -1,11 +1,8 @@
-boto3==1.35.10
-cfn-lint==1.12.1
-flake8==7.1.1
-flake8-blind-except==0.2.1
-flake8-builtins==2.5.0
-flake8-import-order==0.18.2
-pytest==8.3.2
-moto==5.0.13
+boto3==1.35.76
+cfn-lint==1.22.0
+ruff
+pytest==8.3.4
+moto==5.0.22
 responses==0.25.3
 pyYAML==6.0.2
 -r requirements-find-new.txt

requirements-api.txt

@@ -1,4 +1,4 @@
-flask==3.0.3
+flask==3.1.0
 Flask-Cors==5.0.0
-serverless_wsgi==3.0.4
+serverless_wsgi==3.0.5
 ./database

tests/conftest.py

@@ -32,8 +32,7 @@ class Tables:
                 **get_table_properties_from_template('EventTable'),
             )
             product_table = database.DB.create_table(
-                TableName=environ['PRODUCT_TABLE'],
-                **get_table_properties_from_template('ProductTable')
+                TableName=environ['PRODUCT_TABLE'], **get_table_properties_from_template('ProductTable')
             )
 
         tables = Tables()

tests/test_api.py

@@ -6,10 +6,7 @@
 
 def seed_data(tables):
     events = [
-        {
-            'event_id': 'event1',
-            'decimal_value': Decimal(1.0)
-        },
+        {'event_id': 'event1', 'decimal_value': Decimal(1.0)},
         {
             'event_id': 'event2',
         },

tests/test_find_new.py

@@ -107,7 +107,7 @@ def test_get_unprocessed_granules(tables):
                     'frame': 456,
                     'wkt': 'someWKT',
                 }
-            ]
+            ],
         },
         {
             'product_id': str(uuid4()),
@@ -121,7 +121,7 @@ def test_get_unprocessed_granules(tables):
                     'frame': 789,
                     'wkt': 'someWKT',
                 },
-            ]
+            ],
         },
     ]
     for item in mock_products:
@@ -225,15 +225,15 @@ def test_get_neighbors(mock_product_search: MagicMock, mock_stack_from_product:
     responses.post(
         url=find_new.SEARCH_URL,
         body=json.dumps(mock_response_1),
-        match=[responses.matchers.query_param_matcher(params_1)]
+        match=[responses.matchers.query_param_matcher(params_1)],
     )
 
     mock_response_2 = {'results': [{'granuleName': 'granule1'}, {'granuleName': 'granule2'}]}
     params_2 = {'product_list': 'file-1,file-2', 'output': 'jsonlite'}
     responses.post(
         url=find_new.SEARCH_URL,
         body=json.dumps(mock_response_2),
-        match=[responses.matchers.query_param_matcher(params_2)]
+        match=[responses.matchers.query_param_matcher(params_2)],
     )
 
     mock_response_3 = {
@@ -243,7 +243,7 @@ def test_get_neighbors(mock_product_search: MagicMock, mock_stack_from_product:
     responses.post(
         url=find_new.SEARCH_URL,
         body=json.dumps(mock_response_3),
-        match=[responses.matchers.query_param_matcher(params_3)]
+        match=[responses.matchers.query_param_matcher(params_3)],
     )
 
     assert find_new.get_neighbors('test-product', max_neighbors=1) == mock_response_1['results']
@@ -267,11 +267,7 @@ def test_get_neighbors_response_400(mock_product_search: MagicMock, mock_stack_f
     ]
 
     params = {'product_list': 'file1', 'output': 'jsonlite'}
-    responses.post(
-        url=find_new.SEARCH_URL,
-        status=400,
-        match=[responses.matchers.query_param_matcher(params)]
-    )
+    responses.post(url=find_new.SEARCH_URL, status=400, match=[responses.matchers.query_param_matcher(params)])
 
     with pytest.raises(asf_search.ASFSearch4xxError):
         find_new.get_neighbors('test-product')
@@ -292,11 +288,7 @@ def test_get_neighbors_response_500(mock_product_search: MagicMock, mock_stack_f
     ]
 
     params = {'product_list': 'file1', 'output': 'jsonlite'}
-    responses.post(
-        url=find_new.SEARCH_URL,
-        status=500,
-        match=[responses.matchers.query_param_matcher(params)]
-    )
+    responses.post(url=find_new.SEARCH_URL, status=500, match=[responses.matchers.query_param_matcher(params)])
 
     with pytest.raises(asf_search.ASFSearch5xxError):
         find_new.get_neighbors('test-product')

tests/test_harvest_products.py

@@ -12,12 +12,7 @@
 @responses.activate
 def test_harvest_file(s3_stubber):
     responses.add(responses.GET, 'https://foo.com/file.png', body='image_content')
-    params = {
-        'Bucket': environ['BUCKET_NAME'],
-        'Key': 'prefix/file.png',
-        'ContentType': 'image/png',
-        'Body': ANY
-    }
+    params = {'Bucket': environ['BUCKET_NAME'], 'Key': 'prefix/file.png', 'ContentType': 'image/png', 'Body': ANY}
     s3_stubber.add_response(method='put_object', expected_params=params, service_response={})
     response = harvest_products.harvest_file('https://foo.com/file.png', 'prefix')
 
@@ -31,7 +26,7 @@ def test_harvest(mock_harvest_file: MagicMock):
         'product_id': 'product_id',
         'granules': [],
         'status_code': 'PENDING',
-        'processing_date': '2020-01-01T00:00:00+00:00'
+        'processing_date': '2020-01-01T00:00:00+00:00',
     }
 
     class MockJob:
@@ -57,7 +52,7 @@ class MockJob:
         'thumbnail_url': 'https://foo.com/file.png',
         'product_name': 'product.zip',
         'product_size': 123,
-        'product_url': 'https://foo.com/file.png'
+        'product_url': 'https://foo.com/file.png',
     }
 
     assert mock_harvest_file.mock_calls == [
@@ -73,7 +68,7 @@ def test_update_product_succeeded(tables):
         'product_id': 'foo',
         'granules': [],
         'status_code': 'PENDING',
-        'processing_date': '2020-01-01T00:00:00+00:00'
+        'processing_date': '2020-01-01T00:00:00+00:00',
     }
 
     job = Job(
@@ -99,7 +94,7 @@ def test_update_product_succeeded(tables):
         'thumbnail_url': 'THUMBNAIL_IMAGE_URL',
         'product_name': 'product.zip',
         'product_size': 123,
-        'product_url': f'https://{environ["BUCKET_NAME"]}.s3.amazonaws.com/1/foo/product.zip'
+        'product_url': f'https://{environ["BUCKET_NAME"]}.s3.amazonaws.com/1/foo/product.zip',
     }
 
     with patch('harvest_products.harvest', lambda x, y: mock_harvest):
@@ -117,7 +112,7 @@ def test_update_product_failed(tables):
         'product_id': 'foo',
         'granules': [],
         'status_code': 'PENDING',
-        'processing_date': '2020-01-01T00:00:00+00:00'
+        'processing_date': '2020-01-01T00:00:00+00:00',
     }
 
     job = Job(