diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d15f0cdd..eb9eeb7cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [6.3.0] + +### Changed +- `/costs` API endpoint now returns a list of job cost dictionaries, instead of a dictionary of dictionaries. +- Cost table parameters are now contained within the `parameter_value` dictionary key. +- Cost table costs are now contained within the `cost` dictionary key. + ## [6.2.0] HyP3 is in the process of transitioning from a monthly job quota to a credits system. [HyP3 v6.0.0](https://github.com/ASFHyP3/hyp3/releases/tag/v6.0.0) implemented the new credits system without changing the number of jobs that users can run per month. This release implements the capability to assign a different credit cost to each type of job, again without actually changing the number of jobs that users can run per month. diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 index e4f257f69..309b2d699 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 @@ -25,7 +25,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/costs" + $ref: "#/components/schemas/costs_response" /jobs: @@ -117,9 +117,34 @@ paths: components: schemas: - costs: - description: Table of job costs. - type: object + costs_response: + description: List of job costs. + type: array + items: + type: object + required: + - job_type + properties: + job_type: + $ref: "./job_parameters.yml#/components/schemas/job_type" + cost: + $ref: "#/components/schemas/credits" + cost_parameter: + type: string + cost_table: + type: array + items: + type: object + required: + - parameter_value + - cost + properties: + parameter_value: + oneOf: + - type: string + - type: number + cost: + $ref: "#/components/schemas/credits" post_jobs_body: description: List for new jobs to submit for processing. diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index a41fc7766..5306123ad 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -11,7 +11,6 @@ from hyp3_api.util import get_granules DEM_COVERAGE = None -DEM_COVERAGE_LEGACY = None class GranuleValidationError(Exception): diff --git a/apps/render_cf.py b/apps/render_cf.py index f767f0d71..1907d065a 100644 --- a/apps/render_cf.py +++ b/apps/render_cf.py @@ -48,12 +48,15 @@ def render_default_params_by_job_type(job_types: dict) -> None: def render_costs(job_types: dict, cost_profile: str) -> None: - costs = { - job_type: job_spec['cost_profiles'][cost_profile] + costs = [ + { + 'job_type': job_type, + **job_spec['cost_profiles'][cost_profile], + } for job_type, job_spec in job_types.items() - } - with open(Path('lib') / 'dynamo' / 'dynamo' / 'costs.yml', 'w') as f: - yaml.safe_dump(costs, f) + ] + with open(Path('lib') / 'dynamo' / 'dynamo' / 'costs.json', 'w') as f: + json.dump(costs, f, indent=2) def main(): diff --git a/job_spec/INSAR_GAMMA.yml b/job_spec/INSAR_GAMMA.yml index 6b662b4d6..7f447c1c6 100644 --- a/job_spec/INSAR_GAMMA.yml +++ b/job_spec/INSAR_GAMMA.yml @@ -74,8 +74,10 @@ INSAR_GAMMA: EDC: cost_parameter: looks cost_table: - 20x4: 10.0 - 10x2: 15.0 + - parameter_value: 20x4 + cost: 10.0 + - parameter_value: 10x2 + cost: 15.0 DEFAULT: cost: 1.0 validators: diff --git a/job_spec/RTC_GAMMA.yml b/job_spec/RTC_GAMMA.yml index 27a77e176..3b29ec668 100644 --- a/job_spec/RTC_GAMMA.yml +++ b/job_spec/RTC_GAMMA.yml @@ -93,9 +93,12 @@ RTC_GAMMA: EDC: cost_parameter: resolution cost_table: - 30.0: 5.0 - 20.0: 15.0 - 10.0: 60.0 + - parameter_value: 30.0 + cost: 5.0 + - parameter_value: 20.0 + cost: 15.0 + - parameter_value: 10.0 + cost: 60.0 DEFAULT: cost: 1.0 validators: diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py index 9187369f1..5048df66f 100644 --- a/lib/dynamo/dynamo/jobs.py +++ b/lib/dynamo/dynamo/jobs.py @@ -6,14 +6,13 @@ from typing import List, Optional from uuid import uuid4 -import yaml from boto3.dynamodb.conditions import Attr, Key import dynamo.user from dynamo.util import DYNAMODB_RESOURCE, convert_floats_to_decimals, format_time, get_request_time_expression -costs_file = Path(__file__).parent / 'costs.yml' -COSTS = convert_floats_to_decimals(yaml.safe_load(costs_file.read_text())) +costs_file = Path(__file__).parent / 'costs.json' +COSTS = convert_floats_to_decimals(json.loads(costs_file.read_text())) default_params_file = Path(__file__).parent / 'default_params_by_job_type.json' if default_params_file.exists(): @@ -99,18 +98,24 @@ def _prepare_job_for_database( return prepared_job -def _get_credit_cost(job: dict, costs: dict) -> Decimal: +def _get_credit_cost(job: dict, costs: list[dict]) -> Decimal: job_type = job['job_type'] - cost_definition = costs[job_type] - - if cost_definition.keys() not in ({'cost_parameter', 'cost_table'}, {'cost'}): - raise ValueError(f'Cost definition for job type {job_type} has invalid keys: {cost_definition.keys()}') - - if 'cost_parameter' in cost_definition: - parameter_value = job['job_parameters'][cost_definition['cost_parameter']] - return cost_definition['cost_table'][parameter_value] - - return cost_definition['cost'] + for cost_definition in costs: + if cost_definition['job_type'] == job_type: + + if cost_definition.keys() not in ({'job_type', 'cost_parameter', 'cost_table'}, {'job_type', 'cost'}): + raise ValueError(f'Cost definition for job type {job_type} has invalid keys: {cost_definition.keys()}') + + if 'cost_parameter' in cost_definition: + cost_parameter = cost_definition['cost_parameter'] + parameter_value = job['job_parameters'][cost_parameter] + for item in cost_definition['cost_table']: + if item['parameter_value'] == parameter_value: + return item['cost'] + raise ValueError(f'Cost not found for job type {job_type} with {cost_parameter} == {parameter_value}') + + return cost_definition['cost'] + raise ValueError(f'Cost not found for job type {job_type}') def query_jobs(user, start=None, end=None, status_code=None, name=None, job_type=None, start_key=None): diff --git a/lib/dynamo/setup.py b/lib/dynamo/setup.py index 9cf00f174..62d7e31c0 100644 --- a/lib/dynamo/setup.py +++ b/lib/dynamo/setup.py @@ -8,11 +8,10 @@ install_requires=[ 'boto3', 'python-dateutil', - 'pyyaml', ], python_requires='~=3.9', packages=find_packages(), - package_data={'dynamo': ['*.json', '*.yml']}, + package_data={'dynamo': ['*.json']}, ) diff --git a/requirements-all.txt b/requirements-all.txt index 512ba6e70..720b62615 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -5,10 +5,10 @@ -r requirements-apps-start-execution-worker.txt -r requirements-apps-disable-private-dns.txt -r requirements-apps-update-db.txt -boto3==1.34.48 +boto3==1.34.60 jinja2==3.1.3 -moto[dynamodb]==5.0.3.dev5 -pytest==8.0.1 +moto[dynamodb]==5.0.3.dev33 +pytest==8.1.1 PyYAML==6.0.1 responses==0.25.0 flake8==7.0.0 @@ -17,4 +17,4 @@ flake8-blind-except==0.2.1 flake8-builtins==2.2.0 setuptools==69.1.1 openapi-spec-validator==0.7.1 -cfn-lint==0.85.2 +cfn-lint==0.86.0 diff --git a/requirements-apps-api-binary.txt b/requirements-apps-api-binary.txt index e772b712f..3beaa30e8 100644 --- a/requirements-apps-api-binary.txt +++ b/requirements-apps-api-binary.txt @@ -1 +1 @@ -cryptography==42.0.4 +cryptography==42.0.5 diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt index 7fa5f1b16..79406d53d 100644 --- a/requirements-apps-disable-private-dns.txt +++ b/requirements-apps-disable-private-dns.txt @@ -1 +1 @@ -boto3==1.34.48 +boto3==1.34.60 diff --git a/requirements-apps-start-execution-manager.txt b/requirements-apps-start-execution-manager.txt index 7589f65aa..3e16dda09 100644 --- a/requirements-apps-start-execution-manager.txt +++ b/requirements-apps-start-execution-manager.txt @@ -1,3 +1,3 @@ -boto3==1.34.48 +boto3==1.34.60 ./lib/dynamo/ ./lib/lambda_logging/ diff --git a/requirements-apps-start-execution-worker.txt b/requirements-apps-start-execution-worker.txt index 8a62963c6..9dd48947d 100644 --- a/requirements-apps-start-execution-worker.txt +++ b/requirements-apps-start-execution-worker.txt @@ -1,2 +1,2 @@ -boto3==1.34.48 +boto3==1.34.60 ./lib/lambda_logging/ diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 8dfb90df3..d3fd061d0 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -51,14 +51,6 @@ def test_has_sufficient_coverage(): poly = rectangle(40.1, 40, -126, -125.000140) assert not validation.has_sufficient_coverage(poly) - # minimum sufficient legacy coverage off the coast of Eureka, CA - poly = rectangle(40.1, 40, -126, -124.845) - assert validation.has_sufficient_coverage(poly) - - # almost minimum sufficient legacy coverage off the coast of Eureka, CA - poly = rectangle(40.1, 40, -126, -124.849) - assert validation.has_sufficient_coverage(poly) - # polygon in missing tile over Gulf of California poly = rectangle(26.9, 26.1, -110.1, -110.9) assert not validation.has_sufficient_coverage(poly) diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py index 46cf68334..68b715522 100644 --- a/tests/test_dynamo/test_jobs.py +++ b/tests/test_dynamo/test_jobs.py @@ -183,19 +183,30 @@ def test_query_jobs_by_type(tables): def test_get_credit_cost(): - costs = { - 'RTC_GAMMA': { + costs = [ + { + 'job_type': 'RTC_GAMMA', 'cost_parameter': 'resolution', - 'cost_table': { - 10: 60.0, - 20: 15.0, - 30: 5.0, - }, - }, - 'INSAR_ISCE_BURST': { + 'cost_table': [ + { + 'parameter_value': 10.0, + 'cost': 60.0, + }, + { + 'parameter_value': 20.0, + 'cost': 15.0, + }, + { + 'parameter_value': 30.0, + 'cost': 5.0, + }, + ], + }, + { + 'job_type': 'INSAR_ISCE_BURST', 'cost': 1.0, } - } + ] assert dynamo.jobs._get_credit_cost( {'job_type': 'RTC_GAMMA', 'job_parameters': {'resolution': 10.0}}, costs @@ -208,7 +219,7 @@ def test_get_credit_cost(): {'job_type': 'RTC_GAMMA', 'job_parameters': {'resolution': 30.0}}, costs ) == 5.0 - with pytest.raises(KeyError): + with pytest.raises(ValueError): dynamo.jobs._get_credit_cost( {'job_type': 'RTC_GAMMA', 'job_parameters': {'resolution': 13.0}}, costs @@ -224,14 +235,14 @@ def test_get_credit_cost(): def test_get_credit_cost_validate_keys(): - costs = { - 'JOB_TYPE_A': {'cost_parameter': 'foo', 'cost_table': {'bar': 3.0}}, - 'JOB_TYPE_B': {'cost': 5.0}, - 'JOB_TYPE_C': {'cost_parameter': ''}, - 'JOB_TYPE_D': {'cost_table': {}}, - 'JOB_TYPE_E': {'cost_parameter': '', 'cost_table': {}, 'cost': 1.0}, - 'JOB_TYPE_F': {'cost_parameter': '', 'cost_table': {}, 'foo': None}, - } + costs = [ + {'job_type': 'JOB_TYPE_A', 'cost_parameter': 'foo', 'cost_table': [{'parameter_value': 'bar', 'cost': 3.0}]}, + {'job_type': 'JOB_TYPE_B', 'cost': 5.0}, + {'job_type': 'JOB_TYPE_C', 'cost_parameter': ''}, + {'job_type': 'JOB_TYPE_D', 'cost_table': {}}, + {'job_type': 'JOB_TYPE_E', 'cost_parameter': '', 'cost_table': [], 'cost': 1.0}, + {'job_type': 'JOB_TYPE_F', 'cost_parameter': '', 'cost_table': [], 'foo': None}, + ] assert dynamo.jobs._get_credit_cost({'job_type': 'JOB_TYPE_A', 'job_parameters': {'foo': 'bar'}}, costs) == 3.0 assert dynamo.jobs._get_credit_cost({'job_type': 'JOB_TYPE_B'}, costs) == 5.0 @@ -281,11 +292,11 @@ def test_put_jobs_default_params(tables): 'JOB_TYPE_B': {'b1': 'b1_default'}, 'JOB_TYPE_C': {}, } - costs = { - 'JOB_TYPE_A': {'cost': Decimal('1.0')}, - 'JOB_TYPE_B': {'cost': Decimal('1.0')}, - 'JOB_TYPE_C': {'cost': Decimal('1.0')}, - } + costs = [ + {'job_type': 'JOB_TYPE_A', 'cost': Decimal('1.0')}, + {'job_type': 'JOB_TYPE_B', 'cost': Decimal('1.0')}, + {'job_type': 'JOB_TYPE_C', 'cost': Decimal('1.0')}, + ] payload = [ {}, {'job_type': 'JOB_TYPE_A'}, @@ -323,24 +334,44 @@ def test_put_jobs_default_params(tables): def test_put_jobs_costs(tables): tables.users_table.put_item(Item={'user_id': 'user1', 'remaining_credits': Decimal(100)}) - costs = { - 'RTC_GAMMA': { + costs = [ + { + 'job_type': 'RTC_GAMMA', 'cost_parameter': 'resolution', - 'cost_table': { - 30: Decimal('5.0'), - 20: Decimal('15.0'), - 10: Decimal('60.0'), - }, - }, - 'INSAR_ISCE_BURST': { + 'cost_table': [ + { + 'parameter_value': 30.0, + 'cost': Decimal('5.0'), + }, + { + 'parameter_value': 20.0, + 'cost': Decimal('15.0'), + }, + { + 'parameter_value': 10.0, + 'cost': Decimal('60.0'), + }, + ], + }, + { + 'job_type': 'INSAR_ISCE_BURST', 'cost_parameter': 'looks', - 'cost_table': { - '20x4': Decimal('0.4'), - '10x2': Decimal('0.7'), - '5x1': Decimal('1.8'), - }, + 'cost_table': [ + { + 'parameter_value': '20x4', + 'cost': Decimal('0.4'), + }, + { + 'parameter_value': '10x2', + 'cost': Decimal('0.7'), + }, + { + 'parameter_value': '5x1', + 'cost': Decimal('1.8'), + }, + ], }, - } + ] default_params = { 'RTC_GAMMA': {'resolution': 30}, 'INSAR_ISCE_BURST': {'looks': '20x4'},