diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 05900982e..7c18bda52 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.0.14 +current_version = 0.0.15 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/CHANGELOG.md b/CHANGELOG.md index f642fdf80..3192ddf5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,5 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). [Unreleased] + - Fixed database intialisation - Project scaffolding +- TLC for python 3.9 diff --git a/orchestrator/__init__.py b/orchestrator/__init__.py index 5f01191e8..1d9021c32 100644 --- a/orchestrator/__init__.py +++ b/orchestrator/__init__.py @@ -13,7 +13,7 @@ """This is the orchestrator workflow engine.""" -__version__ = "0.0.14" +__version__ = "0.0.15" from orchestrator.app import OrchestratorCore from orchestrator.settings import app_settings, oauth2_settings diff --git a/orchestrator/domain/base.py b/orchestrator/domain/base.py index e03f44213..f28b339b7 100644 --- a/orchestrator/domain/base.py +++ b/orchestrator/domain/base.py @@ -23,6 +23,7 @@ from pydantic import BaseModel, Field, ValidationError from pydantic.main import ModelMetaclass from pydantic.types import ConstrainedList +from pydantic.typing import get_args, get_origin from sqlalchemy import and_ from sqlalchemy.orm import selectinload @@ -49,8 +50,8 @@ def _is_constrained_list_type(type: Type) -> bool: except Exception: # Strip generic arguments, it still might be a subclass - if hasattr(type, "__origin__"): - return _is_constrained_list_type(type.__origin__) + if get_origin(type): + return _is_constrained_list_type(get_origin(type)) else: return False @@ -91,7 +92,7 @@ def __init_subclass__( # Check if child subscription instance models conform to the same lifecycle for product_block_field_name, product_block_field_type in cls._product_block_fields_.items(): if is_list_type(product_block_field_type) or is_optional_type(product_block_field_type): - product_block_field_type = product_block_field_type.__args__[0] + product_block_field_type = get_args(product_block_field_type)[0] if lifecycle: for lifecycle_status in lifecycle: @@ -149,7 +150,7 @@ def _init_instances( if is_list_type(product_block_field_type): if _is_constrained_list_type(product_block_field_type): - product_block_model = product_block_field_type.__args__[0] + product_block_model = get_args(product_block_field_type)[0] default_value = product_block_field_type() # if constrainedlist has minimum, return that minimum else empty list if product_block_field_type.min_items: @@ -232,7 +233,7 @@ def domain_filter(instance: SubscriptionInstanceTable) -> bool: else: product_block_model_list = instances[product_block_field_name] - product_block_model = product_block_field_type.__args__[0] + product_block_model = get_args(product_block_field_type)[0] instance_list: List[SubscriptionInstanceTable] = list( filter( filter_func, flatten(grouped_instances.get(name, []) for name in product_block_model.__names__) @@ -247,7 +248,7 @@ def domain_filter(instance: SubscriptionInstanceTable) -> bool: else: product_block_model = product_block_field_type if is_optional_type(product_block_field_type): - product_block_model = product_block_model.__args__[0] + product_block_model = get_args(product_block_model)[0] instance = only( list( @@ -426,19 +427,19 @@ def _load_instances_values(cls, instance_values: List[SubscriptionInstanceValueT """ instance_values_dict: State = {} - list_field_names = [] + list_field_names = set() # Set default values for field_name, field_type in cls._non_product_block_fields_.items(): # Ensure that empty lists are handled OK if is_list_type(field_type): instance_values_dict[field_name] = [] - list_field_names.append(field_name) + list_field_names.add(field_name) for siv in instance_values: # check the type of the siv in the instance and act accordingly: only lists and scalar values supported resource_type_name = siv.resource_type.resource_type - if is_list_type(cls._non_product_block_fields_[resource_type_name]): + if resource_type_name in list_field_names: instance_values_dict[resource_type_name].append(siv.value) else: instance_values_dict[resource_type_name] = siv.value @@ -760,7 +761,7 @@ def find_product_block_in(cls: Type[DomainModel]) -> List[ProductBlockModel]: product_blocks_in_model = [] for product_block_field_type in cls._product_block_fields_.values(): if is_list_type(product_block_field_type) or is_optional_type(product_block_field_type): - product_block_model = product_block_field_type.__args__[0] + product_block_model = get_args(product_block_field_type)[0] else: product_block_model = product_block_field_type @@ -1003,12 +1004,8 @@ def __init_subclass__(cls, **kwargs: Any) -> None: # This makes a lot of assuptions about the internals of `typing` if "__orig_bases__" in cls.__dict__ and cls.__dict__["__orig_bases__"]: generic_base_cls = cls.__dict__["__orig_bases__"][0] - if ( - not hasattr(generic_base_cls, "item_type") - and hasattr(generic_base_cls, "__args__") - and generic_base_cls.__args__ - ): - cls.item_type = generic_base_cls.__args__[0] + if not hasattr(generic_base_cls, "item_type") and get_args(generic_base_cls): + cls.item_type = get_args(generic_base_cls)[0] # Make sure __args__ is set cls.__args__ = (cls.item_type,) diff --git a/orchestrator/types.py b/orchestrator/types.py index 039ca2509..a214c7282 100644 --- a/orchestrator/types.py +++ b/orchestrator/types.py @@ -16,6 +16,7 @@ from typing import Any, Callable, Dict, Generator, List, Literal, Optional, Tuple, Type, TypedDict, TypeVar, Union from pydantic import BaseModel +from pydantic.typing import get_args, get_origin UUIDstr = str State = Dict[str, Any] @@ -102,10 +103,10 @@ def is_of_type(t: Any, test_type: Any) -> bool: """ if ( - hasattr(t, "__origin__") - and hasattr(test_type, "__origin__") - and t.__origin__ is test_type.__origin__ - and t.__args__ == test_type.__args__ + get_origin(t) + and get_origin(test_type) + and get_origin(t) is get_origin(test_type) + and get_args(t) == get_args(test_type) ): return True @@ -148,16 +149,16 @@ def is_list_type(t: Any, test_type: Optional[type] = None) -> bool: >>> is_list_type(Literal[1,2,3]) False """ - if hasattr(t, "__origin__"): + if get_origin(t): if is_optional_type(t): - for arg in t.__args__: + for arg in get_args(t): if is_list_type(arg, test_type): return True - elif t.__origin__ == Literal: + elif get_origin(t) == Literal: # type:ignore return False # Literal cannot contain lists see pep 586 - elif issubclass(t.__origin__, list): - if test_type and t.__args__: - return is_of_type(t.__args__[0], test_type) + elif issubclass(get_origin(t), list): + if test_type and get_args(t): + return is_of_type(get_args(t)[0], test_type) else: return True @@ -184,9 +185,9 @@ def is_optional_type(t: Any, test_type: Optional[type] = None) -> bool: >>> is_optional_type(int) False """ - if hasattr(t, "__origin__"): - if t.__origin__ == Union and len(t.__args__) == 2: - for arg in t.__args__: + if get_origin(t): + if get_origin(t) == Union and len(get_args(t)) == 2 and None.__class__ in get_args(t): # type:ignore + for arg in get_args(t): if arg is None.__class__: continue diff --git a/orchestrator/utils/state.py b/orchestrator/utils/state.py index 5bb89b6a0..a1cb21dfe 100644 --- a/orchestrator/utils/state.py +++ b/orchestrator/utils/state.py @@ -17,6 +17,8 @@ from typing import Any, Callable, List, Optional, Tuple, Union, cast from uuid import UUID +from pydantic.typing import get_args + from orchestrator.domain.base import SubscriptionModel from orchestrator.types import ( FormGenerator, @@ -142,9 +144,7 @@ def _build_arguments(func: Union[StepFunc, InputStepFunc], state: State) -> List Domain models are retrieved from the DB (after `subscription_id` lookup in the state). Everything else is retrieved from the state. - One exception: if a domain model is requested, but no key (variable name) is found for it in the state, it is - interpreted as a request to instantiate it on behalf of the step function. To do so it does lookup `product` and - `customer` values (both UUIDs) in the state. + For domain models only ``Optional`` and ``List`` are supported as container types. Union, Dict and others are not supported Args: func: step function to inspect for requested arguments @@ -191,15 +191,15 @@ def _build_arguments(func: Union[StepFunc, InputStepFunc], state: State) -> List subscription_ids = map(_get_sub_id, state.get(name, [])) subscriptions = [ # Actual type is first argument from list type - param.annotation.__args__[0].from_subscription(subscription_id) + get_args(param.annotation)[0].from_subscription(subscription_id) for subscription_id in subscription_ids ] arguments.append(subscriptions) elif is_optional_type(param.annotation, SubscriptionModel): subscription_id = _get_sub_id(state.get(name)) if subscription_id: - # Actual type is first argument from union type - sub_mod = param.annotation.__args__[0].from_subscription(subscription_id) + # Actual type is first argument from optional type + sub_mod = get_args(param.annotation)[0].from_subscription(subscription_id) arguments.append(sub_mod) else: arguments.append(None) @@ -245,7 +245,7 @@ def load_initial_state_for_modify(organisation: UUID, subscription_id: UUID) -> and passed as values to the step function. The dict `new_state` returned by the step function will be merged with that of the original `state` dict and returned as the final result. - It knows how to deal with Optional parameters. Eg, given:: + It knows how to deal with parameters that have a default. Eg, given:: @inject_args def do_stuff_with_saps(subscription_id: UUID, sap1: Dict, sap2: Optional[Dict] = None) -> State: @@ -284,6 +284,8 @@ def do_stuff(light_path: Sn8LightPath) -> State: present in the state. This will not work for more than one domain model. Eg. you can't request two domain models to be created as we will not know to which of the two domain models `product` is applicable to. + Also supported is wrapping a domain model in ``Optional`` or ``List``. Other types are not supported. + Args: func: a step function with parameters (that should be keys into the state dict, except for optional ones) diff --git a/pyproject.toml b/pyproject.toml index 997e7cef2..e0b20004b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ requires = [ "oauth2-lib~=1.0.4" ] description-file = "README.md" -requires-python = ">=3.6,<3.9" +requires-python = ">=3.6,<3.10" [tool.flit.metadata.urls] Documentation = "https://workfloworchestrator.org/" diff --git a/test/unit_tests/api/test_processes.py b/test/unit_tests/api/test_processes.py index 93d88380d..42cb42dbb 100644 --- a/test/unit_tests/api/test_processes.py +++ b/test/unit_tests/api/test_processes.py @@ -37,7 +37,7 @@ def long_running_step(): @workflow("Long Running Workflow") def long_running_workflow_py(): - return init >> long_running_step >> done + return init >> long_running_step >> long_running_step >> done with WorkflowInstanceForTests(long_running_workflow_py, "long_running_workflow_py"): @@ -143,23 +143,29 @@ def test_long_running_pause(test_client, long_running_workflow): assert response.json()["global_status"] == "PAUSED" response = test_client.get(f"api/processes/{pid}") - assert len(response.json()["steps"]) == 3 + assert len(response.json()["steps"]) == 4 assert response.json()["current_state"]["done"] is True # assume ordered steplist - assert response.json()["steps"][2]["status"] == "pending" + assert response.json()["steps"][3]["status"] == "pending" response = test_client.put("/api/settings/status", json={"global_lock": False}) + + # Make sure it started again + time.sleep(1) + assert response.json()["global_lock"] is False assert response.json()["running_processes"] == 1 assert response.json()["global_status"] == "RUNNING" - # Let it finish + # Let it finish after second lock step + with test_condition: + test_condition.notify_all() time.sleep(1) response = test_client.get(f"api/processes/{pid}") assert HTTPStatus.OK == response.status_code # assume ordered steplist - assert response.json()["steps"][2]["status"] == "complete" + assert response.json()["steps"][3]["status"] == "complete" app_settings.TESTING = True diff --git a/test/unit_tests/domain/test_base.py b/test/unit_tests/domain/test_base.py index 8994a9373..45e714322 100644 --- a/test/unit_tests/domain/test_base.py +++ b/test/unit_tests/domain/test_base.py @@ -2,7 +2,7 @@ from uuid import uuid4 import pytest -from pydantic import ValidationError, conlist +from pydantic import Field, ValidationError, conlist from sqlalchemy.orm.exc import NoResultFound from orchestrator.db import ( @@ -21,6 +21,7 @@ @pytest.fixture def test_product(): + resource_type_list = ResourceTypeTable(resource_type="list_field", description="") resource_type_int = ResourceTypeTable(resource_type="int_field", description="") resource_type_str = ResourceTypeTable(resource_type="str_field", description="") product_block = ProductBlockTable(name="BlockForTest", description="Test Block", tag="TEST", status="active") @@ -31,7 +32,7 @@ def test_product(): name="TestProduct", description="Test ProductTable", product_type="Test", tag="TEST", status="active" ) - product_block.resource_types = [resource_type_int, resource_type_str] + product_block.resource_types = [resource_type_int, resource_type_str, resource_type_list] product_sub_block.resource_types = [resource_type_int, resource_type_str] product.product_blocks = [product_block, product_sub_block] @@ -64,21 +65,27 @@ def test_product_block(test_product_sub_block): class BlockForTestInactive(ProductBlockModel, product_block_name="BlockForTest"): sub_block: SubBlockForTestInactive - sub_block_2: SubBlockForTestInactive + sub_block_2: Optional[SubBlockForTestInactive] = None + sub_block_list: List[SubBlockForTestInactive] = [] int_field: Optional[int] = None str_field: Optional[str] = None + list_field: List[int] = Field(default_factory=list) class BlockForTestProvisioning(BlockForTestInactive, lifecycle=[SubscriptionLifecycle.PROVISIONING]): sub_block: SubBlockForTestProvisioning sub_block_2: SubBlockForTestProvisioning + sub_block_list: List[SubBlockForTestProvisioning] int_field: int str_field: Optional[str] = None + list_field: List[int] class BlockForTest(BlockForTestProvisioning, lifecycle=[SubscriptionLifecycle.ACTIVE]): sub_block: SubBlockForTest sub_block_2: SubBlockForTest + sub_block_list: List[SubBlockForTest] int_field: int str_field: str + list_field: List[int] return BlockForTestInactive, BlockForTestProvisioning, BlockForTest @@ -169,6 +176,7 @@ def test_lifecycle_specific(test_product_model, test_product_type, test_product_ block=BlockForTest.new( int_field=3, str_field="", + list_field=[1], sub_block=SubBlockForTest.new(int_field=3, str_field=""), sub_block_2=SubBlockForTest.new(int_field=3, str_field=""), ), @@ -190,6 +198,7 @@ def test_lifecycle_specific(test_product_model, test_product_type, test_product_ block=BlockForTest.new( int_field=3, str_field="", + list_field=[1], sub_block=SubBlockForTest.new(int_field=3, str_field=""), sub_block_2=SubBlockForTest.new(int_field=3, str_field=""), ), @@ -211,6 +220,7 @@ def test_lifecycle_specific(test_product_model, test_product_type, test_product_ note=None, block=BlockForTestProvisioning.new( int_field=3, + list_field=[1], sub_block=SubBlockForTestProvisioning.new(int_field=3), sub_block_2=SubBlockForTest.new(int_field=3, str_field=""), ), @@ -229,6 +239,7 @@ def test_lifecycle_specific(test_product_model, test_product_type, test_product_ note=None, block=BlockForTestProvisioning.new( int_field=3, + list_field=[1], sub_block=SubBlockForTestProvisioning.new(int_field=3), sub_block_2=SubBlockForTest.new(int_field=3, str_field=""), ), @@ -256,6 +267,7 @@ def test_product_blocks_per_lifecycle( block=BlockForTest.new( int_field=3, str_field="", + list_field=[1], sub_block=SubBlockForTest.new(int_field=3, str_field=""), sub_block_2=SubBlockForTest.new(int_field=3, str_field=""), ), @@ -287,6 +299,7 @@ def test_product_blocks_per_lifecycle( block=BlockForTest.new( int_field=3, str_field="", + list_field=[1], sub_block=SubBlockForTest.new(int_field=3, str_field=""), sub_block_2=SubBlockForTest.new(int_field=3, str_field=""), ), @@ -305,6 +318,7 @@ def test_product_blocks_per_lifecycle( block=BlockForTest.new( int_field=3, str_field="", + list_field=[1], sub_block=SubBlockForTest.new(int_field=3, str_field=""), sub_block_2=SubBlockForTest.new(int_field=3, str_field=""), ), @@ -323,6 +337,7 @@ def test_product_blocks_per_lifecycle( block=BlockForTest.new( int_field=3, str_field="", + list_field=[1], sub_block=SubBlockForTest.new(int_field=3, str_field=""), sub_block_2=SubBlockForTest.new(int_field=3, str_field=""), ), @@ -346,7 +361,7 @@ def test_product_blocks_per_lifecycle( status=SubscriptionLifecycle.ACTIVE, ) - with pytest.raises(ValidationError, match=r"6 validation errors for ProductTypeForTest"): + with pytest.raises(ValidationError, match=r"5 validation errors for ProductTypeForTest"): ProductTypeForTest( product=test_product_model, customer_id=uuid4(), @@ -387,7 +402,8 @@ def test_product_blocks_per_lifecycle( ) -def test_change_lifecycle(test_product_model, test_product_type, test_product_block): +def test_change_lifecycle(test_product_model, test_product_type, test_product_block, test_product_sub_block): + SubBlockForTestInactive, SubBlockForTestProvisioning, SubBlockForTest = test_product_sub_block BlockForTestInactive, BlockForTestProvisioning, BlockForTest = test_product_block ProductTypeForTestInactive, ProductTypeForTestProvisioning, ProductTypeForTest = test_product_type @@ -401,7 +417,7 @@ def test_change_lifecycle(test_product_model, test_product_type, test_product_bl end_date=None, note=None, status=SubscriptionLifecycle.INITIAL, - block=BlockForTestInactive.new(), + block=BlockForTestInactive.new(sub_block_2=SubBlockForTestInactive.new()), ) # Does not work if constraints are not met @@ -414,6 +430,7 @@ def test_change_lifecycle(test_product_model, test_product_type, test_product_bl product_type.block.int_field = 3 product_type.block.sub_block.int_field = 3 product_type.block.sub_block_2.int_field = 3 + product_type.block.list_field = [1] # Does not work if constraints are not met with pytest.raises(ValidationError, match=r"str_field\n none is not an allowed value"): @@ -483,6 +500,29 @@ class TestListProductType(SubscriptionModel, is_base=True): assert ip.dict() == ip2.dict() +def test_update_optional(test_product, test_product_block): + BlockForTestInactive, BlockForTestProvisioning, BlockForTest = test_product_block + + class TestListProductType(SubscriptionModel, is_base=True): + sap: Optional[BlockForTestInactive] = None + + # Creates + ip = TestListProductType.from_product_id(product_id=test_product, customer_id=uuid4()) + ip.save() + + sap = BlockForTestInactive.new(int_field=3, str_field="") + + # Set new sap + ip.sap = sap + + ip.save() + + ip2 = TestListProductType.from_subscription(ip.subscription_id) + + # Old default saps should not be saved + assert ip.dict() == ip2.dict() + + def test_generic_from_subscription(test_product, test_product_type): ProductTypeForTestInactive, ProductTypeForTestProvisioning, ProductTypeForTest = test_product_type @@ -505,10 +545,13 @@ def test_label_is_saved(test_product, test_product_type): assert instance_in_db.label == "My label" -def test_domain_model_attrs_saving_loading(test_product, test_product_type): +def test_domain_model_attrs_saving_loading(test_product, test_product_type, test_product_sub_block): + SubBlockForTestInactive, SubBlockForTestProvisioning, SubBlockForTest = test_product_sub_block ProductTypeForTestInactive, ProductTypeForTestProvisioning, ProductTypeForTest = test_product_type test_model = ProductTypeForTestInactive.from_product_id(product_id=test_product, customer_id=uuid4()) + test_model.block.sub_block_2 = SubBlockForTestInactive.new() + test_model.block.sub_block_list = [SubBlockForTestInactive.new()] test_model.save() db.session.commit() @@ -520,15 +563,21 @@ def test_domain_model_attrs_saving_loading(test_product, test_product_type): SubscriptionInstanceRelationTable.child_id == test_model.block.sub_block_2.subscription_instance_id ).one() assert relation.domain_model_attr == "sub_block_2" - + relation = SubscriptionInstanceRelationTable.query.filter( + SubscriptionInstanceRelationTable.child_id == test_model.block.sub_block_list[0].subscription_instance_id + ).one() + assert relation.domain_model_attr == "sub_block_list" test_model_2 = ProductTypeForTestInactive.from_subscription(test_model.subscription_id) assert test_model == test_model_2 -def test_removal_of_domain_attrs(test_product, test_product_type): +def test_removal_of_domain_attrs(test_product, test_product_type, test_product_sub_block): + SubBlockForTestInactive, SubBlockForTestProvisioning, SubBlockForTest = test_product_sub_block ProductTypeForTestInactive, ProductTypeForTestProvisioning, ProductTypeForTest = test_product_type test_model = ProductTypeForTestInactive.from_product_id(product_id=test_product, customer_id=uuid4()) + test_model.block.sub_block_2 = SubBlockForTestInactive.new() + test_model.save() db.session.commit() relation = SubscriptionInstanceRelationTable.query.filter(