diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 25bb56bdf4..a6bb44ec85 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -24,6 +24,8 @@ RuntimeSpec, Result, SpecInfo, + LazyIn, + LazyOut, LazyField, TaskHook, attr_fields, @@ -225,7 +227,7 @@ def __setstate__(self, state): def __getattr__(self, name): if name == "lzout": # lazy output - return LazyField(self, "output") + return LazyOut(self) return self.__getattribute__(name) def help(self, returnhelp=False): @@ -932,7 +934,7 @@ def __init__( def __getattr__(self, name): if name == "lzin": - return LazyField(self, "input") + return LazyIn(self) if name == "lzout": return super().__getattr__(name) if name in self.name2obj: diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 432efb1768..384ee127a5 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -1,9 +1,10 @@ """Administrative support for the engine framework.""" import asyncio import asyncio.subprocess as asp -import attr import itertools -import abc +import inspect + +# import abc from pathlib import Path import os import sys @@ -14,8 +15,10 @@ from time import strftime from traceback import format_exception import typing as ty -import inspect -import warnings + +# import inspect +# import warnings +import attr from filelock import SoftFileLock, Timeout import cloudpickle as cp @@ -28,9 +31,9 @@ Result, LazyField, MultiOutputObj, - MultiInputObj, - MultiInputFile, - MultiOutputFile, + # MultiInputObj, + # MultiInputFile, + # MultiOutputFile, ) from .helpers_file import hash_file, hash_dir, copyfile, is_existing_file from ..utils.hash import hash_object @@ -259,15 +262,14 @@ def make_klass(spec): return None fields = spec.fields if fields: - newfields = dict() + newfields = {} for item in fields: if len(item) == 2: name = item[0] if isinstance(item[1], attr._make._CountingAttr): - newfields[name] = item[1] - newfields[name].validator(custom_validator) + newfield = item[1] else: - newfields[name] = attr.ib(type=item[1], validator=custom_validator) + newfield = attr.ib(type=item[1]) else: if ( any([isinstance(ii, attr._make._CountingAttr) for ii in item]) @@ -278,210 +280,447 @@ def make_klass(spec): "(name, type, default), (name, type, default, metadata)" "or (name, type, metadata)" ) - else: - if len(item) == 3: - name, tp = item[:2] - if isinstance(item[-1], dict) and "help_string" in item[-1]: - mdata = item[-1] - newfields[name] = attr.ib( - type=tp, metadata=mdata, validator=custom_validator - ) - else: - dflt = item[-1] - newfields[name] = attr.ib( - type=tp, default=dflt, validator=custom_validator - ) - elif len(item) == 4: - name, tp, dflt, mdata = item - newfields[name] = attr.ib( - type=tp, - default=dflt, - metadata=mdata, - validator=custom_validator, - ) - # if type has converter, e.g. MultiInputObj - if hasattr(newfields[name].type, "converter"): - newfields[name].converter = newfields[name].type.converter + kwargs = {} + if len(item) == 3: + name, tp = item[:2] + if isinstance(item[-1], dict) and "help_string" in item[-1]: + mdata = item[-1] + kwargs["metadata"] = mdata + else: + kwargs["default"] = item[-1] + elif len(item) == 4: + name, tp, dflt, mdata = item + kwargs["default"] = dflt + kwargs["metadata"] = mdata + newfield = attr.ib( + type=tp, + **kwargs, + ) + newfield.converter = TypeCoercer[newfield.type]( + newfield.type, + coercible=[ + (os.PathLike, os.PathLike), + (str, os.PathLike), + (os.PathLike, str), + (ty.Sequence, ty.Sequence), + (ty.Mapping, ty.Mapping), + ], + not_coercible=[(str, ty.Sequence), (ty.Sequence, str)], + ) + try: + newfield.metadata["allowed_values"] + except KeyError: + pass + else: + newfield.validator = allowed_values_validator + newfields[name] = newfield fields = newfields return attr.make_class(spec.name, fields, bases=spec.bases, kw_only=True) -def custom_validator(instance, attribute, value): - """simple custom validation - take into account ty.Union, ty.List, ty.Dict (but only one level depth) - adding an additional validator, if allowe_values provided +T = ty.TypeVar("T") +TypeOrAny = ty.Union[type, ty.Any] + + +class TypeCoercer(ty.Generic[T]): + """Coerces an object to the given type, expanding container classes and unions. + + Parameters + ---------- + tp : type + the type objects will be coerced to + coercible: Iterable[tuple[type or Any, type or Any]], optional + limits coercing between the pairs of types where they appear within the + tree of more complex nested container types. + not_coercible: Iterable[tuple[type or Any, type or Any]], optional + excludes the limits coercing between the pairs of types where they appear within + the tree of more complex nested container types. Overrides 'coercible' to enable + you to carve out exceptions, such as + TypeCoercer(list, coercible=[(ty.Iterable, list)], not_coercible=[(str, list)]) """ - validators = [] - tp_attr = attribute.type - # a flag that could be changed to False, if the type is not recognized - check_type = True - if ( - value is attr.NOTHING - or value is None - or attribute.name.startswith("_") # e.g. _func - or isinstance(value, LazyField) - or tp_attr - in [ - ty.Any, - inspect._empty, - MultiOutputObj, - MultiInputObj, - MultiOutputFile, - MultiInputFile, - ] + + coercible: list[tuple[TypeOrAny, TypeOrAny]] + not_coercible: list[tuple[TypeOrAny, TypeOrAny]] + + def __init__( + self, + tp, + coercible: ty.Optional[ty.Iterable[tuple[TypeOrAny, TypeOrAny]]] = None, + not_coercible: ty.Optional[ty.Iterable[tuple[TypeOrAny, TypeOrAny]]] = None, ): - check_type = False # no checking of the type - elif isinstance(tp_attr, type) or tp_attr in [File, Directory]: - tp = _single_type_update(tp_attr, name=attribute.name) - cont_type = None - else: # more complex types - cont_type, tp_attr_list = _check_special_type(tp_attr, name=attribute.name) - if cont_type is ty.Union: - tp, check_type = _types_updates(tp_attr_list, name=attribute.name) - elif cont_type is list: - tp, check_type = _types_updates(tp_attr_list, name=attribute.name) - elif cont_type is dict: - # assuming that it should have length of 2 for keys and values - if len(tp_attr_list) != 2: - check_type = False - else: - tp_attr_key, tp_attr_val = tp_attr_list - # updating types separately for keys and values - tp_k, check_k = _types_updates([tp_attr_key], name=attribute.name) - tp_v, check_v = _types_updates([tp_attr_val], name=attribute.name) - # assuming that I have to be able to check keys and values - if not (check_k and check_v): - check_type = False - else: - tp = {"key": tp_k, "val": tp_v} - else: - warnings.warn( - f"no type check for {attribute.name} field, " - f"no type check implemented for value {value} and type {tp_attr}" - ) - check_type = False + def expand(t): + origin = ty.get_origin(t) + if origin is None: + return t + args = ty.get_args(t) + if not args or args == (Ellipsis,): + assert isinstance(origin, type) + return origin + return (origin, [expand(a) for a in args]) + + self.coercible = ( + list(coercible) if coercible is not None else [(ty.Any, ty.Any)] + ) + self.not_coercible = list(not_coercible) if not_coercible is not None else [] + self.pattern = expand(tp) - if check_type: - validators.append(_type_validator(instance, attribute, value, tp, cont_type)) + def __call__(self, object_: ty.Any) -> T: + """Attempts to coerce - # checking additional requirements for values (e.g. allowed_values) - meta_attr = attribute.metadata - if "allowed_values" in meta_attr: - validators.append(_allowed_values_validator(isinstance, attribute, value)) - return validators + Parameters + ---------- + object_ : ty.Any + the object to coerce + Returns + ------- + T + the coerced object -def _type_validator(instance, attribute, value, tp, cont_type): - """creating a customized type validator, - uses validator.deep_iterable/mapping if the field is a container - (i.e. ty.List or ty.Dict), - it also tries to guess when the value is a list due to the splitter - and validates the elements - """ - if cont_type is None or cont_type is ty.Union: - # if tp is not (list,), we are assuming that the value is a list - # due to the splitter, so checking the member types - if isinstance(value, list) and tp != (list,): - return attr.validators.deep_iterable( - member_validator=attr.validators.instance_of( - tp + (attr._make._Nothing,) + Raises + ------ + TypeError + if the coercion is not possible, or not specified by the `coercible`/`not_coercible` + parameters, then a TypeError is raised + """ + + def expand_and_coerce(obj, pattern: ty.Union[type | tuple]): + """Attempt to expand the object along the lines of the coercion pattern""" + if not isinstance(pattern, tuple): + return coerce_single(obj, pattern) + origin, pattern_args = pattern + if origin is ty.Union: + # Return the first argument in the union that is coercible + for arg in pattern_args: + try: + return expand_and_coerce(obj, arg) + except TypeError: + pass + raise TypeError( + f"Could not coerce {obj} to any of the union types {pattern_args}" ) - )(instance, attribute, value) - else: - return attr.validators.instance_of(tp + (attr._make._Nothing,))( - instance, attribute, value + if not self.is_instance(obj, origin): + self._check_coercible(obj, origin) + type_ = origin + else: + type_ = type(obj) + if issubclass(type_, ty.Mapping): + return coerce_mapping(obj, type_, pattern_args) + return coerce_sequence(obj, type_, pattern_args) + + def coerce_single(obj, pattern): + """Coerce a "single" object, i.e. one not nested within a container""" + if ( + obj is attr.NOTHING + or pattern is inspect._empty + or self.is_instance(obj, pattern) + ): + return obj + if isinstance(obj, LazyField): + self._check_coercible(obj.type, pattern) + return obj + self._check_coercible(obj, pattern) + return coerce_to_type(obj, pattern) + + def coerce_mapping( + obj: ty.Mapping, type_: ty.Type[ty.Mapping], pattern_args: list + ): + """Coerce a mapping (e.g. dict)""" + assert len(pattern_args) == 2 + try: + items = obj.items() + except AttributeError as e: + msg = ( + f" (part of coercion from {object_} to {self.pattern}" + if obj is not object_ + else "" + ) + raise TypeError( + f"Could not coerce to {type_} as {obj} is not a mapping type{msg}" + ) from e + return coerce_to_type( + ( + ( + expand_and_coerce(k, pattern_args[0]), + expand_and_coerce(v, pattern_args[1]), + ) + for k, v in items + ), + type_, ) - elif cont_type is list: - return attr.validators.deep_iterable( - member_validator=attr.validators.instance_of(tp + (attr._make._Nothing,)) - )(instance, attribute, value) - elif cont_type is dict: - return attr.validators.deep_mapping( - key_validator=attr.validators.instance_of(tp["key"]), - value_validator=attr.validators.instance_of( - tp["val"] + (attr._make._Nothing,) - ), - )(instance, attribute, value) - else: - raise Exception( - f"container type of {attribute.name} should be None, list, dict or ty.Union, " - f"and not {cont_type}" - ) + def coerce_sequence( + obj: ty.Sequence, type_: ty.Type[ty.Sequence], pattern_args: list + ): + """Coerce a sequence object (e.g. list, tuple, ...)""" + try: + args = list(obj) + except TypeError as e: + msg = ( + f" (part of coercion from {object_} to {self.pattern}" + if obj is not object_ + else "" + ) + raise TypeError( + f"Could not coerce to {type_} as {obj} is not iterable{msg}" + ) from e + if issubclass(type_, ty.Tuple): # type: ignore[arg-type] + if pattern_args[-1] is Ellipsis: + pattern_args = itertools.chain( + pattern_args[:-2], itertools.repeat(pattern_args[-2]) + ) + elif len(pattern_args) != len(args): + raise TypeError( + f"Incorrect number of items in {obj}, expected " + f"{len(pattern_args)}, got {len(args)}" + ) + return coerce_to_type( + [expand_and_coerce(o, p) for o, p in zip(args, pattern_args)], type_ + ) + assert len(pattern_args) == 1 + return coerce_to_type( + [expand_and_coerce(o, pattern_args[0]) for o in args], type_ + ) -def _types_updates(tp_list, name): - """updating the type's tuple with possible additional types""" - tp_upd_list = [] - check = True - for tp_el in tp_list: - tp_upd = _single_type_update(tp_el, name, simplify=True) - if tp_upd is None: - check = False - break - else: - tp_upd_list += list(tp_upd) - tp_upd = tuple(set(tp_upd_list)) - return tp_upd, check + def coerce_to_type(obj, type_): + """Attempt to do the innermost (i.e. non-nested) coercion and fail with + helpful message + """ + try: + return type_(obj) + except TypeError as e: + msg = ( + f" (part of coercion from {object_} to {self.pattern}" + if obj is not object_ + else "" + ) + raise TypeError(f"Cannot coerce {obj} into {type_}{msg}") from e + return expand_and_coerce(object_, self.pattern) -def _single_type_update(tp, name, simplify=False): - """updating a single type with other related types - e.g. adding bytes for str - if simplify is True, than changing typing.List to list etc. - (assuming that I validate only one depth, so have to simplify at some point) - """ - if isinstance(tp, type) or tp in [File, Directory]: - if tp is str: - return (str, bytes) - elif tp in [File, Directory, os.PathLike]: - return (os.PathLike, str) - elif tp is float: - return (float, int) - else: - return (tp,) - elif simplify is True: - warnings.warn(f"simplify validator for {name} field, checking only one depth") - cont_tp, types_list = _check_special_type(tp, name=name) - if cont_tp is list: - return (list,) - elif cont_tp is dict: - return (dict,) - elif cont_tp is ty.Union: - return types_list - else: - warnings.warn( - f"no type check for {name} field, type check not implemented for type of {tp}" - ) - return None - else: - warnings.warn( - f"no type check for {name} field, type check not implemented for type - {tp}, " - f"consider using simplify=True" - ) - return None + def _check_coercible(self, source: object | type, target: type | ty.Any): + """Checks whether the source object or type is coercible to the target type + given the coercion rules defined in the `coercible` and `not_coercible` attrs + Parameters + ---------- + source : object | type + source object or type to be coerced + target : type | ty.Any + target type for the source to be coerced to + """ -def _check_special_type(tp, name): - """checking if the type is a container: ty.List, ty.Dict or ty.Union""" - if sys.version_info.minor >= 8: - return ty.get_origin(tp), ty.get_args(tp) - else: - if isinstance(tp, type): # simple type - return None, () - else: - if tp._name == "List": - return list, tp.__args__ - elif tp._name == "Dict": - return dict, tp.__args__ - elif tp.__origin__ is ty.Union: - return ty.Union, tp.__args__ - else: - warnings.warn( - f"not type check for {name} field, type check not implemented for type {tp}" - ) - return None, () + source_check = ( + self.is_or_subclass if inspect.isclass(source) else self.is_instance + ) + def matches(criteria): + return [ + (src, tgt) + for src, tgt in criteria + if source_check(source, src) and self.is_or_subclass(target, tgt) + ] + + if not matches(self.coercible): + raise TypeError( + f"Cannot coerce {source} into {target} as the coercion doesn't match " + f"any of the explicit inclusion criteria {self.coercible}" + ) + matches_not_coercible = matches(self.not_coercible) + if matches_not_coercible: + raise TypeError( + f"Cannot coerce {source} into {target} as it is explicitly excluded by " + f"the following coercion criteria {matches_not_coercible}" + ) -def _allowed_values_validator(instance, attribute, value): + @staticmethod + def is_instance(obj, cls): + """Checks whether the object is an instance of cls or that cls is typing.Any""" + return cls is ty.Any or isinstance(obj, cls) + + @staticmethod + def is_or_subclass(a, b): + """Checks whether the class a is either the same as b, a subclass of b or b is + typing.Any""" + return a is b or b is ty.Any or issubclass(a, b) + + +# def custom_validator(instance, attribute, value): +# """simple custom validation +# take into account ty.Union, ty.List, ty.Dict (but only one level depth) +# adding an additional validator, if allowe_values provided +# """ +# validators = [] +# tp_attr = attribute.type +# # a flag that could be changed to False, if the type is not recognized +# check_type = True +# if ( +# value is attr.NOTHING +# or value is None +# or attribute.name.startswith("_") # e.g. _func +# or isinstance(value, LazyField) +# or tp_attr +# in [ +# ty.Any, +# inspect._empty, +# MultiOutputObj, +# MultiInputObj, +# MultiOutputFile, +# MultiInputFile, +# ] +# ): +# check_type = False # no checking of the type +# elif isinstance(tp_attr, type) or tp_attr in [File, Directory]: +# tp = _single_type_update(tp_attr, name=attribute.name) +# cont_type = None +# else: # more complex types +# cont_type, tp_attr_list = _check_special_type(tp_attr, name=attribute.name) +# if cont_type is ty.Union: +# tp, check_type = _types_updates(tp_attr_list, name=attribute.name) +# elif cont_type is list: +# tp, check_type = _types_updates(tp_attr_list, name=attribute.name) +# elif cont_type is dict: +# # assuming that it should have length of 2 for keys and values +# if len(tp_attr_list) != 2: +# check_type = False +# else: +# tp_attr_key, tp_attr_val = tp_attr_list +# # updating types separately for keys and values +# tp_k, check_k = _types_updates([tp_attr_key], name=attribute.name) +# tp_v, check_v = _types_updates([tp_attr_val], name=attribute.name) +# # assuming that I have to be able to check keys and values +# if not (check_k and check_v): +# check_type = False +# else: +# tp = {"key": tp_k, "val": tp_v} +# else: +# warnings.warn( +# f"no type check for {attribute.name} field, " +# f"no type check implemented for value {value} and type {tp_attr}" +# ) +# check_type = False + +# if check_type: +# validators.append(_type_validator(instance, attribute, value, tp, cont_type)) + +# # checking additional requirements for values (e.g. allowed_values) +# meta_attr = attribute.metadata +# if "allowed_values" in meta_attr: +# validators.append(_allowed_values_validator(isinstance, attribute, value)) +# return validators + + +# def _type_validator(instance, attribute, value, tp, cont_type): +# """creating a customized type validator, +# uses validator.deep_iterable/mapping if the field is a container +# (i.e. ty.List or ty.Dict), +# it also tries to guess when the value is a list due to the splitter +# and validates the elements +# """ +# if cont_type is None or cont_type is ty.Union: +# # if tp is not (list,), we are assuming that the value is a list +# # due to the splitter, so checking the member types +# if isinstance(value, list) and tp != (list,): +# return attr.validators.deep_iterable( +# member_validator=attr.validators.instance_of( +# tp + (attr._make._Nothing,) +# ) +# )(instance, attribute, value) +# else: +# return attr.validators.instance_of(tp + (attr._make._Nothing,))( +# instance, attribute, value +# ) +# elif cont_type is list: +# return attr.validators.deep_iterable( +# member_validator=attr.validators.instance_of(tp + (attr._make._Nothing,)) +# )(instance, attribute, value) +# elif cont_type is dict: +# return attr.validators.deep_mapping( +# key_validator=attr.validators.instance_of(tp["key"]), +# value_validator=attr.validators.instance_of( +# tp["val"] + (attr._make._Nothing,) +# ), +# )(instance, attribute, value) +# else: +# raise Exception( +# f"container type of {attribute.name} should be None, list, dict or ty.Union, " +# f"and not {cont_type}" +# ) + + +# def _types_updates(tp_list, name): +# """updating the type's tuple with possible additional types""" +# tp_upd_list = [] +# check = True +# for tp_el in tp_list: +# tp_upd = _single_type_update(tp_el, name, simplify=True) +# if tp_upd is None: +# check = False +# break +# else: +# tp_upd_list += list(tp_upd) +# tp_upd = tuple(set(tp_upd_list)) +# return tp_upd, check + + +# def _single_type_update(tp, name, simplify=False): +# """updating a single type with other related types - e.g. adding bytes for str +# if simplify is True, than changing typing.List to list etc. +# (assuming that I validate only one depth, so have to simplify at some point) +# """ +# if isinstance(tp, type) or tp in [File, Directory]: +# if tp is str: +# return (str, bytes) +# elif tp in [File, Directory, os.PathLike]: +# return (os.PathLike, str) +# elif tp is float: +# return (float, int) +# else: +# return (tp,) +# elif simplify is True: +# warnings.warn(f"simplify validator for {name} field, checking only one depth") +# cont_tp, types_list = _check_special_type(tp, name=name) +# if cont_tp is list: +# return (list,) +# elif cont_tp is dict: +# return (dict,) +# elif cont_tp is ty.Union: +# return types_list +# else: +# warnings.warn( +# f"no type check for {name} field, type check not implemented for type of {tp}" +# ) +# return None +# else: +# warnings.warn( +# f"no type check for {name} field, type check not implemented for type - {tp}, " +# f"consider using simplify=True" +# ) +# return None + + +# def _check_special_type(tp, name): +# """checking if the type is a container: ty.List, ty.Dict or ty.Union""" +# if sys.version_info.minor >= 8: +# return ty.get_origin(tp), ty.get_args(tp) +# else: +# if isinstance(tp, type): # simple type +# return None, () +# else: +# if tp._name == "List": +# return list, tp.__args__ +# elif tp._name == "Dict": +# return dict, tp.__args__ +# elif tp.__origin__ is ty.Union: +# return ty.Union, tp.__args__ +# else: +# warnings.warn( +# f"not type check for {name} field, type check not implemented for type {tp}" +# ) +# return None, () + + +def allowed_values_validator(_, attribute, value): """checking if the values is in allowed_values""" allowed = attribute.metadata["allowed_values"] if value is attr.NOTHING or isinstance(value, LazyField): @@ -926,126 +1165,3 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_value, traceback): self.lock.release() return None - - -T = ty.TypeVar("T") -TypeOrAny = ty.Union[type, ty.Any] - - -class TypeCoercer(ty.Generic[T]): - """Coerces an object to the given type, expanding container classes and unions. - - Parameters - ---------- - tp : type - the type objects will be coerced to - coercible: Iterable[tuple[type or Any, type or Any]], optional - limits coercing between the pairs of types where they appear within the - tree of more complex nested container types. - not_coercible: Iterable[tuple[type or Any, type or Any]], optional - excludes the limits coercing between the pairs of types where they appear within - the tree of more complex nested container types. Overrides 'coercible' to enable - you to carve out exceptions, such as - TypeCoercer(list, coercible=[(ty.Iterable, list)], not_coercible=[(str, list)]) - """ - - coercible: list[tuple[TypeOrAny, TypeOrAny]] - not_coercible: list[tuple[TypeOrAny, TypeOrAny]] - - def __init__( - self, - tp, - coercible: ty.Optional[ty.Iterable[tuple[TypeOrAny, TypeOrAny]]] = None, - not_coercible: ty.Optional[ty.Iterable[tuple[TypeOrAny, TypeOrAny]]] = None, - ): - def expand(t): - origin = ty.get_origin(t) - if origin is None: - if any( - t is k or k is ty.Any or issubclass(t, k) - for k in self.coercible_targets - ): - return t - return None - if isinstance(origin, abc.ABCMeta): - raise TypeError( - f"Cannot coerce to abstract type {tp} ({origin} is abstract)" - ) - args = ty.get_args(t) - if not args or args == (Ellipsis,): - assert isinstance(origin, type) - return origin - return (origin, [expand(a) for a in args]) - - self.coercible = ( - list(coercible) if coercible is not None else [(ty.Any, ty.Any)] - ) - self.not_coercible = list(not_coercible) if not_coercible is not None else [] - self.pattern = expand(tp) - - def __call__(self, obj: ty.Any) -> T: - def coerce(obj, pattern: ty.Union[type | tuple | None]): - if not isinstance(pattern, tuple): - if ( - pattern is None - or isinstance(obj, pattern) - or not self._is_coercible(obj, pattern) - ): - return obj - return pattern(obj) - origin, args = pattern - if origin is ty.Union: - # Return the first argument in the union that is coercible - for arg in args: - try: - return coerce(obj, arg) - except TypeError: - pass - raise TypeError( - f"Could not coerce {obj} to any of the union types {args}" - ) - if issubclass(origin, ty.Mapping): - assert len(args) == 2 - return origin( - (coerce(k, args[0]), coerce(v, args[1])) for k, v in obj.items() - ) - type_ = origin if self._is_coercible(obj, origin) else type(obj) - if issubclass(origin, ty.Tuple): # type: ignore[arg-type] - if args[-1] is Ellipsis: - args = itertools.chain(args[:-2], itertools.repeat(args[-2])) - elif len(args) != len(obj): - raise TypeError( - f"Incorrect number of items in {obj}, expected {len(args)}, " - f"got {len(obj)}" - ) - return type_(coerce(o, p) for o, p in zip(obj, args)) - assert len(args) == 1 - return type_(coerce(o, args[0]) for o in obj) - - return coerce(obj, self.pattern) - - def _is_coercible(self, source, target): - def is_instance(o, c): - return c is ty.Any or isinstance(o, c) - - def is_or_subclass(a, b): - return a is b or b is ty.Any or issubclass(a, b) - - return ( - any( - is_instance(source, src) and is_or_subclass(target, tgt) - for src, tgt in self.coercible - ) - or self.coercible is None - ) and not any( - is_instance(source, src) and is_or_subclass(target, tgt) - for src, tgt in self.not_coercible - ) - - @property - def coercible_targets(self): - return [t for _, t in self.coercible] - - @property - def not_coercible_targets(self): - return [t for _, t in self.not_coercible] diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 42b03fac9c..543962491f 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -48,37 +48,54 @@ def __bytes_repr__(self, cache): # class Directory: # """An :obj:`os.pathlike` object, designating a folder.""" +T = ty.TypeVar("T") -class MultiInputObj: + +class MultiInputObj(ty.List[T]): """A ty.List[ty.Any] object, converter changes a single values to a list""" - @classmethod - def converter(cls, value): - from .helpers import ensure_list + def __init__(self, items): + if not isinstance(items, ty.Iterable): + items = (items,) + super().__init__(items) - if value == attr.NOTHING: - return value - else: - return ensure_list(value) + # @classmethod + # def converter(cls, value): + # from .helpers import ensure_list + # if value == attr.NOTHING: + # return value + # else: + # return ensure_list(value) -class MultiOutputObj: - """A ty.List[ty.Any] object, converter changes an 1-el list to the single value""" - @classmethod - def converter(cls, value): - if isinstance(value, list) and len(value) == 1: - return value[0] - else: - return value +# class MultiOutputObj: +# """A ty.List[ty.Any] object, converter changes an 1-el list to the single value""" + +# @classmethod +# def converter(cls, value): +# if isinstance(value, list) and len(value) == 1: +# return value[0] +# else: +# return value +# Not attempting to do the conversion from list to singular value as this seems like +# poor design. Downstream nodes will need to handle the case where it is a list in any +# case so no point creating extra work by requiring them to handle the single value case +# as well +MultiOutputObj = ty.List -class MultiInputFile(MultiInputObj): - """A ty.List[File] object, converter changes a single file path to a list""" +# class MultiInputFile(MultiInputObj): +# """A ty.List[File] object, converter changes a single file path to a list""" +MultiInputFile = MultiInputObj[File] -class MultiOutputFile(MultiOutputObj): - """A ty.List[File] object, converter changes an 1-el list to the single value""" + +# class MultiOutputFile(MultiOutputObj): +# """A ty.List[File] object, converter changes an 1-el list to the single value""" + +# See note on MultiOutputObj +MultiOutputFile = ty.List[File] @attr.s(auto_attribs=True, kw_only=True) @@ -758,43 +775,68 @@ class SingularitySpec(ContainerSpec): container: str = attr.ib("singularity", metadata={"help_string": "container type"}) -class LazyField: - """Lazy fields implement promises.""" - - def __init__(self, node, attr_type): - """Initialize a lazy field.""" - self.name = node.name - if attr_type == "input": - self.fields = [field[0] for field in node.input_spec.fields] - elif attr_type == "output": - self.fields = node.output_names - else: - raise ValueError(f"LazyField: Unknown attr_type: {attr_type}") - self.attr_type = attr_type - self.field = None +@attr.s +class LazyInterface: + _node: "core.TaskBase" = attr.ib() + _attr_type: str def __getattr__(self, name): - if name in self.fields or name == "all_": - self.field = name - return self - if name in dir(self): - return self.__getattribute__(name) - raise AttributeError( - f"Task {self.name} has no {self.attr_type} attribute {name}" + if name not in self._field_names: + raise AttributeError( + f"Task {self._node.name} has no {self._attr_type} attribute {name}" + ) + return LazyField( + name=self._node.name, + field=name, + attr_type=self._attr_type, + type=self._get_type(name), ) - def __getstate__(self): - state = self.__dict__.copy() - state["name"] = self.name - state["fields"] = self.fields - state["field"] = self.field - return state - def __setstate__(self, state): - self.__dict__.update(state) +class LazyIn(LazyInterface): + _attr_type = "input" + + def _get_type(self, name): + return next(t for n, t in self._node.input_spec.fields if n == name).type + + @property + def _field_names(self): + return [field[0] for field in self._node.input_spec.fields] + + +class LazyOut(LazyInterface): + _attr_type = "output" + + def _get_type(self, name): + return next(t for n, t in self._node.output_spec.fields if n == name) + + @property + def _field_names(self): + return self._node.output_names + + +@attr.s(auto_attribs=True, kw_only=True) +class LazyField: + """Lazy fields implement promises.""" + + name: str + field: str + attr_type: str + type: ty.Type[ty.Any] + + # def __getstate__(self): + # state = self.__dict__.copy() + # state["name"] = self.name + # state["field"] = self.field + # state["attr_type"] = self.attr_type + # state["type"] = self.type + # return state + + # def __setstate__(self, state): + # self.__dict__.update(state) def __repr__(self): - return f"LF('{self.name}', '{self.field}')" + return f"LF('{self.name}', '{self.field}', {self.type})" def get_value(self, wf, state_index=None): """Return the value of a lazy field.""" @@ -859,3 +901,6 @@ def path_to_string(value): elif isinstance(value, list) and len(value) and isinstance(value[0], Path): value = [str(val) for val in value] return value + + +from . import core # noqa diff --git a/pydra/engine/task.py b/pydra/engine/task.py index c6125fbadd..7ac5bb456e 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -137,11 +137,11 @@ def __init__( ), ) ) - fields.append(("_func", attr.ib(default=cp.dumps(func), type=str))) + fields.append(("_func", attr.ib(default=cp.dumps(func), type=bytes))) input_spec = SpecInfo(name="Inputs", fields=fields, bases=(BaseSpec,)) else: input_spec.fields.append( - ("_func", attr.ib(default=cp.dumps(func), type=str)) + ("_func", attr.ib(default=cp.dumps(func), type=bytes)) ) self.input_spec = input_spec if name is None: diff --git a/pydra/engine/tests/test_helpers.py b/pydra/engine/tests/test_helpers.py index fec3900cae..ac8e05384d 100644 --- a/pydra/engine/tests/test_helpers.py +++ b/pydra/engine/tests/test_helpers.py @@ -1,6 +1,5 @@ import os import hashlib -import tempfile import typing as ty from pathlib import Path import random @@ -311,57 +310,51 @@ def test_position_sort(pos_args): assert final_args == ["a", "b", "c"] -def test_type_coercion_basic(): +def test_type_coercion_basic(tmpdir): assert TypeCoercer(int)(1.0) == 1 - assert TypeCoercer(int, coercible=[(ty.Any, int)])(1.0) == 1 # coerced - assert TypeCoercer(int, coercible=[(ty.Any, float)])(1.0) == 1.0 # not coerced - assert TypeCoercer(int, not_coercible=[(ty.Any, str)])(1.0) == 1 # coerced - assert TypeCoercer(int, not_coercible=[(float, int)])(1.0) == 1.0 # not coerced + assert TypeCoercer(int, coercible=[(ty.Any, int)])(1.0) == 1 + with pytest.raises(TypeError, match="doesn't match any of the explicit inclusion"): + assert TypeCoercer(int, coercible=[(ty.Any, float)])(1.0) == 1.0 + assert TypeCoercer(int, not_coercible=[(ty.Any, str)])(1.0) == 1 + with pytest.raises(TypeError, match="explicitly excluded"): + assert TypeCoercer(int, not_coercible=[(float, int)])(1.0) == 1.0 - assert ( - TypeCoercer(Path, coercible=[(os.PathLike, os.PathLike)])("/a/path") - == "/a/path" - ) # not coerced - assert TypeCoercer(str, coercible=[(os.PathLike, os.PathLike)])( - Path("/a/path") - ) == Path( - "/a/path" - ) # not coerced + path_coercer = TypeCoercer(Path, coercible=[(os.PathLike, os.PathLike)]) - PathTypes = ty.Union[str, bytes, os.PathLike] + assert path_coercer(Path("/a/path")) == Path("/a/path") + + with pytest.raises(TypeError, match="doesn't match any of the explicit inclusion"): + path_coercer("/a/path") + + PathTypes = ty.Union[str, os.PathLike] assert TypeCoercer(Path, coercible=[(PathTypes, PathTypes)])("/a/path") == Path( "/a/path" - ) # coerced + ) assert ( TypeCoercer(str, coercible=[(PathTypes, PathTypes)])(Path("/a/path")) == "/a/path" - ) # coerced + ) - tmpdir = Path(tempfile.mkdtemp()) a_file = tmpdir / "a-file.txt" Path.touch(a_file) - assert TypeCoercer(File, coercible=[(PathTypes, File)])(a_file) == File( - a_file - ) # coerced - assert TypeCoercer(File, coercible=[(PathTypes, File)])(str(a_file)) == File( - a_file - ) # coerced + file_coercer = TypeCoercer(File, coercible=[(PathTypes, File)]) - assert TypeCoercer(str, coercible=[(PathTypes, File)])(File(a_file)) == File( - a_file - ) # not coerced - assert TypeCoercer(str, coercible=[(PathTypes, File)])(File(a_file)) == File( - a_file - ) # not coerced + assert file_coercer(a_file) == File(a_file) + assert file_coercer(str(a_file)) == File(a_file) + + impotent_str_coercer = TypeCoercer(str, coercible=[(PathTypes, File)]) + + with pytest.raises(TypeError, match="doesn't match any of the explicit inclusion"): + impotent_str_coercer(File(a_file)) assert TypeCoercer(str, coercible=[(PathTypes, PathTypes)])(File(a_file)) == str( a_file - ) # coerced + ) assert TypeCoercer(File, coercible=[(PathTypes, PathTypes)])(str(a_file)) == File( a_file - ) # coerced + ) assert TypeCoercer( list, @@ -369,21 +362,19 @@ def test_type_coercion_basic(): not_coercible=[(str, ty.Sequence)], )((1, 2, 3)) == [1, 2, 3] - assert ( + with pytest.raises(TypeError, match="explicitly excluded"): TypeCoercer( list, coercible=[(ty.Sequence, ty.Sequence)], not_coercible=[(str, ty.Sequence)], )("a-string") - == "a-string" - ) assert TypeCoercer(ty.Union[Path, File, int])(1.0) == 1 assert TypeCoercer(ty.Union[Path, File, bool, int])(1.0) is True + assert TypeCoercer(ty.Sequence)((1, 2, 3)) == (1, 2, 3) -def test_type_coercion_nested(): - tmpdir = Path(tempfile.mkdtemp()) +def test_type_coercion_nested(tmpdir): a_file = tmpdir / "a-file.txt" another_file = tmpdir / "another-file.txt" yet_another_file = tmpdir / "yet-another-file.txt" @@ -417,12 +408,11 @@ def test_type_coercion_nested(): assert TypeCoercer(ty.Tuple[int, int, int])([1.0, 2.0, 3.0]) == (1, 2, 3) assert TypeCoercer(ty.Tuple[int, ...])([1.0, 2.0, 3.0]) == (1, 2, 3) - assert TypeCoercer( - ty.Tuple[int, ...], - not_coercible=[(ty.Sequence, ty.Tuple)], - )( - [1.0, 2.0, 3.0] - ) == [1, 2, 3] + with pytest.raises(TypeError, match="explicitly excluded"): + TypeCoercer( + ty.Tuple[int, ...], + not_coercible=[(ty.Sequence, ty.Tuple)], + )([1.0, 2.0, 3.0]) def test_type_coercion_fail(): @@ -432,5 +422,16 @@ def test_type_coercion_fail(): with pytest.raises(TypeError, match="to any of the union types"): TypeCoercer(ty.Union[Path, File])(1) - with pytest.raises(TypeError, match="Cannot coerce to abstract type"): - TypeCoercer(ty.Sequence) + with pytest.raises(TypeError, match="doesn't match any of the explicit inclusion"): + TypeCoercer(ty.Sequence, coercible=[(ty.Sequence, ty.Sequence)])( + {"a": 1, "b": 2} + ) + + with pytest.raises(TypeError, match="Cannot coerce {'a': 1} into"): + TypeCoercer(ty.Sequence)({"a": 1}) + + with pytest.raises(TypeError, match="as 1 is not iterable"): + TypeCoercer(ty.List[int])(1) + + with pytest.raises(TypeError, match="is not a mapping type"): + TypeCoercer(ty.List[ty.Dict[str, str]])((1, 2, 3))