From ffb7513293a3c6f895c40925b886a17933e5d796 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 9 Sep 2021 15:30:24 +0200 Subject: [PATCH] cleanup requests calculations --- cwltool/builder.py | 2 +- cwltool/context.py | 4 ++-- cwltool/executors.py | 27 +++++++++------------------ cwltool/expression.py | 2 +- cwltool/process.py | 19 +++++++------------ 5 files changed, 20 insertions(+), 34 deletions(-) diff --git a/cwltool/builder.py b/cwltool/builder.py index 2c2ec1891..1136a8b33 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -164,7 +164,7 @@ def __init__( names: Names, requirements: List[CWLObjectType], hints: List[CWLObjectType], - resources: Dict[str, Union[int, float, str]], + resources: Dict[str, Union[int, float]], mutation_manager: Optional[MutationManager], formatgraph: Optional[Graph], make_fs_access: Type[StdFsAccess], diff --git a/cwltool/context.py b/cwltool/context.py index daf6aa867..1919804e5 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -87,8 +87,8 @@ class RuntimeContext(ContextBase): def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: """Initialize the RuntimeContext from the kwargs.""" select_resources_callable = Callable[ # pylint: disable=unused-variable - [Dict[str, Union[int, float, str]], RuntimeContext], - Dict[str, Union[int, float, str]], + [Dict[str, Union[int, float]], RuntimeContext], + Dict[str, Union[int, float]], ] self.user_space_docker_cmd = "" # type: Optional[str] self.secret_store = None # type: Optional[SecretStore] diff --git a/cwltool/executors.py b/cwltool/executors.py index c04c78b0c..567fb9615 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -283,35 +283,26 @@ def __init__(self) -> None: self.allocated_cores = float(0) def select_resources( - self, request, runtime_context - ): # pylint: disable=unused-argument - # type: (Dict[str, Union[int, float, str]], RuntimeContext) -> Dict[str, Union[int, float, str]] + self, request: Dict[str, Union[int, float]], runtime_context: RuntimeContext + ) -> Dict[str, Union[int, float]]: # pylint: disable=unused-argument """Naïve check for available cpu cores and memory.""" - result = {} # type: Dict[str, Union[int, float, str]] + result: Dict[str, Union[int, float]] = {} maxrsc = {"cores": self.max_cores, "ram": self.max_ram} for rsc in ("cores", "ram"): rsc_min = request[rsc + "Min"] - if not isinstance(rsc_min, str) and rsc_min > maxrsc[rsc]: + if rsc_min > maxrsc[rsc]: raise WorkflowException( - "Requested at least %d %s but only %d available" - % (rsc_min, rsc, maxrsc[rsc]) + f"Requested at least {rsc_min} {rsc} but only " + f"{maxrsc[rsc]} available" ) rsc_max = request[rsc + "Max"] - if not isinstance(rsc_max, str) and rsc_max < maxrsc[rsc]: + if rsc_max < maxrsc[rsc]: result[rsc] = math.ceil(rsc_max) else: result[rsc] = maxrsc[rsc] - result["tmpdirSize"] = ( - math.ceil(request["tmpdirMin"]) - if not isinstance(request["tmpdirMin"], str) - else request["tmpdirMin"] - ) - result["outdirSize"] = ( - math.ceil(request["outdirMin"]) - if not isinstance(request["outdirMin"], str) - else request["outdirMin"] - ) + result["tmpdirSize"] = math.ceil(request["tmpdirMin"]) + result["outdirSize"] = math.ceil(request["outdirMin"]) return result diff --git a/cwltool/expression.py b/cwltool/expression.py index 8057886b6..8474dfe02 100644 --- a/cwltool/expression.py +++ b/cwltool/expression.py @@ -359,7 +359,7 @@ def do_eval( requirements: List[CWLObjectType], outdir: Optional[str], tmpdir: Optional[str], - resources: Dict[str, Union[float, int, str]], + resources: Dict[str, Union[float, int]], context: Optional[CWLOutputType] = None, timeout: float = default_timeout, force_docker_pull: bool = False, diff --git a/cwltool/process.py b/cwltool/process.py index 9079e29b0..58becf757 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -966,7 +966,7 @@ def inc(d): # type: (List[int]) -> None def evalResources( self, builder: Builder, runtimeContext: RuntimeContext - ) -> Dict[str, Union[int, float, str]]: + ) -> Dict[str, Union[int, float]]: resourceReq, _ = self.get_requirement("ResourceRequirement") if resourceReq is None: resourceReq = {} @@ -1010,19 +1010,14 @@ def evalResources( request[a + "Min"] = mn request[a + "Max"] = cast(Union[int, float], mx) + request_evaluated = cast(Dict[str, Union[int, float]], request) if runtimeContext.select_resources is not None: - return runtimeContext.select_resources(request, runtimeContext) + return runtimeContext.select_resources(request_evaluated, runtimeContext) return { - "cores": request["coresMin"], - "ram": math.ceil(request["ramMin"]) - if not isinstance(request["ramMin"], str) - else request["ramMin"], - "tmpdirSize": math.ceil(request["tmpdirMin"]) - if not isinstance(request["tmpdirMin"], str) - else request["tmpdirMin"], - "outdirSize": math.ceil(request["outdirMin"]) - if not isinstance(request["outdirMin"], str) - else request["outdirMin"], + "cores": request_evaluated["coresMin"], + "ram": math.ceil(request_evaluated["ramMin"]), + "tmpdirSize": math.ceil(request_evaluated["tmpdirMin"]), + "outdirSize": math.ceil(request_evaluated["outdirMin"]), } def validate_hints(