Automatically detect UNSET parameters and interpret them as None in clickify_parameters #345
Conversation
…cy in _validate_list/tuple in clickify parameters
| @@ -187,7 +193,10 @@ def clickify_parameters(schemas: Union[str, Dict[str, Any]], | |||
| if io is outputs and not (schema.is_file_type and not schema.implicit): | |||
| continue | |||
|
|
|||
| policies = OmegaConf.merge(default_policies, schema.policies) | |||
| # None should not take precedence in the merge | |||
There was a problem hiding this comment.
This seems to break something in breifast. I get
### running stack_time_cube
# Traceback (most recent call last):
# File "<string>", line 7, in <module>
# File "/home/bester/software/breifast/breifast-tron/breifast/cubes.py", line 29, in <module>
# from breifast.main import cli, schemas
# File "/home/bester/software/breifast/breifast-tron/breifast/main.py", line 26, in <module>
# from . import breifast, catalogs, cubes, image_tools
# File "/home/bester/software/breifast/breifast-tron/breifast/image_tools.py", line 23, in <module>
# def measure_source_spectrum(images: List\[str],
# File "/home/bester/software/breifast/breifast-tron/breifast/main.py", line 23, in <lambda>
# clickify_parameters(schemas.cabs.get(f'breifast.{schema_name}'))(func)
# File "/home/bester/software/stimela2/scabha/schema_utils.py", line 197, in clickify_parameters
# merge_policies = {k: v for k, v in schema.policies.items() if v is not None}
# AttributeError: 'ParameterPolicies' object has no attribute 'items'
2024-11-07 16:09:41 STIMELA.tron.stack-cds ERROR: step 'tron.stack-cds' has failed, aborting the recipe
There was a problem hiding this comment.
I can get around this by casting the ParameterPolicies object to a dict using dataclasses.asdict but again not sure if this will have detrimental consequences elsewhere
There was a problem hiding this comment.
Yep, that works, but I would rather you use if isinstance(schema.policies, ParameterPolicies) do the asdict thing else do the other thing rather than catching exceptions. But that;s because I generally don't like using exceptions for managing "normal" code flow. Exceptions should be reserved for true errors as much as possible -- this also makes debugging easier (when you enable breakpoints on exception raised).
There was a problem hiding this comment.
Yeah, this was a bit naughty. Not sure why I did it like that
stimela/backends/kube/run_kube.py
Outdated
| @@ -145,10 +145,15 @@ def dprint(level, *args, **kw): | |||
|
|
|||
| # apply pod type specifications | |||
| if kube.dask_cluster.worker_pod: | |||
| print(kube.dask_cluster.worker_pod) | |||
There was a problem hiding this comment.
@landmanbester These debug print statements ought to be replaced with dprint(1, content). See examples in same file. That way you can enable the printing by setting debug: verbose: 1 in the backend options.
There was a problem hiding this comment.
Cool, didn't know that one. Thanks
stimela/backends/kube/run_kube.py
Outdated
| if kube.dask_cluster.scheduler_pod: | ||
| print(kube.dask_cluster.scheduler_pod) |
| @@ -187,7 +193,10 @@ def clickify_parameters(schemas: Union[str, Dict[str, Any]], | |||
| if io is outputs and not (schema.is_file_type and not schema.implicit): | |||
| continue | |||
|
|
|||
| policies = OmegaConf.merge(default_policies, schema.policies) | |||
| # None should not take precedence in the merge | |||
There was a problem hiding this comment.
Yep, that works, but I would rather you use if isinstance(schema.policies, ParameterPolicies) do the asdict thing else do the other thing rather than catching exceptions. But that;s because I generally don't like using exceptions for managing "normal" code flow. Exceptions should be reserved for true errors as much as possible -- this also makes debugging easier (when you enable breakpoints on exception raised).
I sed to have a manual check for this. I think it works as expected now but not sure if it may have unintended consequences. Could you take a look at this please @o-smirnov?