Allow passing configuration through a yaml file (#70)

* add callbacl for yaml config * add env var * add config to all commands * add yaml to deps * Fix mypy error --------- Co-authored-by: Eirini Koutsaniti <[email protected]>
eth-cscs · Nov 20, 2023 · 6b9e4e0 · 6b9e4e0
1 parent 7d65ecb
commit 6b9e4e0
Show file tree

Hide file tree

Showing 2 changed files with 169 additions and 2 deletions.
diff --git a/firecrest/cli/__init__.py b/firecrest/cli/__init__.py
@@ -6,6 +6,7 @@
 #
 import logging
 import typer
+import yaml
 
 import firecrest as fc
 
@@ -97,6 +98,24 @@ def version_callback(value: bool):
         raise typer.Exit()
 
 
+def config_parent_load_callback(ctx: typer.Context, param: typer.CallbackParam, value: str):
+    ctx.default_map = ctx.parent.default_map  # type: ignore
+
+
+def config_callback(ctx: typer.Context, param: typer.CallbackParam, value: str):
+    if value:
+        try:
+            with open(value, 'r') as f:
+                config = yaml.safe_load(f)
+
+            ctx.default_map = ctx.default_map or {}
+            ctx.default_map.update(config)
+        except Exception as ex:
+            raise typer.BadParameter(str(ex))
+
+    return value
+
+
 @app.command(rich_help_panel="Status commands")
 def services(
     name: Optional[str] = typer.Option(
@@ -225,6 +244,11 @@ def tasks(
 
 @app.command(rich_help_panel="Utilities commands")
 def ls(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -268,6 +292,11 @@ def ls(
 
 @app.command(rich_help_panel="Utilities commands")
 def mkdir(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -292,6 +321,11 @@ def mkdir(
 
 @app.command(rich_help_panel="Utilities commands")
 def mv(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -312,6 +346,11 @@ def mv(
 
 @app.command(rich_help_panel="Utilities commands")
 def chmod(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -332,6 +371,11 @@ def chmod(
 
 @app.command(rich_help_panel="Utilities commands")
 def chown(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -356,6 +400,11 @@ def chown(
 
 @app.command(rich_help_panel="Utilities commands")
 def cp(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -376,6 +425,11 @@ def cp(
 
 @app.command(rich_help_panel="Utilities commands")
 def file(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -395,6 +449,11 @@ def file(
 
 @app.command(rich_help_panel="Utilities commands")
 def stat(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -484,6 +543,11 @@ def stat(
 
 @app.command(rich_help_panel="Utilities commands")
 def symlink(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -504,6 +568,11 @@ def symlink(
 
 @app.command(rich_help_panel="Utilities commands")
 def rm(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -532,6 +601,11 @@ def rm(
 
 @app.command(rich_help_panel="Utilities commands")
 def checksum(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -551,6 +625,11 @@ def checksum(
 
 @app.command(rich_help_panel="Utilities commands")
 def head(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -606,6 +685,11 @@ def head(
 
 @app.command(rich_help_panel="Utilities commands")
 def tail(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -661,6 +745,11 @@ def tail(
 
 @app.command(rich_help_panel="Utilities commands")
 def whoami(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         None,
         "-s",
@@ -686,6 +775,11 @@ class TransferType(str, Enum):
 
 @app.command(rich_help_panel="Storage commands")
 def download(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -737,6 +831,11 @@ def download(
 
 @app.command(rich_help_panel="Storage commands")
 def upload(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -799,6 +898,11 @@ def upload(
 
 @app.command(rich_help_panel="Compute commands")
 def submit(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ..., "-s", "--system", help="The name of the system.", envvar="FIRECREST_SYSTEM"
     ),
@@ -829,6 +933,11 @@ def submit(
 
 @submit_template_app.command("mv")
 def submit_mv(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -872,6 +981,11 @@ def submit_mv(
 
 @submit_template_app.command("cp")
 def submit_cp(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -915,6 +1029,11 @@ def submit_cp(
 
 @submit_template_app.command("rsync")
 def submit_rsync(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -958,6 +1077,11 @@ def submit_rsync(
 
 @submit_template_app.command("rm")
 def submit_rm(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ...,
         "-s",
@@ -998,6 +1122,11 @@ def submit_rm(
 
 @app.command(rich_help_panel="Compute commands")
 def poll(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ..., "-s", "--system", help="The name of the system.", envvar="FIRECREST_SYSTEM"
     ),
@@ -1044,6 +1173,11 @@ def poll(
 
 @app.command(rich_help_panel="Compute commands")
 def poll_active(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ..., "-s", "--system", help="The name of the system.", envvar="FIRECREST_SYSTEM"
     ),
@@ -1082,6 +1216,11 @@ def poll_active(
 
 @app.command(rich_help_panel="Compute commands")
 def cancel(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ..., "-s", "--system", help="The name of the system.", envvar="FIRECREST_SYSTEM"
     ),
@@ -1097,6 +1236,11 @@ def cancel(
 
 @reservation_app.command(rich_help_panel="Reservation commands")
 def list(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ..., "-s", "--system", help="The name of the system.", envvar="FIRECREST_SYSTEM"
     ),
@@ -1112,6 +1256,11 @@ def list(
 
 @reservation_app.command(rich_help_panel="Reservation commands")
 def create(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ..., "-s", "--system", help="The name of the system.", envvar="FIRECREST_SYSTEM"
     ),
@@ -1142,6 +1291,11 @@ def create(
 
 @reservation_app.command(rich_help_panel="Reservation commands")
 def update(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ..., "-s", "--system", help="The name of the system.", envvar="FIRECREST_SYSTEM"
     ),
@@ -1172,6 +1326,11 @@ def update(
 
 @reservation_app.command(rich_help_panel="Reservation commands")
 def delete(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
     system: str = typer.Option(
         ..., "-s", "--system", help="The name of the system.", envvar="FIRECREST_SYSTEM"
     ),
@@ -1187,6 +1346,12 @@ def delete(
 
 @app.callback()
 def main(
+    config: Optional[str] = typer.Option(
+        None,
+        envvar="FIRECREST_CONFIG",
+        callback=config_callback,
+        is_eager=True,
+    ),
     version: Optional[bool] = typer.Option(
         None,
         "--version",