cylc vr (#5229)

vr: validate and reinstall a workflow

* Add option for validating workflow source against options set in an installed workflow in order to ensure that reinstallation will not result in failure.
* Add a combined command for validating-reinstalling-(restarting|reloading).

CHANGES.md

@@ -20,10 +20,16 @@ ones in. -->
 
 ### Enhancements
 
+[#5229](https://github.com/cylc/cylc-flow/pull/5229) -
+- Added a single command to validate a previously run workflow against changes
+  to its source and reinstall a workflow.
+- Allows Cylc commands (including validate, list, view, config, and graph) to load template variables
+  configured by `cylc install` and `cylc play`.
+
 [#5184](https://github.com/cylc/cylc-flow/pull/5184) - scan for active
 runs of the same workflow at install time.
 
-[#5094](https://github.com/cylc/cylc-flow/pull/5094) - Added a single
+[#5121](https://github.com/cylc/cylc-flow/pull/5121) - Added a single
 command to validate, install and play a workflow.
 
 [#5032](https://github.com/cylc/cylc-flow/pull/5032) - set a default limit of

cylc/flow/option_parsers.py

@@ -80,9 +80,16 @@ def __init__(self, argslist, sources=None, useif=None, **kwargs):
             self.kwargs.update({kwarg: value})
 
     def __eq__(self, other):
-        """Args and Kwargs, but not other props equal."""
+        """Args and Kwargs, but not other props equal.
+
+        (Also make an exception for kwargs['help'] to allow lists of sources
+        prepended to 'help' to be passed through.)
+        """
         return (
-            self.kwargs == other.kwargs
+            (
+                {k: v for k, v in self.kwargs.items() if k != 'help'}
+                == {k: v for k, v in other.kwargs.items() if k != 'help'}
+            )
             and self.args == other.args
         )
 
@@ -117,6 +124,22 @@ def _update_sources(self, other):
     dest="icp"
 )
 
+AGAINST_SOURCE_OPTION = OptionSettings(
+    ['--against-source'],
+    help=(
+        "Load the workflow configuration from the source directory it was"
+        " installed from using any options (e.g. template variables) which"
+        " have been set in the installation."
+        " This is useful if you want to see how changes made to the workflow"
+        " source would affect the installation if reinstalled."
+        " Note if this option is used the provided workflow must have been"
+        " installed by `cylc install`."
+    ),
+    dest='against_source',
+    action='store_true',
+    default=False
+)
+
 
 icp_option = Option(
     *ICP_OPTION.args, **ICP_OPTION.kwargs)  # type: ignore[arg-type]
@@ -713,9 +736,10 @@ def combine_options_pair(first_list, second_list):
                 and first & second
             ):
                 # if any of the args are different:
+
                 if first.args == second.args:
-                    # if none of the arg names are different.
-                    raise Exception(f'Clashing Options \n{first}\n{second}')
+                    raise Exception(
+                        f'Clashing Options \n{first.args}\n{second.args}')
                 else:
                     first_args = first - second
                     second.args = second - first

cylc/flow/parsec/fileparse.py

@@ -31,6 +31,7 @@
 """
 
 import os
+from optparse import Values
 from pathlib import Path
 import re
 import sys
@@ -45,6 +46,9 @@
 from cylc.flow.parsec.OrderedDict import OrderedDictWithDefaults
 from cylc.flow.parsec.include import inline
 from cylc.flow.parsec.util import itemstr
+from cylc.flow.templatevars import get_template_vars_from_db
+from cylc.flow.workflow_files import (
+    get_workflow_source_dir, check_flow_file)
 
 
 # heading/sections can contain commas (namespace name lists) and any
@@ -343,6 +347,43 @@ def merge_template_vars(
         return native_tvars
 
 
+def _prepend_old_templatevars(fpath: str, template_vars: t.Dict) -> t.Dict:
+    """If the fpath is in a rundir, extract template variables from database.
+
+    Args:
+        fpath: filepath of workflow config file.
+        template_vars: Template vars to prepend old tvars to.
+    """
+    rundir = Path(fpath).parent
+    old_tvars = get_template_vars_from_db(rundir)
+    old_tvars.update(template_vars)
+    template_vars = old_tvars
+    return template_vars
+
+
+def _get_fpath_for_source(fpath: str, opts: "Values") -> str:
+    """If `--against-source` is set and a sourcedir can be found, return
+    sourcedir.
+    Else return fpath unchanged.
+
+    Raises:
+        ParsecError: If validate against source is set and this is not
+        an installed workflow then we don't want to continue.
+    """
+    if getattr(opts, 'against_source', False):
+        thispath = Path(fpath)
+        source_dir = get_workflow_source_dir(thispath.parent)[0]
+        if source_dir:
+            retpath = check_flow_file(source_dir)
+            return str(retpath)
+        else:
+            raise ParsecError(
+                f'Cannot validate {thispath.parent} against source: '
+                'it is not an installed workflow.')
+    else:
+        return fpath
+
+
 def read_and_proc(
     fpath: str,
     template_vars: t.Optional[t.Dict[str, t.Any]] = None,
@@ -355,7 +396,9 @@ def read_and_proc(
     Jinja2 processing must be done before concatenation - it could be
     used to generate continuation lines.
     """
-
+    template_vars = template_vars if template_vars is not None else {}
+    template_vars = _prepend_old_templatevars(fpath, template_vars)
+    fpath = _get_fpath_for_source(fpath, opts)
     fdir = os.path.dirname(fpath)
 
     # Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).

cylc/flow/scripts/config.py

@@ -49,14 +49,17 @@
   $ cylc config --initial-cycle-point=now myflow
 """
 
+import asyncio
+
 import os.path
 from typing import List, Optional, TYPE_CHECKING
 
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.config import WorkflowConfig
-from cylc.flow.id_cli import parse_id
+from cylc.flow.id_cli import parse_id_async
 from cylc.flow.exceptions import InputError
 from cylc.flow.option_parsers import (
+    AGAINST_SOURCE_OPTION,
     WORKFLOW_ID_OR_PATH_ARG_DOC,
     CylcOptionParser as COP,
     icp_option,
@@ -128,6 +131,9 @@ def get_option_parser() -> COP:
         action='store_true', default=False, dest='print_platforms'
     )
 
+    parser.add_option(
+        *AGAINST_SOURCE_OPTION.args, **AGAINST_SOURCE_OPTION.kwargs)
+
     parser.add_cylc_rose_options()
 
     return parser
@@ -149,6 +155,14 @@ def main(
     options: 'Values',
     *ids,
 ) -> None:
+    asyncio.run(_main(parser, options, *ids))
+
+
+async def _main(
+    parser: COP,
+    options: 'Values',
+    *ids,
+) -> None:
 
     if options.print_platform_names and options.print_platforms:
         options.print_platform_names = False
@@ -178,7 +192,7 @@ def main(
         )
         return
 
-    workflow_id, _, flow_file = parse_id(
+    workflow_id, _, flow_file = await parse_id_async(
         *ids,
         src=True,
         constraint='workflows',

cylc/flow/scripts/cylc.py

@@ -202,7 +202,8 @@ def get_version(long=False):
     'shutdown': 'stop',
     'task-message': 'message',
     'unhold': 'release',
-    'validate-install-play': 'vip'
+    'validate-install-play': 'vip',
+    'validate-reinstall': 'vr',
 }
 
 

cylc/flow/scripts/graph.py

@@ -35,6 +35,7 @@
     $ cylc graph one -o 'one.svg'
 """
 
+import asyncio
 from difflib import unified_diff
 from shutil import which
 from subprocess import Popen, PIPE
@@ -45,8 +46,9 @@
 from cylc.flow.config import WorkflowConfig
 from cylc.flow.exceptions import InputError, CylcError
 from cylc.flow.id import Tokens
-from cylc.flow.id_cli import parse_id
+from cylc.flow.id_cli import parse_id_async
 from cylc.flow.option_parsers import (
+    AGAINST_SOURCE_OPTION,
     WORKFLOW_ID_OR_PATH_ARG_DOC,
     CylcOptionParser as COP,
     icp_option,
@@ -108,9 +110,10 @@ def get_nodes_and_edges(
     workflow_id,
     start,
     stop,
+    flow_file,
 ) -> Tuple[List[Node], List[Edge]]:
     """Return graph sorted nodes and edges."""
-    config = get_config(workflow_id, opts)
+    config = get_config(workflow_id, opts, flow_file)
     if opts.namespaces:
         nodes, edges = _get_inheritance_nodes_and_edges(config)
     else:
@@ -194,13 +197,8 @@ def _get_inheritance_nodes_and_edges(
     return sorted(nodes), sorted(edges)
 
 
-def get_config(workflow_id: str, opts: 'Values') -> WorkflowConfig:
+def get_config(workflow_id: str, opts: 'Values', flow_file) -> WorkflowConfig:
     """Return a WorkflowConfig object for the provided reg / path."""
-    workflow_id, _, flow_file = parse_id(
-        workflow_id,
-        src=True,
-        constraint='workflows',
-    )
     template_vars = get_template_vars(opts)
     return WorkflowConfig(
         workflow_id, flow_file, opts, template_vars=template_vars
@@ -334,7 +332,7 @@ def open_image(filename):
         img.show()
 
 
-def graph_render(opts, workflow_id, start, stop) -> int:
+def graph_render(opts, workflow_id, start, stop, flow_file) -> int:
     """Render the workflow graph to the specified format.
 
     Graph is rendered to the specified format. The Graphviz "dot" format
@@ -349,6 +347,7 @@ def graph_render(opts, workflow_id, start, stop) -> int:
         workflow_id,
         start,
         stop,
+        flow_file
     )
 
     # format the graph in graphviz-dot format
@@ -382,28 +381,42 @@ def graph_render(opts, workflow_id, start, stop) -> int:
     return 0
 
 
-def graph_reference(opts, workflow_id, start, stop, write=print) -> int:
+def graph_reference(
+    opts, workflow_id, start, stop, flow_file, write=print,
+) -> int:
     """Format the workflow graph using the cylc reference format."""
     # get nodes and edges
     nodes, edges = get_nodes_and_edges(
         opts,
         workflow_id,
         start,
         stop,
+        flow_file
     )
     for line in format_cylc_reference(opts, nodes, edges):
         write(line)
 
     return 0
 
 
-def graph_diff(opts, workflow_a, workflow_b, start, stop) -> int:
+async def graph_diff(
+    opts, workflow_a, workflow_b, start, stop, flow_file
+) -> int:
     """Difference the workflow graphs using the cylc reference format."""
+
+    workflow_b, _, flow_file_b = await parse_id_async(
+        workflow_b,
+        src=True,
+        constraint='workflows',
+    )
+
     # load graphs
     graph_a: List[str] = []
     graph_b: List[str] = []
-    graph_reference(opts, workflow_a, start, stop, write=graph_a.append),
-    graph_reference(opts, workflow_b, start, stop, write=graph_b.append),
+    graph_reference(
+        opts, workflow_a, start, stop, flow_file, write=graph_a.append),
+    graph_reference(
+        opts, workflow_b, start, stop, flow_file_b, write=graph_b.append),
 
     # compare graphs
     diff_lines = list(
@@ -494,6 +507,9 @@ def get_option_parser() -> COP:
         action='store',
     )
 
+    parser.add_option(
+        *AGAINST_SOURCE_OPTION.args, **AGAINST_SOURCE_OPTION.kwargs)
+
     parser.add_cylc_rose_options()
 
     return parser
@@ -507,20 +523,34 @@ def main(
     start: Optional[str] = None,
     stop: Optional[str] = None
 ) -> None:
+    result = asyncio.run(_main(parser, opts, workflow_id, start, stop))
+    sys.exit(result)
+
+
+async def _main(
+    parser: COP,
+    opts: 'Values',
+    workflow_id: str,
+    start: Optional[str] = None,
+    stop: Optional[str] = None
+) -> int:
     """Implement ``cylc graph``."""
     if opts.grouping and opts.namespaces:
         raise InputError('Cannot combine --group and --namespaces.')
     if opts.cycles and opts.namespaces:
         raise InputError('Cannot combine --cycles and --namespaces.')
 
+    workflow_id, _, flow_file = await parse_id_async(
+        workflow_id,
+        src=True,
+        constraint='workflows',
+    )
+
     if opts.diff:
-        sys.exit(
-            graph_diff(opts, workflow_id, opts.diff, start, stop)
-        )
+        return await graph_diff(
+            opts, workflow_id, opts.diff, start, stop, flow_file)
     if opts.reference:
-        sys.exit(
-            graph_reference(opts, workflow_id, start, stop)
-        )
-    sys.exit(
-        graph_render(opts, workflow_id, start, stop)
-    )
+        return graph_reference(
+            opts, workflow_id, start, stop, flow_file)
+
+    return graph_render(opts, workflow_id, start, stop, flow_file)