remote: ensure all remote commands use a platform config

* Remote interfaces (SSH & rsync) now all require a platform
  object for configuration purposes (e.g. "ssh command").
* Convenience interfaces added for remote calls to Cylc servers
  which use the localhost platform configuration. These
  are now used for:
  * `cylc play` command re-invocation on a Cylc server.
  * Evaluation of host selection rankings (via `cylc psutil`).
  * The detect old contact file check, which tests whether
    a workflow is still running on the server recored in the
    contact file.

cylc/flow/cfgspec/globalcfg.py

@@ -1614,18 +1614,26 @@ def default_for(
         with Conf('localhost', meta=Platform, desc='''
             A default platform for running jobs on the the scheduler host.
 
-            .. attention::
+            This platform configures the host on which
+            :term:`schedulers <scheduler>` run. By default this is the
+            host where ``cylc play`` is run, however, we often configure
+            Cylc to start schedulers on dedicated hosts by configuring
+            :cylc:conf:`global.cylc[scheduler][run hosts]available`.
 
-               It is common practice to start Cylc schedulers on dedicated
-               hosts, in which case **"localhost" is the scheduler host and
-               not necessarily where you ran "cylc play"**.
+            This platform affects connections made to the scheduler host and
+            any jobs run on it.
 
             .. versionadded:: 8.0.0
         '''):
             Conf('hosts', VDR.V_STRING_LIST, ['localhost'], desc='''
                 List of hosts for the localhost platform. You are unlikely to
                 need to change this.
 
+                The scheduler hosts are configured by
+                :cylc:conf:`global.cylc[scheduler][run hosts]available`.
+                See :ref:`Submitting Workflows To a Pool Of Hosts` for
+                more information.
+
                 .. seealso::
 
                    :cylc:conf:`global.cylc[platforms][<platform name>]hosts`

cylc/flow/host_select.py

@@ -29,7 +29,7 @@
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.exceptions import CylcError, HostSelectException
 from cylc.flow.hostuserutil import get_fqdn_by_host, is_remote_host
-from cylc.flow.remote import _remote_cylc_cmd, run_cmd
+from cylc.flow.remote import run_cmd, cylc_server_cmd
 from cylc.flow.terminal import parse_dirty_json
 
 
@@ -533,7 +533,7 @@ def _get_metrics(hosts, metrics, data=None):
     }
     for host in hosts:
         if is_remote_host(host):
-            proc_map[host] = _remote_cylc_cmd(cmd, host=host, **kwargs)
+            proc_map[host] = cylc_server_cmd(cmd, host=host, **kwargs)
         else:
             proc_map[host] = run_cmd(['cylc'] + cmd, **kwargs)
 

cylc/flow/network/ssh_client.py

@@ -23,7 +23,7 @@
 from cylc.flow.exceptions import ClientError, ClientTimeout
 from cylc.flow.network.client_factory import CommsMeth
 from cylc.flow.network import get_location
-from cylc.flow.remote import _remote_cylc_cmd
+from cylc.flow.remote import remote_cylc_cmd
 from cylc.flow.workflow_files import load_contact_file, ContactFileFields
 
 
@@ -60,15 +60,20 @@ async def async_request(self, command, args=None, timeout=None):
         try:
             async with ascyncto(timeout):
                 cmd, ssh_cmd, login_sh, cylc_path, msg = self.prepare_command(
-                    command, args, timeout)
-                proc = _remote_cylc_cmd(
+                    command, args, timeout
+                )
+                platform = {
+                    'ssh command': ssh_cmd,
+                    'cylc path': cylc_path,
+                    'use login shell': login_sh,
+                }
+                proc = remote_cylc_cmd(
                     cmd,
+                    platform,
                     host=self.host,
                     stdin_str=msg,
-                    ssh_cmd=ssh_cmd,
-                    remote_cylc_path=cylc_path,
-                    ssh_login_shell=login_sh,
-                    capture_process=True)
+                    capture_process=True
+                )
                 while True:
                     if proc.poll() is not None:
                         break

cylc/flow/remote.py

@@ -31,9 +31,10 @@
 from typing import Any, Dict, List, Tuple
 
 import cylc.flow.flags
-from cylc.flow import __version__ as CYLC_VERSION
+from cylc.flow import __version__ as CYLC_VERSION, LOG
 from cylc.flow.option_parsers import verbosity_to_opts
 from cylc.flow.platforms import get_platform, get_host_from_platform
+from cylc.flow.util import format_cmd
 
 
 def get_proc_ancestors():
@@ -121,6 +122,7 @@ def run_cmd(
         stdin = read
 
     try:
+        LOG.debug(f'running command:\n$ {format_cmd(command)}')
         proc = Popen(  # nosec
             command,
             stdin=stdin,
@@ -226,83 +228,51 @@ def construct_rsync_over_ssh_cmd(
 
 
 def construct_ssh_cmd(
-    raw_cmd, platform, host, **kwargs
-):
-    """Build an SSH command for execution on a remote platform.
-
-    Constructs the SSH command according to the platform configuration.
-
-    See _construct_ssh_cmd for argument documentation.
-    """
-    return _construct_ssh_cmd(
-        raw_cmd,
-        host=host,
-        ssh_cmd=platform['ssh command'],
-        remote_cylc_path=platform['cylc path'],
-        ssh_login_shell=platform['use login shell'],
-        **kwargs
-    )
-
-
-def _construct_ssh_cmd(
-        raw_cmd,
-        host=None,
-        forward_x11=False,
-        stdin=False,
-        ssh_cmd=None,
-        ssh_login_shell=None,
-        remote_cylc_path=None,
-        set_UTC=False,
-        set_verbosity=False,
-        timeout=None
+    raw_cmd,
+    platform,
+    host,
+    forward_x11=False,
+    stdin=False,
+    set_UTC=False,
+    set_verbosity=False,
+    timeout=None,
 ):
     """Build an SSH command for execution on a remote platform hosts.
 
     Arguments:
         raw_cmd (list):
             primitive command to run remotely.
+        platform (dict):
+            The Cylc job "platform" to run the command on. This is used
+            to determine the settings used e.g. "ssh command".
         host (string):
             remote host name. Use 'localhost' if not specified.
         forward_x11 (boolean):
             If True, use 'ssh -Y' to enable X11 forwarding, else just 'ssh'.
         stdin:
             If None, the `-n` option will be added to the SSH command line.
-        ssh_cmd (string):
-            ssh command to use: If unset defaults to localhost ssh cmd.
-        ssh_login_shell (boolean):
-            If True, launch remote command with `bash -l -c 'exec "$0" "$@"'`.
-        remote_cylc_path (string):
-            Path containing the `cylc` executable.
-            This is required if the remote executable is not in $PATH.
         set_UTC (boolean):
             If True, check UTC mode and specify if set to True (non-default).
         set_verbosity (boolean):
             If True apply -q, -v opts to match cylc.flow.flags.verbosity.
         timeout (str):
             String for bash timeout command.
 
-    Return:
+    Returns:
         list - A list containing a chosen command including all arguments and
         options necessary to directly execute the bare command on a given host
         via ssh.
+
     """
     # If ssh cmd isn't given use the default from localhost settings.
-    if ssh_cmd is None:
-        command = shlex.split(get_platform()['ssh command'])
-    else:
-        command = shlex.split(ssh_cmd)
+    command = shlex.split(platform['ssh command'])
 
     if forward_x11:
         command.append('-Y')
     if stdin is None:
         command.append('-n')
 
-    user_at_host = ''
-    if host:
-        user_at_host += host
-    else:
-        user_at_host += 'localhost'
-    command.append(user_at_host)
+    command.append(host)
 
     # Pass CYLC_VERSION and optionally, CYLC_CONF_PATH & CYLC_UTC through.
     command += ['env', quote(r'CYLC_VERSION=%s' % CYLC_VERSION)]
@@ -323,8 +293,7 @@ def _construct_ssh_cmd(
         command.append(quote(r'TZ=UTC'))
 
     # Use bash -l?
-    if ssh_login_shell is None:
-        ssh_login_shell = get_platform()['use login shell']
+    ssh_login_shell = platform['use login shell']
     if ssh_login_shell:
         # A login shell will always source /etc/profile and the user's bash
         # profile file. To avoid having to quote the entire remote command
@@ -335,14 +304,11 @@ def _construct_ssh_cmd(
         command += ['timeout', timeout]
 
     # 'cylc' on the remote host
-    if not remote_cylc_path:
-        remote_cylc_path = get_platform()['cylc path']
-
+    remote_cylc_path = platform['cylc path']
     if remote_cylc_path:
         cylc_cmd = str(Path(remote_cylc_path) / 'cylc')
     else:
         cylc_cmd = 'cylc'
-
     command.append(cylc_cmd)
 
     # Insert core raw command after ssh, but before its own, command options.
@@ -354,56 +320,92 @@ def _construct_ssh_cmd(
     return command
 
 
-def remote_cylc_cmd(cmd, platform, bad_hosts=None, **kwargs):
-    """Execute a Cylc command on a remote platform.
+def construct_cylc_server_ssh_cmd(
+    cmd,
+    host,
+    **kwargs,
+):
+    """Concenience function to building SSH commands for remote Cylc servers.
+
+    Build an SSH command that connects to the specified host using the
+    localhost platform config.
+
+    * To run commands on job platforms use construct_ssh_cmd.
+    * Use this interface to connect to:
+      * Cylc servers (i.e. `[scheduler][run hosts]available`).
+      * The host `cylc play` was run on, use this interface.
 
-    Uses the platform configuration to construct the command.
+    This assumes the host you are connecting to shares the $HOME filesystem
+    with the localhost platform.
 
-    See _construct_ssh_cmd for argument documentation.
+    For arguments and returns see construct_ssh_cmd.
     """
-    return _remote_cylc_cmd(
+    return construct_ssh_cmd(
         cmd,
-        host=get_host_from_platform(platform, bad_hosts=bad_hosts),
-        ssh_cmd=platform['ssh command'],
-        remote_cylc_path=platform['cylc path'],
-        ssh_login_shell=platform['use login shell'],
-        **kwargs
+        get_platform(),  # use localhost settings
+        host,
+        **kwargs,
     )
 
 
-def _remote_cylc_cmd(
-        cmd,
-        host=None,
-        stdin=None,
-        stdin_str=None,
-        ssh_login_shell=None,
-        ssh_cmd=None,
-        remote_cylc_path=None,
-        capture_process=False,
-        manage=False
+def remote_cylc_cmd(
+    cmd,
+    platform,
+    bad_hosts=None,
+    host=None,
+    stdin=None,
+    stdin_str=None,
+    ssh_login_shell=None,
+    ssh_cmd=None,
+    remote_cylc_path=None,
+    capture_process=False,
+    manage=False
 ):
     """Execute a Cylc command on a remote platform.
 
-    See run_cmd and _construct_ssh_cmd for argument documentation.
-
-    Returns:
-        subprocess.Popen or int - If capture_process=True, return the Popen
-        object if created successfully. Otherwise, return the exit code of the
-        remote command.
+    Uses the provided platform configuration to construct the command.
 
+    For arguments and returns see construct_ssh_cmd and run_cmd.
     """
+    if not host:
+        # no host selected => perform host selection from platform config
+        host = get_host_from_platform(platform, bad_hosts=bad_hosts)
+
     return run_cmd(
-        _construct_ssh_cmd(
+        construct_ssh_cmd(
             cmd,
+            platform,
             host=host,
             stdin=True if stdin_str else stdin,
-            ssh_login_shell=ssh_login_shell,
-            ssh_cmd=ssh_cmd,
-            remote_cylc_path=remote_cylc_path
         ),
         stdin=stdin,
         stdin_str=stdin_str,
         capture_process=capture_process,
         capture_status=True,
         manage=manage
     )
+
+
+def cylc_server_cmd(cmd, host=None, **kwargs):
+    """Convenience function for running commands on remote Cylc servers.
+
+    Executes a Cylc command on the specified host using localhost platform
+    config.
+
+    * To run commands on job platforms use remote_cylc_cmd.
+    * Use this interface to run commands on:
+      * Cylc servers (i.e. `[scheduler][run hosts]available`).
+      * The host `cylc play` was run on, use this interface.
+
+    Runs a command via SSH using the configuration for the localhost platform.
+    This assumes the host you are connecting to shares the $HOME filesystem
+    with the localhost platform.
+
+    For arguments and returns see construct_ssh_cmd and run_cmd.
+    """
+    return remote_cylc_cmd(
+        cmd,
+        get_platform(),  # use localhost settings
+        host=host,
+        **kwargs,
+    )

cylc/flow/scheduler_cli.py

@@ -41,7 +41,7 @@
     icp_option,
 )
 from cylc.flow.pathutil import get_workflow_run_scheduler_log_path
-from cylc.flow.remote import _remote_cylc_cmd
+from cylc.flow.remote import cylc_server_cmd
 from cylc.flow.scheduler import Scheduler, SchedulerError
 from cylc.flow.scripts.common import cylc_header
 from cylc.flow.workflow_files import (
@@ -393,7 +393,7 @@ def _distribute(host, workflow_id_raw, workflow_id):
         cmd.append("--host=localhost")
 
         # Re-invoke the command
-        _remote_cylc_cmd(cmd, host=host)
+        cylc_server_cmd(cmd, host=host)
         sys.exit(0)