Add mpi hosts file to wave forecasts configuration

Enables wwatch3 to be run on different VMs to those where NEMO is run.
This is a step towards running wwatch3 nowcast/forecast runs concurrently with
nowcast-green.

config/nowcast.yaml

@@ -768,6 +768,10 @@ wave forecasts:
   # Template for NEMO hourly results file name
   # **Must be quoted to project {} characters**
   NEMO file template: 'SalishSea_1h_{s_yyyymmdd}_{e_yyyymmdd}_grid_{grid}.nc'
+  # Location on the compute host of the file that contains IP addresses
+  # and MPI slots specifications.
+  # Only required for runs on cloud hosts.
+  mpi hosts file: ${HOME}/mpi_hosts.wwatch3
   # Path to the wwatch3 executables directory
   wwatch3 exe path: /nemoShare/MEOPAR/nowcast-sys/wwatch3-5.16/exe
   # Path to the salishsea command processor executable to use in the run script

nowcast/workers/run_ww3.py

@@ -463,13 +463,14 @@ def _definitions(run_date, run_type, run_dir_path, results_path, config):
     """
     ddmmmyy = run_date.format("DDMMMYY").lower()
     wwatch3_exe_path = config["wave forecasts"]["wwatch3 exe path"]
+    mpi_hosts_file = config["wave forecasts"]["mpi hosts file"]
     salishsea_cmd = config["wave forecasts"]["salishsea cmd"]
     defns = (
         f'RUN_ID="{ddmmmyy}ww3-{run_type}"\n'
         f'WORK_DIR="{run_dir_path}"\n'
         f'RESULTS_DIR="{results_path/ddmmmyy}"\n'
         f'WW3_EXE="{wwatch3_exe_path}"\n'
-        f'MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile ${{HOME}}/mpi_hosts"\n'
+        f'MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile {mpi_hosts_file}"\n'
         f'GATHER="{salishsea_cmd} gather"\n'
     )
     return defns

tests/workers/test_run_ww3.py

@@ -51,6 +51,7 @@ def config(base_config, tmp_path):
                 f"""\
                 wave forecasts:
                     run prep dir: {run_prep_dir}
+                    mpi hosts file: ${{HOME}}/mpi_hosts.wwatch3
                     wwatch3 exe path: wwatch3-5.16/exe
                     salishsea cmd: salishsea
                     results:
@@ -146,6 +147,10 @@ def test_run_prep_dir(self, prod_config):
         run_prep_dir = prod_config["wave forecasts"]["run prep dir"]
         assert run_prep_dir == "/nemoShare/MEOPAR/nowcast-sys/wwatch3-runs"
 
+    def test_mpi_hosts_file(self, prod_config):
+        mpi_hosts_file = prod_config["wave forecasts"]["mpi hosts file"]
+        assert mpi_hosts_file == "${HOME}/mpi_hosts.wwatch3"
+
     def test_wwatch3_exe(self, prod_config):
         wwatch3_exe_path = prod_config["wave forecasts"]["wwatch3 exe path"]
         assert wwatch3_exe_path == "/nemoShare/MEOPAR/nowcast-sys/wwatch3-5.16/exe"
@@ -500,7 +505,7 @@ def test_definitions(self, run_type, config):
         WORK_DIR="wwatch3-runs/tmp_run_dir"
         RESULTS_DIR="wwatch3-{run_type}/29mar17"
         WW3_EXE="wwatch3-5.16/exe"
-        MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile ${{HOME}}/mpi_hosts"
+        MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile ${{HOME}}/mpi_hosts.wwatch3"
         GATHER="salishsea gather"
         """