From 96f8cc4a65162304016d251f5a62686cc62f7d22 Mon Sep 17 00:00:00 2001
From: Doug Latornell <dlatornell@eoas.ubc.ca>
Date: Mon, 6 Nov 2023 12:26:36 -0800
Subject: [PATCH] Add mpi hosts file to wave forecasts configuration

Enables wwatch3 to be run on different VMs to those where NEMO is run.
This is a step towards running wwatch3 nowcast/forecast runs concurrently with
nowcast-green.
---
 config/nowcast.yaml           | 4 ++++
 nowcast/workers/run_ww3.py    | 3 ++-
 tests/workers/test_run_ww3.py | 7 ++++++-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/config/nowcast.yaml b/config/nowcast.yaml
index 49b596fa..a1a6f524 100644
--- a/config/nowcast.yaml
+++ b/config/nowcast.yaml
@@ -768,6 +768,10 @@ wave forecasts:
   # Template for NEMO hourly results file name
   # **Must be quoted to project {} characters**
   NEMO file template: 'SalishSea_1h_{s_yyyymmdd}_{e_yyyymmdd}_grid_{grid}.nc'
+  # Location on the compute host of the file that contains IP addresses
+  # and MPI slots specifications.
+  # Only required for runs on cloud hosts.
+  mpi hosts file: ${HOME}/mpi_hosts.wwatch3
   # Path to the wwatch3 executables directory
   wwatch3 exe path: /nemoShare/MEOPAR/nowcast-sys/wwatch3-5.16/exe
   # Path to the salishsea command processor executable to use in the run script
diff --git a/nowcast/workers/run_ww3.py b/nowcast/workers/run_ww3.py
index e62e465a..e807686f 100644
--- a/nowcast/workers/run_ww3.py
+++ b/nowcast/workers/run_ww3.py
@@ -463,13 +463,14 @@ def _definitions(run_date, run_type, run_dir_path, results_path, config):
     """
     ddmmmyy = run_date.format("DDMMMYY").lower()
     wwatch3_exe_path = config["wave forecasts"]["wwatch3 exe path"]
+    mpi_hosts_file = config["wave forecasts"]["mpi hosts file"]
     salishsea_cmd = config["wave forecasts"]["salishsea cmd"]
     defns = (
         f'RUN_ID="{ddmmmyy}ww3-{run_type}"\n'
         f'WORK_DIR="{run_dir_path}"\n'
         f'RESULTS_DIR="{results_path/ddmmmyy}"\n'
         f'WW3_EXE="{wwatch3_exe_path}"\n'
-        f'MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile ${{HOME}}/mpi_hosts"\n'
+        f'MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile {mpi_hosts_file}"\n'
         f'GATHER="{salishsea_cmd} gather"\n'
     )
     return defns
diff --git a/tests/workers/test_run_ww3.py b/tests/workers/test_run_ww3.py
index 455daf4f..6c47315e 100644
--- a/tests/workers/test_run_ww3.py
+++ b/tests/workers/test_run_ww3.py
@@ -51,6 +51,7 @@ def config(base_config, tmp_path):
                 f"""\
                 wave forecasts:
                     run prep dir: {run_prep_dir}
+                    mpi hosts file: ${{HOME}}/mpi_hosts.wwatch3
                     wwatch3 exe path: wwatch3-5.16/exe
                     salishsea cmd: salishsea
                     results:
@@ -146,6 +147,10 @@ def test_run_prep_dir(self, prod_config):
         run_prep_dir = prod_config["wave forecasts"]["run prep dir"]
         assert run_prep_dir == "/nemoShare/MEOPAR/nowcast-sys/wwatch3-runs"
 
+    def test_mpi_hosts_file(self, prod_config):
+        mpi_hosts_file = prod_config["wave forecasts"]["mpi hosts file"]
+        assert mpi_hosts_file == "${HOME}/mpi_hosts.wwatch3"
+
     def test_wwatch3_exe(self, prod_config):
         wwatch3_exe_path = prod_config["wave forecasts"]["wwatch3 exe path"]
         assert wwatch3_exe_path == "/nemoShare/MEOPAR/nowcast-sys/wwatch3-5.16/exe"
@@ -500,7 +505,7 @@ def test_definitions(self, run_type, config):
         WORK_DIR="wwatch3-runs/tmp_run_dir"
         RESULTS_DIR="wwatch3-{run_type}/29mar17"
         WW3_EXE="wwatch3-5.16/exe"
-        MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile ${{HOME}}/mpi_hosts"
+        MPIRUN="mpirun --mca btl ^openib --mca orte_tmpdir_base /dev/shm --hostfile ${{HOME}}/mpi_hosts.wwatch3"
         GATHER="salishsea gather"
         """