Fix race condition in set_run_complete

lib/pavilion/series/series.py

@@ -30,7 +30,7 @@
 from pavilion.test_run import TestRun
 from pavilion.types import ID_Pair
 from pavilion.micro import partition
-from pavilion.limiter import TimeLimiter
+from pavilion.timing import TimeLimiter
 from yaml_config import YAMLError, RequiredError
 from .info import SeriesInfo
 from .test_set import TestSet

lib/pavilion/test_run/test_run.py

@@ -13,6 +13,7 @@
 import threading
 import time
 import uuid
+import os
 from pathlib import Path
 from typing import TextIO, Union, Dict, Optional
 import yc_yaml as yaml
@@ -38,6 +39,7 @@
 from pavilion.test_config.utils import parse_timeout
 from pavilion.types import ID_Pair
 from pavilion.micro import get_nested
+from pavilion.timing import wait
 from .test_attrs import TestAttributes
 
 
@@ -847,10 +849,20 @@ def set_run_complete(self):
         # run.
         complete_path = self.path/self.COMPLETE_FN
         complete_tmp_path = complete_path.with_suffix('.tmp')
+
         with complete_tmp_path.open('w') as run_complete:
             json.dump(
                 {'complete': time.time()},
                 run_complete)
+
+            # Ensure that the filesystem is updated before proceeding
+            run_complete.flush()
+            os.fsync(run_complete.fileno())
+
+        # Wait for the file to be written to disk before proceeding
+        wait(complete_tmp_path.exists, interval=0.2, timeout=2)
+
+        # Finalize the written file
         complete_tmp_path.rename(complete_path)
 
         self._complete = True

lib/pavilion/limiter.py → lib/pavilion/timing.py

@@ -1,7 +1,11 @@
+"""Functions and objects related to controlling the timing of code execution."""
+
 import time
 import math
 from typing import Callable, Tuple, Any
 
+from pavilion.micro import set_default
+
 
 class TimeLimiter:
     """Wraps a call to a function and only calls it if the specified
@@ -26,3 +30,20 @@ def __call__(self) -> Tuple[bool, Any]:
             return (True, res)
 
         return (False, None)
+
+
+def wait(cond: Callable[[], bool], interval: float, timeout: float = None) -> None:
+    """Waits until the given condition becomes true before continuing execution,
+    optionally timing out after the given duration."""
+
+    timeout = set_default(timeout, math.inf)
+
+    start_time = time.time()
+
+    while time.time() - start_time < timeout:
+        if cond():
+            return
+
+        time.sleep(interval)
+
+    raise TimeoutError(f"Timeout exceeded while waiting for condition {cond} to become true.")