diff --git a/lib/pavilion/series/series.py b/lib/pavilion/series/series.py index dd62e5af6..a345a3b47 100644 --- a/lib/pavilion/series/series.py +++ b/lib/pavilion/series/series.py @@ -30,7 +30,7 @@ from pavilion.test_run import TestRun from pavilion.types import ID_Pair from pavilion.micro import partition -from pavilion.limiter import TimeLimiter +from pavilion.timing import TimeLimiter from yaml_config import YAMLError, RequiredError from .info import SeriesInfo from .test_set import TestSet diff --git a/lib/pavilion/test_run/test_run.py b/lib/pavilion/test_run/test_run.py index 1f1393815..202793b15 100644 --- a/lib/pavilion/test_run/test_run.py +++ b/lib/pavilion/test_run/test_run.py @@ -13,6 +13,7 @@ import threading import time import uuid +import os from pathlib import Path from typing import TextIO, Union, Dict, Optional import yc_yaml as yaml @@ -38,6 +39,7 @@ from pavilion.test_config.utils import parse_timeout from pavilion.types import ID_Pair from pavilion.micro import get_nested +from pavilion.timing import wait from .test_attrs import TestAttributes @@ -847,10 +849,20 @@ def set_run_complete(self): # run. complete_path = self.path/self.COMPLETE_FN complete_tmp_path = complete_path.with_suffix('.tmp') + with complete_tmp_path.open('w') as run_complete: json.dump( {'complete': time.time()}, run_complete) + + # Ensure that the filesystem is updated before proceeding + run_complete.flush() + os.fsync(run_complete.fileno()) + + # Wait for the file to be written to disk before proceeding + wait(complete_tmp_path.exists, interval=0.2, timeout=2) + + # Finalize the written file complete_tmp_path.rename(complete_path) self._complete = True diff --git a/lib/pavilion/limiter.py b/lib/pavilion/timing.py similarity index 58% rename from lib/pavilion/limiter.py rename to lib/pavilion/timing.py index 76cbdb7e0..6c3329ad5 100644 --- a/lib/pavilion/limiter.py +++ b/lib/pavilion/timing.py @@ -1,7 +1,11 @@ +"""Functions and objects related to controlling the timing of code execution.""" + import time import math from typing import Callable, Tuple, Any +from pavilion.micro import set_default + class TimeLimiter: """Wraps a call to a function and only calls it if the specified @@ -26,3 +30,20 @@ def __call__(self) -> Tuple[bool, Any]: return (True, res) return (False, None) + + +def wait(cond: Callable[[], bool], interval: float, timeout: float = None) -> None: + """Waits until the given condition becomes true before continuing execution, + optionally timing out after the given duration.""" + + timeout = set_default(timeout, math.inf) + + start_time = time.time() + + while time.time() - start_time < timeout: + if cond(): + return + + time.sleep(interval) + + raise TimeoutError(f"Timeout exceeded while waiting for condition {cond} to become true.")