From 27595f50142962da75c1a5c269ca7d638a3845df Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Tue, 21 Nov 2023 16:14:48 +0100 Subject: [PATCH] SacessOptimizer: Fix occasional deadlocks (#1204) Previously, it could happen that SacessOptimizer runs into a deadlock due to a copied thread. Fixed here. --- pypesto/optimize/ess/sacess.py | 6 +++++- test/optimize/test_optimize.py | 11 +++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pypesto/optimize/ess/sacess.py b/pypesto/optimize/ess/sacess.py index 586dc0d69..cc2e5eee8 100644 --- a/pypesto/optimize/ess/sacess.py +++ b/pypesto/optimize/ess/sacess.py @@ -142,7 +142,6 @@ def minimize( logging_thread = logging.handlers.QueueListener( multiprocessing.Queue(-1), logging_handler ) - logging_thread.start() # shared memory manager to handle shared state # (simulates the sacess manager process) @@ -183,6 +182,11 @@ def minimize( ] for p in worker_processes: p.start() + + # start logging thread only AFTER starting the worker processes + # to prevent deadlocks + logging_thread.start() + # wait for finish for p in worker_processes: p.join() diff --git a/test/optimize/test_optimize.py b/test/optimize/test_optimize.py index 4fe36f2cc..28d6a670b 100644 --- a/test/optimize/test_optimize.py +++ b/test/optimize/test_optimize.py @@ -441,6 +441,9 @@ def test_history_beats_optimizer(): ) +@pytest.mark.filterwarnings( + "ignore:Passing `startpoint_method` directly is deprecated.*:DeprecationWarning" +) @pytest.mark.parametrize("ess_type", ["ess", "cess", "sacess"]) @pytest.mark.parametrize("local_optimizer", [None, optimize.FidesOptimizer()]) @pytest.mark.flaky(reruns=3) @@ -485,8 +488,12 @@ def test_ess(problem, local_optimizer, ess_type, request): ): # Not pickleable - incompatible with CESS pytest.skip() - # SACESS with 4 processes - ess_init_args = get_default_ess_options(num_workers=4, dim=problem.dim) + # SACESS with 12 processes + # We use a higher number than reasonable to be more likely to trigger + # any potential race conditions (gh-1204) + ess_init_args = get_default_ess_options( + num_workers=12, dim=problem.dim + ) for x in ess_init_args: x['local_optimizer'] = local_optimizer ess = SacessOptimizer(