Skip to content

Commit

Permalink
fix: Fix issue with logging in parallel
Browse files Browse the repository at this point in the history
  • Loading branch information
ChristianMichelsen committed Aug 18, 2022
1 parent 7a35b3a commit 9e065aa
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 15 deletions.
11 changes: 11 additions & 0 deletions src/metaDMG/fit/serial.py
Original file line number Diff line number Diff line change
Expand Up @@ -546,3 +546,14 @@ def run_single_config(

logger.info("Finished.")
return df_mismatches, df_fit_results, df_results


def run_single_config_count_errors(
config: Config,
) -> int:

try:
run_single_config(config)
return 0
except:
return 1
19 changes: 4 additions & 15 deletions src/metaDMG/fit/workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from logger_tt import logger

from metaDMG.fit.serial import run_single_config
from metaDMG.fit.serial import run_single_config_count_errors
from metaDMG.utils import Configs


Expand All @@ -17,29 +17,18 @@ def run_workflow(configs: Configs) -> int:

N_errors = 0

print(current_process().name)

if parallel_samples == 1 or len(configs) == 1:
logger.info(f"Running in serial (1 core)")
for config in configs:
try:
dfs = run_single_config(config)
except:
N_errors += 1
# df_mismatches, df_fit_results, df_results = dfs
N_errors += run_single_config_count_errors(config)

else:
logger.info(f"Running with {parallel_samples} processes in parallel")
configs.check_number_of_jobs()

with Pool(max_workers=parallel_samples) as pool:
# for dfs in pool.imap_unordered(serial.run_single_config, configs):
try:
for dfs in pool.map(run_single_config, configs):
pass
# df_mismatches, df_fit_results, df_results = dfs
except:
N_errors += 1
for n_error in pool.map(run_single_config_count_errors, configs):
N_errors += n_error

# set back the name of the process to the original name
current_process().name = "MainProcess"
Expand Down

0 comments on commit 9e065aa

Please sign in to comment.