From 4f0eabe1ef9b16577e3c862b7ff213c86c56a8d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaquier=20Aur=C3=A9lien=20Tristan?= Date: Tue, 10 Oct 2023 12:29:17 +0200 Subject: [PATCH 1/4] integrate ExportHoc to luigi workflow Change-Id: I563dc5ee1142a0563152eef9b8af430bff3cf7a7 --- bluepyemodel/access_point/local.py | 10 + bluepyemodel/export_emodel/utils.py | 15 +- .../tasks/emodel_creation/optimisation.py | 186 +++++++++++++++++- 3 files changed, 209 insertions(+), 2 deletions(-) diff --git a/bluepyemodel/access_point/local.py b/bluepyemodel/access_point/local.py index 89292ea5..74267b6f 100644 --- a/bluepyemodel/access_point/local.py +++ b/bluepyemodel/access_point/local.py @@ -34,6 +34,8 @@ from bluepyemodel.evaluation.evaluator import LEGACY_PRE_PROTOCOLS from bluepyemodel.evaluation.evaluator import PRE_PROTOCOLS from bluepyemodel.evaluation.fitness_calculator_configuration import FitnessCalculatorConfiguration +from bluepyemodel.export_emodel.utils import get_output_path_from_metadata +from bluepyemodel.export_emodel.utils import get_hoc_file_path from bluepyemodel.model.mechanism_configuration import MechanismConfiguration from bluepyemodel.model.neuron_model_configuration import NeuronModelConfiguration from bluepyemodel.tools.mechanisms import get_mechanism_currents @@ -826,3 +828,11 @@ def store_emodels_sonata( ): """Not Implemented""" raise NotImplementedError + + def sonata_exists(self, seed): + """Returns True if the sonata hoc file has been exported""" + output_path = get_output_path_from_metadata( + "export_emodels_sonata", self.emodel_metadata, seed + ) + hoc_file_path = get_hoc_file_path(output_path) + return Path(hoc_file_path).is_file() diff --git a/bluepyemodel/export_emodel/utils.py b/bluepyemodel/export_emodel/utils.py index d9d5550a..2cfc8b09 100644 --- a/bluepyemodel/export_emodel/utils.py +++ b/bluepyemodel/export_emodel/utils.py @@ -22,6 +22,17 @@ logger = logging.getLogger(__name__) +def get_output_path_from_metadata(output_base_dir, emodel_metadata, seed): + """Get the output path from the emodel_metadata. + + Args: + output_base_dir (str): output base directory + emodel_metadata (EModelMetadata): emodel metadata + seed (int): seed + """ + return f"./{output_base_dir}/{emodel_metadata.as_string(seed=seed)}/" + + def get_output_path(emodel, output_dir=None, output_base_dir="export_emodels_hoc"): """Get the output path. @@ -32,7 +43,9 @@ def get_output_path(emodel, output_dir=None, output_base_dir="export_emodels_hoc using also emodel metadata in the path """ if output_dir is None: - output_dir = f"./{output_base_dir}/{emodel.emodel_metadata.as_string(seed=emodel.seed)}/" + output_dir = get_output_path_from_metadata( + output_base_dir, emodel.emodel_metadata, emodel.seed + ) output_path = pathlib.Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) diff --git a/bluepyemodel/tasks/emodel_creation/optimisation.py b/bluepyemodel/tasks/emodel_creation/optimisation.py index 70333c90..86591541 100644 --- a/bluepyemodel/tasks/emodel_creation/optimisation.py +++ b/bluepyemodel/tasks/emodel_creation/optimisation.py @@ -658,6 +658,191 @@ def output(self): ) +class StoreHocTarget(WorkflowTarget): + """Check if the hoc files have been stored.""" + + def __init__(self, emodel, etype, ttype, mtype, species, brain_region, iteration_tag, seed): + """Constructor. + + Args: + emodel (str): name of the emodel. Has to match the name of the emodel + under which the configuration data are stored. + ttype (str): name of the ttype. + seed (int): seed used in the optimisation. + """ + super().__init__( + emodel=emodel, + etype=etype, + ttype=ttype, + mtype=mtype, + species=species, + brain_region=brain_region, + iteration_tag=iteration_tag, + ) + + self.seed = seed + + def exists(self): + """Check if the hoc is stored for all given seeds.""" + batch_size = self.access_point.pipeline_settings.optimisation_batch_size + checked_for_all_seeds = [ + self.access_point.sonata_exists(seed=seed) + for seed in range(self.seed, self.seed + batch_size) + ] + return all(checked_for_all_seeds) + + +class ExportHoc(WorkflowTaskRequiringMechanisms, IPyParallelTask): + """Luigi wrapper for export_emodels_sonata. + + Parameters: + seed (int): seed used in the optimisation. + graceful_killer (multiprocessing.Event): event triggered when USR1 signal is received. + Has to use multiprocessing event for communicating between processes + when there is more than 1 luigi worker. Skip task if set. + """ + + seed = luigi.IntParameter(default=1) + graceful_killer = multiprocessing.Event() + + def requires(self): + """ """ + + compile_mechanisms = self.access_point.pipeline_settings.compile_mechanisms + + to_run = [ + StoreBestModels( + emodel=self.emodel, + etype=self.etype, + ttype=self.ttype, + mtype=self.mtype, + species=self.species, + brain_region=self.brain_region, + iteration_tag=self.iteration_tag, + seed=self.seed, + ) + ] + if compile_mechanisms: + to_run.append( + CompileMechanisms( + emodel=self.emodel, + etype=self.etype, + ttype=self.ttype, + mtype=self.mtype, + species=self.species, + brain_region=self.brain_region, + iteration_tag=self.iteration_tag, + ) + ) + + return to_run + + @WorkflowTask.check_mettypes + def run(self): + """Prepare self.args, then call bbp-workflow's IPyParallelTask's run().""" + # set self.batch_size here to easily handle it in the argparse argument passing + self.batch_size = self.access_point.pipeline_settings.optimisation_batch_size + attrs = [ + "backend", + "emodel", + "etype", + "ttype", + "mtype", + "species", + "brain_region", + "iteration_tag", + "seed", + "batch_size", + ] + + self.prepare_args_for_remote_script(attrs) + + super().run() + + def remote_script(self): + """Catch arguments from parsing, and run validation.""" + # -- imports -- # + import argparse + import json + + from bluepyemodel import access_point + from bluepyemodel.tools.multiprocessing import get_mapper + from bluepyemodel.export_emodel.export_emodel import export_emodels_sonata + + # -- parsing -- # + parser = argparse.ArgumentParser() + parser.add_argument("--backend", default=None, type=str) + parser.add_argument("--api_from_config", default="local", type=str) + parser.add_argument( + "--api_args_from_config", + default=", ".join( + ( + '{"emodel_dir": None', + '"recipes_path": None', + '"final_path": None', + '"legacy_dir_structure": None', + '"extract_config": None}', + ) + ), + type=json.loads, + ) + parser.add_argument("--emodel", default=None, type=str) + parser.add_argument("--etype", default=None, type=str) + parser.add_argument("--ttype", default=None, type=str) + parser.add_argument("--mtype", default=None, type=str) + parser.add_argument("--species", default=None, type=str) + parser.add_argument("--brain_region", default=None, type=str) + parser.add_argument("--iteration_tag", default=None, type=str) + parser.add_argument("--ipyparallel_profile", default=None, type=str) + parser.add_argument("--seed", default=None, type=int) + parser.add_argument("--batch_size", default=None, type=int) + + args = parser.parse_args() + + # -- run validation -- # + mapper = get_mapper(args.backend, ipyparallel_profile=args.ipyparallel_profile) + access_pt = access_point.get_access_point( + access_point=args.api_from_config, + emodel=args.emodel, + etype=args.etype, + ttype=args.ttype, + mtype=args.mtype, + species=args.species, + brain_region=args.brain_region, + iteration_tag=args.iteration_tag, + **args.api_args_from_config, + ) + + export_emodels_sonata( + access_pt, + only_validated=False, + only_best=False, + seeds=list(range(args.seed, args.seed + args.batch_size)), + map_fucntion=mapper, + ) + if args.api_from_config == "nexus": + access_pt.store_emodels_sonata( + only_validated=False, + only_best=False, + seeds=list(range(args.seed, args.seed + args.batch_size)), + map_function=mapper, + ) + + + def output(self): + """ """ + return StoreHocTarget( + emodel=self.emodel, + etype=self.etype, + ttype=self.ttype, + mtype=self.mtype, + species=self.species, + brain_region=self.brain_region, + iteration_tag=self.iteration_tag, + seed=self.seed, + ) + + class EModelCreationTarget(WorkflowTarget): """Check if the the model is validated for any seed.""" @@ -720,7 +905,6 @@ class EModelCreation(WorkflowTask): seed = luigi.IntParameter(default=1) graceful_killer = multiprocessing.Event() - @staticmethod def check_mettypes(func): """Decorator to check mtype, etype and ttype presence on nexus""" From f12e89bdd4aa1b89e58f0557e1fcdb19758df215 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaquier=20Aur=C3=A9lien=20Tristan?= Date: Tue, 10 Oct 2023 12:31:20 +0200 Subject: [PATCH 2/4] remove stale extract_config from workflow Change-Id: Iab4e5ec6a9f26b9a7d3b7bf51bdb21c3ff413634 --- bluepyemodel/tasks/config.py | 4 ---- bluepyemodel/tasks/emodel_creation/optimisation.py | 9 +++------ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/bluepyemodel/tasks/config.py b/bluepyemodel/tasks/config.py index 1c826af1..f267e579 100644 --- a/bluepyemodel/tasks/config.py +++ b/bluepyemodel/tasks/config.py @@ -29,7 +29,6 @@ class EmodelAPIConfig(luigi.Config): recipes_path = luigi.OptionalParameter(default=None) final_path = luigi.OptionalParameter(default="./final.json") legacy_dir_structure = luigi.BoolParameter(default=False) - extract_config = luigi.OptionalParameter(default=None) # nexus parameters forge_path = luigi.OptionalParameter(default=None) @@ -45,15 +44,12 @@ def __init__(self, *args, **kwargs): if self.api == "local": if self.recipes_path is None: raise ValueError("recipes_path cannot be None when api is set to 'local'") - if self.extract_config is None: - raise ValueError("extract_config cannot be None when api is set to 'local'") self.api_args = { "emodel_dir": self.emodel_dir, "recipes_path": self.recipes_path, "final_path": self.final_path, "legacy_dir_structure": self.legacy_dir_structure, - "extract_config": self.extract_config, } if self.api == "nexus": diff --git a/bluepyemodel/tasks/emodel_creation/optimisation.py b/bluepyemodel/tasks/emodel_creation/optimisation.py index 86591541..024a125f 100644 --- a/bluepyemodel/tasks/emodel_creation/optimisation.py +++ b/bluepyemodel/tasks/emodel_creation/optimisation.py @@ -295,8 +295,7 @@ def remote_script(self): '{"emodel_dir": None', '"recipes_path": None', '"final_path": None', - '"legacy_dir_structure": None', - '"extract_config": None}', + '"legacy_dir_structure": None}', ) ), type=json.loads, @@ -611,8 +610,7 @@ def remote_script(self): '{"emodel_dir": None', '"recipes_path": None', '"final_path": None', - '"legacy_dir_structure": None', - '"extract_config": None}', + '"legacy_dir_structure": None}', ) ), type=json.loads, @@ -780,8 +778,7 @@ def remote_script(self): '{"emodel_dir": None', '"recipes_path": None', '"final_path": None', - '"legacy_dir_structure": None', - '"extract_config": None}', + '"legacy_dir_structure": None}', ) ), type=json.loads, From c37a06a2f60732b4892fd68e414226cbd1415a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaquier=20Aur=C3=A9lien=20Tristan?= Date: Tue, 10 Oct 2023 13:26:18 +0200 Subject: [PATCH 3/4] fix path check in evaluator creation Change-Id: Ib21c4994ab65adc615979f4d8d12237ddaad3bac --- bluepyemodel/evaluation/evaluation.py | 5 ++++- bluepyemodel/export_emodel/utils.py | 2 +- bluepyemodel/tasks/emodel_creation/optimisation.py | 3 +-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bluepyemodel/evaluation/evaluation.py b/bluepyemodel/evaluation/evaluation.py index e4ecd9c4..4be4885c 100644 --- a/bluepyemodel/evaluation/evaluation.py +++ b/bluepyemodel/evaluation/evaluation.py @@ -269,7 +269,10 @@ def get_evaluator_from_access_point( mechanisms_directory = access_point.get_mechanisms_directory() if isinstance(access_point, LocalAccessPoint): - if Path.cwd() != access_point.emodel_dir and access_point.emodel_metadata.iteration: + if ( + Path.cwd() != access_point.emodel_dir.resolve() + and access_point.emodel_metadata.iteration + ): delete_compiled_mechanisms() if not (access_point.emodel_dir / "x86_64" / "special").is_file(): compile_mechs_in_emodel_dir(mechanisms_directory) diff --git a/bluepyemodel/export_emodel/utils.py b/bluepyemodel/export_emodel/utils.py index 2cfc8b09..0db08ddd 100644 --- a/bluepyemodel/export_emodel/utils.py +++ b/bluepyemodel/export_emodel/utils.py @@ -24,7 +24,7 @@ def get_output_path_from_metadata(output_base_dir, emodel_metadata, seed): """Get the output path from the emodel_metadata. - + Args: output_base_dir (str): output base directory emodel_metadata (EModelMetadata): emodel metadata diff --git a/bluepyemodel/tasks/emodel_creation/optimisation.py b/bluepyemodel/tasks/emodel_creation/optimisation.py index 024a125f..112178d7 100644 --- a/bluepyemodel/tasks/emodel_creation/optimisation.py +++ b/bluepyemodel/tasks/emodel_creation/optimisation.py @@ -815,7 +815,7 @@ def remote_script(self): only_validated=False, only_best=False, seeds=list(range(args.seed, args.seed + args.batch_size)), - map_fucntion=mapper, + map_function=mapper, ) if args.api_from_config == "nexus": access_pt.store_emodels_sonata( @@ -825,7 +825,6 @@ def remote_script(self): map_function=mapper, ) - def output(self): """ """ return StoreHocTarget( From 897c7aaefeb737b1646c5d0aadb4701d8af78c98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaquier=20Aur=C3=A9lien=20Tristan?= Date: Tue, 10 Oct 2023 13:55:23 +0200 Subject: [PATCH 4/4] lint fix Change-Id: Ic57bba4a80cbad49a9fc2d7a55b3afd93b4b8af2 --- bluepyemodel/access_point/local.py | 2 +- bluepyemodel/tasks/emodel_creation/optimisation.py | 13 ++++++++++--- bluepyemodel/tools/mechanisms.py | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/bluepyemodel/access_point/local.py b/bluepyemodel/access_point/local.py index 74267b6f..992d7fd5 100644 --- a/bluepyemodel/access_point/local.py +++ b/bluepyemodel/access_point/local.py @@ -34,8 +34,8 @@ from bluepyemodel.evaluation.evaluator import LEGACY_PRE_PROTOCOLS from bluepyemodel.evaluation.evaluator import PRE_PROTOCOLS from bluepyemodel.evaluation.fitness_calculator_configuration import FitnessCalculatorConfiguration -from bluepyemodel.export_emodel.utils import get_output_path_from_metadata from bluepyemodel.export_emodel.utils import get_hoc_file_path +from bluepyemodel.export_emodel.utils import get_output_path_from_metadata from bluepyemodel.model.mechanism_configuration import MechanismConfiguration from bluepyemodel.model.neuron_model_configuration import NeuronModelConfiguration from bluepyemodel.tools.mechanisms import get_mechanism_currents diff --git a/bluepyemodel/tasks/emodel_creation/optimisation.py b/bluepyemodel/tasks/emodel_creation/optimisation.py index 112178d7..ba4a3eed 100644 --- a/bluepyemodel/tasks/emodel_creation/optimisation.py +++ b/bluepyemodel/tasks/emodel_creation/optimisation.py @@ -703,6 +703,13 @@ class ExportHoc(WorkflowTaskRequiringMechanisms, IPyParallelTask): seed = luigi.IntParameter(default=1) graceful_killer = multiprocessing.Event() + def __init__(self, *args, **kwargs): + """ """ + super().__init__(*args, **kwargs) + + # set self.batch_size here to easily handle it in the argparse argument passing + self.batch_size = self.access_point.pipeline_settings.optimisation_batch_size + def requires(self): """ """ @@ -738,8 +745,6 @@ def requires(self): @WorkflowTask.check_mettypes def run(self): """Prepare self.args, then call bbp-workflow's IPyParallelTask's run().""" - # set self.batch_size here to easily handle it in the argparse argument passing - self.batch_size = self.access_point.pipeline_settings.optimisation_batch_size attrs = [ "backend", "emodel", @@ -764,8 +769,8 @@ def remote_script(self): import json from bluepyemodel import access_point - from bluepyemodel.tools.multiprocessing import get_mapper from bluepyemodel.export_emodel.export_emodel import export_emodels_sonata + from bluepyemodel.tools.multiprocessing import get_mapper # -- parsing -- # parser = argparse.ArgumentParser() @@ -898,6 +903,8 @@ class EModelCreation(WorkflowTask): when there is more than 1 luigi worker. Exit loop if set. """ + # pylint: disable=no-self-argument, not-callable + seed = luigi.IntParameter(default=1) graceful_killer = multiprocessing.Event() diff --git a/bluepyemodel/tools/mechanisms.py b/bluepyemodel/tools/mechanisms.py index 7360d339..82a7fee2 100644 --- a/bluepyemodel/tools/mechanisms.py +++ b/bluepyemodel/tools/mechanisms.py @@ -88,7 +88,7 @@ def compile_mechs_in_emodel_dir(mechanisms_directory): mechanisms_dir (Path): path to the directory containing the mod files to compile. """ - # pylint: disable=broad-exception-caught + # pylint: disable=broad-except cwd = os.getcwd() try: