Merge pull request #60 from BlueBrain/improve-export

Add EModelScript class

bluepyemodel/access_point/access_point.py

@@ -168,6 +168,33 @@ def get_emodel_names(self):
             dict: keys are emodel names with seed, values are names without seed.
         """
 
+    def store_hocs(
+        self,
+        only_validated=False,
+        only_best=True,
+        seeds=None,
+        map_function=map,
+        new_emodel_name=None,
+        description=None,
+        output_base_dir="export_emodels_hoc",
+    ):
+        """Store the hoc files"""
+
+    def store_emodels_hoc(
+        self, only_validated=False, only_best=True, seeds=None, new_emodel_name=None
+    ):
+        """Store hoc file produced by export_hoc"""
+
+    def store_emodels_sonata(
+        self,
+        only_validated=False,
+        only_best=True,
+        seeds=None,
+        map_function=map,
+        new_emodel_name=None,
+    ):
+        """Store hoc file produced by export_sonata"""
+
     def optimisation_state(self, seed=None, continue_opt=False):
         """Return the state of the optimisation.
 

bluepyemodel/access_point/local.py

@@ -789,3 +789,30 @@ def add_entry_recipes(
 
         with recipes_path.open("w") as f:
             json.dump(recipes, f, indent=2)
+
+    def store_hocs(
+        self,
+        only_validated=False,
+        only_best=True,
+        seeds=None,
+        map_function=map,
+        new_emodel_name=None,
+        description=None,
+        output_base_dir="export_emodels_hoc",
+    ):
+        raise NotImplementedError
+
+    def store_emodels_hoc(
+        self, only_validated=False, only_best=True, seeds=None, new_emodel_name=None
+    ):
+        raise NotImplementedError
+
+    def store_emodels_sonata(
+        self,
+        only_validated=False,
+        only_best=True,
+        seeds=None,
+        map_function=map,
+        new_emodel_name=None,
+    ):
+        raise NotImplementedError

bluepyemodel/emodel_pipeline/emodel_script.py

@@ -0,0 +1,51 @@
+"""EModelScript class"""
+
+"""
+Copyright 2023, EPFL/Blue Brain Project
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+class EModelScript:
+
+    """Contains an emodel hoc file path."""
+
+    def __init__(self, hoc_file_path=None, seed=None, workflow_id=None):
+        """Init
+
+        Args:
+            hoc_file_path (str): path to the hoc file of the emodel
+            seed (str): seed used during optimisation for this emodel.
+            workflow_id (str): id of the emodel workflow resource
+        """
+        self.hoc_file_path = hoc_file_path
+        self.seed = seed
+        self.workflow_id = workflow_id
+
+    def get_related_nexus_ids(self):
+        return {
+            "generation": {
+                "type": "Generation",
+                "activity": {
+                    "type": "Activity",
+                    "followedWorkflow": {"type": "EModelWorkflow", "id": self.workflow_id},
+                },
+            }
+        }
+
+    def as_dict(self):
+        return {
+            "nexus_distributions": [self.hoc_file_path],
+            "seed": self.seed,
+        }

bluepyemodel/export_emodel/export_emodel.py

@@ -25,6 +25,9 @@
 
 from bluepyemodel.evaluation.evaluation import compute_responses
 from bluepyemodel.evaluation.evaluation import get_evaluator_from_access_point
+from bluepyemodel.export_emodel.utils import get_hoc_file_path
+from bluepyemodel.export_emodel.utils import get_output_path
+from bluepyemodel.export_emodel.utils import select_emodels
 
 logger = logging.getLogger(__name__)
 
@@ -102,14 +105,8 @@ def _export_model_sonata(cell_model, emodel, output_dir=None, new_emodel_name=No
     if new_emodel_name is not None:
         emodel.emodel_metadata.emodel = new_emodel_name
 
-    if output_dir is None:
-        output_dir = (
-            f"./export_emodels_sonata/{emodel.emodel_metadata.as_string(seed=emodel.seed)}/"
-        )
-    output_path = pathlib.Path(output_dir)
-    output_path.mkdir(parents=True, exist_ok=True)
-
-    hoc_file_path = str(output_path / "model.hoc")
+    output_path = get_output_path(emodel, output_dir, output_base_dir="export_emodels_sonata")
+    hoc_file_path = get_hoc_file_path(output_path)
     node_file_path = str(output_path / "nodes.h5")
     morphology_path = str(output_path / pathlib.Path(cell_model.morphology.morphology_path).name)
 
@@ -134,33 +131,6 @@ def _export_model_sonata(cell_model, emodel, output_dir=None, new_emodel_name=No
     )
 
 
-def select_emodels(emodel_name, emodels, only_validated=False, only_best=True, seeds=None):
-    if not emodels:
-        logger.warning("In export_emodels_nexus, no emodel for %s", emodel_name)
-        return []
-
-    if only_best:
-        emodels = [sorted(emodels, key=lambda x: x.fitness)[0]]
-
-    if seeds:
-        emodels = [e for e in emodels if e.seed in seeds]
-        if not emodels:
-            logger.warning(
-                "In export_emodels_nexus, no emodel for %s and seeds %s", emodel_name, seeds
-            )
-            return []
-
-    if only_validated:
-        emodels = [e for e in emodels if e.passed_validation]
-        if not emodels:
-            logger.warning(
-                "In export_emodels_nexus, no emodel for %s that passed validation", emodel_name
-            )
-            return []
-
-    return emodels
-
-
 def export_emodels_sonata(
     access_point,
     only_validated=False,
@@ -195,6 +165,9 @@ def export_emodels_sonata(
         seeds=seeds,
     )
     if not emodels:
+        logger.warning(
+            "No emodels were selected in export_emodels_sonata. Stopping sonata export here."
+        )
         return
 
     cell_model = cell_evaluator.cell_model
@@ -212,12 +185,8 @@ def _export_emodel_hoc(cell_model, mo, output_dir=None, new_emodel_name=None):
     if new_emodel_name is not None:
         mo.emodel_metadata.emodel = new_emodel_name
 
-    if output_dir is None:
-        output_dir = f"./export_emodels_hoc/{mo.emodel_metadata.as_string(seed=mo.seed)}/"
-    output_path = pathlib.Path(output_dir)
-    output_path.mkdir(parents=True, exist_ok=True)
-
-    hoc_file_path = str(output_path / "model.hoc")
+    output_path = get_output_path(mo, output_dir, output_base_dir="export_emodels_hoc")
+    hoc_file_path = get_hoc_file_path(output_path)
     morphology_path = str(output_path / pathlib.Path(cell_model.morphology.morphology_path).name)
 
     # Copy the morphology
@@ -238,35 +207,27 @@ def export_emodels_hoc(
     only_validated=False,
     only_best=True,
     seeds=None,
-    map_function=map,
     new_emodel_name=None,
 ):
     """Export a set of emodels to a set of folder named after them. Each folder will contain a hoc
-    version of the model.
-
-    WARNING: this function is not compatible with multiprocessing."""
+    version of the model."""
 
     cell_evaluator = get_evaluator_from_access_point(
         access_point, include_validation_protocols=True
     )
 
-    emodels = compute_responses(
-        access_point,
-        cell_evaluator,
-        map_function,
-        seeds=seeds,
-        preselect_for_validation=False,
-        store_responses=False,
-    )
+    emodels = access_point.get_emodels()
 
     emodels = select_emodels(
         access_point.emodel_metadata.emodel,
         emodels,
         only_validated=only_validated,
         only_best=only_best,
         seeds=seeds,
+        iteration=access_point.emodel_metadata.iteration,
     )
     if not emodels:
+        logger.warning("No emodels were selected in export_emodels_hoc. Stopping hoc export here.")
         return
 
     cell_model = cell_evaluator.cell_model

bluepyemodel/export_emodel/utils.py

@@ -0,0 +1,77 @@
+"""Export the emodels in the SONATA format"""
+
+"""
+Copyright 2023, EPFL/Blue Brain Project
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import logging
+import pathlib
+
+logger = logging.getLogger(__name__)
+
+
+def get_output_path(emodel, output_dir=None, output_base_dir="export_emodels_hoc"):
+    """Get the output path.
+
+    Args:
+        emodel (EModel): emodel
+        output_dir (str): output directory
+        output_base_dir (str): if output_dir is None, export to this directory instead,
+            using also emodel metadata in the path
+    """
+    if output_dir is None:
+        output_dir = f"./{output_base_dir}/{emodel.emodel_metadata.as_string(seed=emodel.seed)}/"
+    output_path = pathlib.Path(output_dir)
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    return output_path
+
+
+def get_hoc_file_path(output_path):
+    """Get the hoc file path."""
+    output_path = pathlib.Path(output_path)
+    return str(output_path / "model.hoc")
+
+
+def select_emodels(
+    emodel_name, emodels, only_validated=False, only_best=True, seeds=None, iteration=None
+):
+    if not emodels:
+        logger.warning("In export_emodels_nexus, no emodel for %s", emodel_name)
+        return []
+
+    if iteration:
+        emodels = [model for model in emodels if model.emodel_metadata.iteration == iteration]
+
+    if only_best:
+        emodels = [sorted(emodels, key=lambda x: x.fitness)[0]]
+
+    if seeds:
+        emodels = [e for e in emodels if e.seed in seeds]
+        if not emodels:
+            logger.warning(
+                "In export_emodels_nexus, no emodel for %s and seeds %s", emodel_name, seeds
+            )
+            return []
+
+    if only_validated:
+        emodels = [e for e in emodels if e.passed_validation]
+        if not emodels:
+            logger.warning(
+                "In export_emodels_nexus, no emodel for %s that passed validation", emodel_name
+            )
+            return []
+
+    return emodels