Add multiprocessing for ceptr (#437)

Docs/sphinx/Ceptr.rst

@@ -1,7 +1,7 @@
 .. highlight:: rst
 
 .. _sec:ceptr:
-               
+
 CEPTR: Chemistry Evaluation for Pele Through Recasting
 ======================================================
 
@@ -19,10 +19,12 @@ To install CEPTR dependencies::
   $ cd ${PELE_PHYSICS_HOME}/Support/ceptr
   $ poetry update
 
-
 Usage
 -----
 
+Generating for a single chemistry
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
 There are three ways to use CEPTR to generate C++ mechanism files for a given chemistry
 
 1. Using CEPTR directly::
@@ -39,6 +41,14 @@ There are three ways to use CEPTR to generate C++ mechanism files for a given ch
      $ bash ${PELE_PHYSICS_HOME}/Support/Mechanism/Models/converter.sh -f ./LiDryer/mechanism.yaml
 
 
+Batched generation
+^^^^^^^^^^^^^^^^^^
+
+.. note::
+
+   If you are using batched generation as outlined here, it will automatically use multiprocessing to generate the files in parallel using all CPUs detected on the machine. If you want to change that you can pass the optional argument ``-n NPCU``, wheren ``NCPU`` is an integer indicating the number of processes you want to use.
+
+
 For non-reduced chemistries, CEPTR can take a file with a list of ``mechanism.yaml`` files to convert::
 
   $ cd ${PELE_PHYSICS_HOME}/Support/ceptr
@@ -99,5 +109,3 @@ The full list of options is::
 For a detailed description of these options and a further information on the way QSS mechanism are treated in `CEPTR` the reader may consult :ref:`the QSS section <sec_qss>`.
 
 See Tutorials (:ref:`Generating NC12H26 QSS mechanism with analytical jacobian <sec_tutqss1>` and :ref:`Generating NC12H26 QSS mechanism without analytical jacobian <sec_tutqss2>`) for generating QSS mechanisms from the ``.yaml`` files.
-
-

Support/ceptr/ceptr/ceptr.py

@@ -3,19 +3,47 @@
 import argparse
 import pathlib
 import time
+from itertools import repeat
+from multiprocessing import Pool, cpu_count
 
 import cantera as ct
 
 import ceptr.converter as converter
 
 
+def parse_lst_file(lst):
+    """Return mechanism paths give a file containing a list of mechanism files."""
+    lpath = pathlib.Path(lst)
+    fnames = []
+    with open(lst, "r") as f:
+        for line in f:
+            if not line.startswith("#"):
+                fnames.append(line)
+    return [lpath.parents[0] / fn.strip() for fn in fnames]
+
+
+def parse_qss_lst_file(lst):
+    """Return mechanism paths give a file containing a list of qss mechanism files."""
+    lpath = pathlib.Path(lst)
+    fnames = []
+    with open(lst, "r") as f:
+        for line in f:
+            if not line.startswith("#"):
+                fnames.append(line)
+
+    mechnames = [lpath.parents[0] / fn.split()[0].strip() for fn in fnames]
+    qss_format_inputs = [lpath.parents[0] / fn.split()[1].strip() for fn in fnames]
+    return mechnames, qss_format_inputs
+
+
 def convert(
     fname,
     jacobian,
     qss_format_input,
     qss_symbolic_jac,
 ):
     """Convert a mechanism file."""
+    print(f"""Converting file {fname}""")
     mechanism = ct.Solution(fname)
     conv = converter.Converter(
         mechanism,
@@ -32,41 +60,36 @@ def convert_lst(
     jacobian,
     qss_format_input,
     qss_symbolic_jac,
+    ncpu,
 ):
     """Convert mechanisms from a file containing a list of directories."""
-    lpath = pathlib.Path(lst)
-    with open(lst, "r") as f:
-        for line in f:
-            if not line.startswith("#"):
-                mechname = lpath.parents[0] / line.strip()
-                print(f"""Converting file {mechname}""")
-                convert(
-                    mechname,
-                    jacobian,
-                    qss_format_input,
-                    qss_symbolic_jac,
-                )
+    mechnames = parse_lst_file(lst)
+    print(f"Using {ncpu} processes")
+    with Pool(ncpu) as pool:
+        pool.starmap(
+            convert,
+            zip(
+                mechnames,
+                repeat(jacobian),
+                repeat(qss_format_input),
+                repeat(qss_symbolic_jac),
+            ),
+        )
 
 
 def convert_lst_qss(
     lst,
     jacobian,
+    ncpu,
 ):
     """Convert QSS mechanisms from a file of directories and format input."""
-    lpath = pathlib.Path(lst)
-    with open(lst, "r") as f:
-        for line in f:
-            if not line.startswith("#"):
-                mech_file, format_file, _, _ = line.split()
-                mechname = lpath.parents[0] / mech_file.strip()
-                qss_format_input = lpath.parents[0] / format_file.strip()
-                print(f"""Converting file {mechname}""")
-                convert(
-                    mechname,
-                    jacobian,
-                    qss_format_input,
-                    True,
-                )
+    mechnames, qss_format_inputs = parse_qss_lst_file(lst)
+    print(f"Using {ncpu} processes")
+    with Pool(ncpu) as pool:
+        pool.starmap(
+            convert,
+            zip(mechnames, repeat(jacobian), qss_format_inputs, repeat(True)),
+        )
 
 
 def main():
@@ -104,6 +127,10 @@ def main():
         help="Do not generate a jacobian",
     )
 
+    parser.add_argument(
+        "-n", "--ncpu", help="Number of processes to use", type=int, default=cpu_count()
+    )
+
     args = parser.parse_args()
 
     if args.fname:
@@ -119,11 +146,13 @@ def main():
             not args.no_jacobian,
             args.qss_format_input,
             args.qss_symbolic_jacobian,
+            args.ncpu,
         )
     elif args.lst_qss:
         convert_lst_qss(
             args.lst_qss,
             not args.no_jacobian,
+            args.ncpu,
         )
     end = time.time()
     print(f"CEPTR run time: {end-start:.2f} s")