Feature/luc/optimizer qoi (#34)

* Add a new command for construction optimization cost functions

CHANGELOG.md

@@ -4,12 +4,17 @@ All notable changes to Merlin Spellbook will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.0]
+
+### Added
+- function `make-barrier-cost` to create optimization cost functions
+
 ## [0.6.1]
 
-## Added
+### Added
 - pylint to dev requirements for format checking
 
-## Changed
+### Changed
 - Conduit tests are skipped when conduit python package is not available. (test_conduit.py is also skipped for format checking)
 - Rename l variable in setup.py to line.
 - Removed unused import libraries.
@@ -20,15 +25,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.6.0]
 
-## Added
+### Added
 - Delimiter option in serialization command.
 
-## Changed
+### Changed
 - In serialize command, grab the first entry as the key and everything else as the value.
 
 ## [0.5.3]
 
-## Fixed
+### Fixed
 - version number in docs and setup.py to match 0.5.3
 
 ## [0.5.2]

Makefile

@@ -41,3 +41,7 @@ fix-style:
 	black --target-version py36 $(PROJ)
 	black --target-version py36 $(TEST)
 	black --target-version py36 *.py
+
+check-style:
+	python3 -m black --check --line-length 88 --target-version py36 $(PROJ)
+	python3 -m pylint $(PROJ) --rcfile=setup.cfg --exit-zero

setup.py

@@ -58,7 +58,10 @@ def _pip_requirement(req):
 def _reqs(*f):
     return [
         _pip_requirement(r)
-        for r in (_strip_comments(line) for line in open(os.path.join(os.getcwd(), "requirements", *f)).readlines())
+        for r in (
+            _strip_comments(line)
+            for line in open(os.path.join(os.getcwd(), "requirements", *f)).readlines()
+        )
         if r
     ]
 

spellbook/commands/make-barrier-cost.py

@@ -0,0 +1,86 @@
+from types import SimpleNamespace
+
+import click
+
+
+@click.command()
+@click.option(
+    "--infile",
+    "-i",
+    required=False,
+    default="results.npz",
+    type=str,
+    help=".npz file with X and y data",
+)
+@click.option(
+    "--outfile",
+    "-o",
+    required=False,
+    default="objective_qoi.npz",
+    type=str,
+    help="npz file to store the objective in",
+)
+@click.option(
+    "--x_names",
+    "-x",
+    required=False,
+    default=None,
+    type=str,
+    help="variable(s) in infile for the input, defaults to X; can be a comma-delimited list",
+)
+@click.option(
+    "--function",
+    "-y",
+    required=False,
+    default=None,
+    type=str,
+    help="variable(s) in infile for the output, defaults to y; can be a comma-delimited list",
+)
+@click.option(
+    "--maximize",
+    "-m",
+    required=False,
+    default=False,
+    is_flag=True,
+    type=bool,
+    help="Create a function for maximizing (instead of minimizing, which is default)",
+)
+@click.option(
+    "--constraints",
+    "-c",
+    required=False,
+    default=None,
+    type=str,
+    help="constraint options to apply, of the form 'g1>1.0,g1<2,g2>4'. Comma-separated string with > or < separating data name and value",
+)
+def cli(infile, outfile, x_names, function, maximize, constraints):
+    """
+    Make a "barrier" cost func for constrained opt.
+
+    Note: the 'maximize' flag indicates this is a function that should be maximized, instead of minimized
+
+    Data are stored in a zipped npz file and saved to another zipped file, which contains two fields "X" and "y".
+    "X" contains the design variables and "y" contains the new objective.
+
+    Example:
+
+    spellbook make-barrier-cost --infile my_data.npz -x "x0,x1" -y "y" --constraints "g<0,g>-1,h>3.141" --outfile objective.npz
+
+    spellbook learn -infile objective.npz -outfile trained_model.pkl
+
+    python -c "import numpy as np; data = np.load('objective.npz'); print(data['X'][data["y"].argmin())])"
+
+    """
+    from spellbook.optimization import qoi
+
+    args = SimpleNamespace(
+        **{
+            "infile": infile,
+            "outfile": outfile,
+            "X": x_names,
+            "objective": function,
+            "maximize_objective": maximize,
+            "constraints": constraints,
+        }
+    )
+    qoi.process_args(args)

spellbook/data_formatting/conduit/python/conduit_bundler.py

@@ -58,7 +58,9 @@ def determine_protocol(fname):
     if ext.startswith("."):
         protocol = ext.lower().strip(".")
     else:
-        raise ValueError("{0} needs an ext (eg .hdf5) to determine protocol!".format(fname))
+        raise ValueError(
+            "{0} needs an ext (eg .hdf5) to determine protocol!".format(fname)
+        )
     # Map .h5 to .hdf5
     if protocol == "h5":
         protocol = "hdf5"
@@ -155,7 +157,10 @@ def dump_node(
         try:
             conduit.relay.io.save(conduit_node, fname, options=save_options)
         except TypeError:  # Conduit version needs to be updated.
-            LOG.error("Unable to customize save: please upgrade conduit to " "expose save options!")
+            LOG.error(
+                "Unable to customize save: please upgrade conduit to "
+                "expose save options!"
+            )
             conduit.relay.io.save(conduit_node, fname)
     else:
         conduit.relay.io.save(conduit_node, fname)

spellbook/data_formatting/conduit/python/translator.py

@@ -86,7 +86,9 @@ def run(_input, output, schema):
             if data_loader.has_path(sample_path):
                 data_loader.read(filtered_node[path], sample_path)
             else:
-                filtered_node[sample_path] = np.nan  # if a value is missing, that could be a problem
+                filtered_node[
+                    sample_path
+                ] = np.nan  # if a value is missing, that could be a problem
         make_data_array_dict(all_dict, filtered_node)
 
     for dat in all_dict.keys():
@@ -127,7 +129,9 @@ def generate_scalar_path_pairs(node, path=""):
     children = node.child_names()
     for child in children:
         if isinstance(node[child], conduit.Node):
-            for pair in generate_scalar_path_pairs(node[child], path=path + child + "/"):
+            for pair in generate_scalar_path_pairs(
+                node[child], path=path + child + "/"
+            ):
                 yield pair
         else:
             yield path + child, node[child]

spellbook/data_formatting/stack_npz.py

@@ -24,7 +24,9 @@ def pad_many(arrays, dims, dont_pad_first=False, value=np.nan):
         if dont_pad_first:
             pad_dist[0] = 0
         padder = np.column_stack((zeros, pad_dist))
-        fixed.append(np.pad(np.atleast_2d(a), padder, mode="constant", constant_values=value))
+        fixed.append(
+            np.pad(np.atleast_2d(a), padder, mode="constant", constant_values=value)
+        )
     return fixed
 
 
@@ -46,7 +48,11 @@ def run(self, target, source, force=False):
 
         if not force:
             if os.path.isfile(target):
-                print("stack_npz error opening target file (does {0} exist?).".format(target))
+                print(
+                    "stack_npz error opening target file (does {0} exist?).".format(
+                        target
+                    )
+                )
                 print('Pass "-f" argument to force re-creation of output file.')
                 return
 

spellbook/data_formatting/translator.py

@@ -13,8 +13,12 @@ def setup_argparse():
         help=".json file with X and y data in each sample",
         default="results.json",
     )
-    parser.add_argument("-output", help=".npz file with the arrays", default="results.npz")
-    parser.add_argument("-schema", help="schema for a single sample data", default="features.json")
+    parser.add_argument(
+        "-output", help=".npz file with the arrays", default="results.npz"
+    )
+    parser.add_argument(
+        "-schema", help="schema for a single sample data", default="features.json"
+    )
     return parser
 
 
@@ -47,7 +51,9 @@ def generate_scalar_path_pairs(node, schema, path=""):
         if child in schema.keys():
             if isinstance(node[child], dict):
                 if isinstance(schema[child], dict):
-                    for pair in generate_scalar_path_pairs(node[child], schema[child], path=path + child + "/"):
+                    for pair in generate_scalar_path_pairs(
+                        node[child], schema[child], path=path + child + "/"
+                    ):
                         yield pair
             else:
                 if not isinstance(schema[child], dict):

spellbook/main.py

@@ -28,6 +28,7 @@ def list_commands(self, ctx):
             "serialize",
             "stack-npz",
             "translate",
+            "make-barrier-cost",
         ]
 
     def list_commands_dynamically(self, ctx):

spellbook/ml/surrogates.py

@@ -65,7 +65,11 @@ def factory(cls, name, *args, **kwargs):
         if name in cls.all_regs:
             return cls.all_regs[name](*args, **kwargs)
         else:
-            raise ValueError("Unknown regressor name " + name + "! For valid choices see sklearnRegressors.names()")
+            raise ValueError(
+                "Unknown regressor name "
+                + name
+                + "! For valid choices see sklearnRegressors.names()"
+            )
 
     @classmethod
     def names(cls):
@@ -81,8 +85,12 @@ def test_factory():
 
 def test_random_forest():
 
-    rf1 = sklearnRegressors.factory("RandomForestRegressor", n_estimators=10, max_depth=5)
-    rf2 = sklearnRegressors.factory("RandomForestRegressor", n_estimators=2, max_depth=3)
+    rf1 = sklearnRegressors.factory(
+        "RandomForestRegressor", n_estimators=10, max_depth=5
+    )
+    rf2 = sklearnRegressors.factory(
+        "RandomForestRegressor", n_estimators=2, max_depth=3
+    )
 
     assert rf1.n_estimators == 10
     assert rf1.max_depth == 5