Add Parameters to Benchmarks

README.md

@@ -49,11 +49,11 @@ record = runner.run("__main__", params={"a": 2, "b": 10})
 rep = nnbench.BenchmarkReporter()
 rep.display(record)  # ...and print the results to the terminal.
 
-# results in a table like the following:
-# name     function    date                   value    time_ns
-# -------  ----------  -------------------  -------  ---------
-# product  product     2024-03-07T10:14:21       20       1000
-# power    power       2024-03-07T10:14:21     1024        750
+# results in a table look like the following:
+# name     function    date                 parameters         value    time_ns
+# -------  ----------  -------------------  -----------------  -------  ---------
+# product  product     2024-03-08T18:03:48  {'a': 2, 'b': 10}       20       1000
+# power    power       2024-03-08T18:03:48  {'a': 2, 'b': 10}     1024        750
 ```
 
 For a more realistic example of how to evaluate a trained model with a benchmark suite, check the [Quickstart](https://aai-institute.github.io/nnbench/latest/quickstart/).

src/nnbench/runner.py

@@ -255,6 +255,7 @@ def run(
         results: list[dict[str, Any]] = []
         for benchmark in self.benchmarks:
             bmparams = {k: v for k, v in dparams.items() if k in benchmark.interface.names}
+            bmdefaults = {k: v for (k, _, v) in benchmark.interface.variables}
             # TODO: Wrap this into an execution context
             res: dict[str, Any] = {
                 "name": benchmark.name,
@@ -263,6 +264,7 @@ def run(
                 "date": datetime.now().isoformat(timespec="seconds"),
                 "error_occurred": False,
                 "error_message": "",
+                "parameters": {**bmdefaults, **bmparams},
             }
             try:
                 benchmark.setUp(**bmparams)

tests/benchmarks/parametrized.py

@@ -0,0 +1,6 @@
+import nnbench
+
+
+@nnbench.parametrize([{"a": 1}, {"a": 2}], tags=("parametrized",))
+def double(a: int) -> int:
+    return 2 * 2

tests/test_runner.py

@@ -71,3 +71,21 @@ def duplicate_context_provider() -> dict[str, str]:
             params={"x": 1, "y": 1},
             context=context_providers,
         )
+
+
+def test_filter_benchmarks_on_params(testfolder: str) -> None:
+    r = nnbench.BenchmarkRunner()
+    results = r.run(testfolder, tags=("parametrized",))
+    print(results)
+    assert len(results.benchmarks) == 2
+    assert (
+        len(
+            list(
+                filter(
+                    lambda bm: bm["parameters"]["a"] == 1,
+                    results.benchmarks,
+                )
+            )
+        )
+        == 1
+    )