Guard cli run with invalid or incomplete config

garak/_config.py

@@ -158,11 +158,12 @@ def load_config(
 
 def parse_plugin_spec(
     spec: str, category: str, probe_tag_filter: str = ""
-) -> List[str]:
+) -> tuple[List[str], List[str]]:
     from garak._plugins import enumerate_plugins
 
     if spec is None or spec.lower() in ("", "auto", "none"):
-        return []
+        return [], []
+    unknown_plugins = []
     if spec.lower() in ("all", "*"):
         plugin_names = [
             name
@@ -173,13 +174,26 @@ def parse_plugin_spec(
         plugin_names = []
         for clause in spec.split(","):
             if clause.count(".") < 1:
-                plugin_names += [
+                found_plugins = [
                     p
                     for p, a in enumerate_plugins(category=category)
                     if p.startswith(f"{category}.{clause}.") and a is True
                 ]
+                if len(found_plugins) > 0:
+                    plugin_names += found_plugins
+                else:
+                    unknown_plugins += [clause]
             else:
-                plugin_names += [f"{category}.{clause}"]  # spec parsing
+                # validate the class exists
+                found_plugins = [
+                    p
+                    for p, a in enumerate_plugins(category=category)
+                    if p == f"{category}.{clause}"
+                ]
+                if len(found_plugins) > 0:
+                    plugin_names += found_plugins
+                else:
+                    unknown_plugins += [clause]
 
     if probe_tag_filter is not None and len(probe_tag_filter) > 1:
         plugins_to_skip = []
@@ -196,4 +210,4 @@ def parse_plugin_spec(
         for plugin_to_skip in plugins_to_skip:
             plugin_names.remove(plugin_to_skip)
 
-    return plugin_names
+    return plugin_names, unknown_plugins

garak/cli.py

@@ -3,7 +3,7 @@
 
 """Flow for invoking garak from the command line"""
 
-command_options = "list_detectors list_probes list_generators list_buffs list_config plugin_info interactive report version".split()
+command_options = "list_detectors list_probes list_generators list_buffs list_config plugin_info interactive report version skip_unknown".split()
 
 
 def main(arguments=[]) -> None:
@@ -20,6 +20,7 @@ def main(arguments=[]) -> None:
     import garak.command as command
     import logging
     import re
+    from colorama import Fore, Style
 
     command.start_logging()
     _config.load_base_config()
@@ -237,6 +238,11 @@ def main(arguments=[]) -> None:
         action="store_true",
         help="Launch garak in interactive.py mode",
     )
+    parser.add_argument(
+        "--skip_unknown",
+        action="store_true",
+        help="allow skip of unknown probes or detectors",
+    )
 
     logging.debug("args - raw argument string received: %s", arguments)
 
@@ -327,9 +333,6 @@ def main(arguments=[]) -> None:
     import garak.evaluators
 
     try:
-        if not args.version and not args.report:
-            command.start_run()
-
         # do a special thing for CLIprobe options, generator options
         if "probe_options" in args or "probe_option_file" in args:
             if "probe_options" in args:
@@ -382,11 +385,14 @@ def main(arguments=[]) -> None:
             from garak.interactive import interactive_mode
 
             try:
+                command.start_run()  # start run to track actions
                 interactive_mode()
             except Exception as e:
                 logging.error(e)
                 print(e)
                 sys.exit(1)
+            finally:
+                command.end_run()
 
         if args.version:
             pass
@@ -429,7 +435,27 @@ def main(arguments=[]) -> None:
                 message = f"⚠️  Model type '{_config.plugins.model_type}' also needs a model name\n You can set one with e.g. --model_name \"billwurtz/gpt-1.0\""
                 logging.error(message)
                 raise ValueError(message)
-            print(f"📜 reporting to {_config.transient.report_filename}")
+
+            parsable_specs = ["probe", "detector", "buff"]
+            parsed_specs = {}
+            for spec_type in parsable_specs:
+                spec_namespace = f"{spec_type}s"
+                config_spec = getattr(_config.plugins, f"{spec_type}_spec", "")
+                config_tags = getattr(_config.run, f"{spec_type}_tags", "")
+                names, rejected = _config.parse_plugin_spec(
+                    config_spec, spec_namespace, config_tags
+                )
+                parsed_specs[spec_type] = names
+                if rejected is not None and len(rejected) > 0:
+                    if not args.skip_unknown:
+                        raise ValueError(f"❌Unknown {spec_namespace}❌: {",".join(rejected)}")
+                    else:
+                        header = f"Unknown {spec_namespace}:"
+                        msg = f"{Fore.LIGHTYELLOW_EX}{header}\n" + "\n".join([ f"{Fore.LIGHTYELLOW_EX + "SKIP" + Style.RESET_ALL} {spec}" for spec in rejected ])
+                        logging.warning(f"{header} " + ",".join(rejected))
+                        print(msg)
+
+            evaluator = garak.evaluators.ThresholdEvaluator(_config.run.eval_threshold)
 
             generator_module_name = _config.plugins.model_type.split(".")[0]
             generator_mod = importlib.import_module(
@@ -445,9 +471,6 @@ def main(arguments=[]) -> None:
             else:
                 generator_class_name = _config.plugins.model_type.split(".")[1]
 
-            #        if 'model_name' not in args:
-            #            generator = getattr(generator_mod, generator_class_name)()
-            #        else:
             generator = getattr(generator_mod, generator_class_name)(
                 _config.plugins.model_name
             )
@@ -472,22 +495,14 @@ def main(arguments=[]) -> None:
                     )
                 autodan_generate(generator=generator, prompt=prompt, target=target)
 
-            probe_names = _config.parse_plugin_spec(
-                _config.plugins.probe_spec, "probes", _config.run.probe_tags
-            )
-            detector_names = _config.parse_plugin_spec(
-                _config.plugins.detector_spec, "detectors"
-            )
-            buff_names = _config.parse_plugin_spec(_config.plugins.buff_spec, "buffs")
-
-            evaluator = garak.evaluators.ThresholdEvaluator(_config.run.eval_threshold)
-
-            if detector_names == []:
-                command.probewise_run(generator, probe_names, evaluator, buff_names)
+            command.start_run()  # start the run now that all config validation is complete
+            print(f"📜 reporting to {_config.transient.report_filename}")
 
+            if parsed_specs["detector"] == []:
+                command.probewise_run(generator, parsed_specs["probe"], evaluator, parsed_specs["buff"])
             else:
                 command.pxd_run(
-                    generator, probe_names, detector_names, evaluator, buff_names
+                    generator, parsed_specs["probe"], parsed_specs["detector"], evaluator, parsed_specs["buff"]
                 )
 
             command.end_run()
@@ -500,3 +515,6 @@ def main(arguments=[]) -> None:
             logging.info("nothing to do 🤷")
     except KeyboardInterrupt:
         print("User cancel received, terminating all runs")
+    except ValueError as e:
+        logging.error(e)
+        print(e)

garak/harnesses/base.py

@@ -75,16 +75,18 @@ def run(self, model, probes, detectors, evaluator, announce_probe=True) -> None:
         :type announce_probe: bool, optional
         """
         if not detectors:
-            logging.warning("No detectors, nothing to do")
+            msg = "No detectors, nothing to do"
+            logging.warning(msg)
             if hasattr(_config.system, "verbose") and _config.system.verbose >= 2:
-                print("No detectors, nothing to do")
-            return None
+                print(msg)
+            raise ValueError(msg)
 
         if not probes:
-            logging.warning("No probes, nothing to do")
+            msg = "No probes, nothing to do"
+            logging.warning(msg)
             if hasattr(_config.system, "verbose") and _config.system.verbose >= 2:
-                print("No probes, nothing to do")
-            return None
+                print(msg)
+            raise ValueError(msg)
 
         for probe in probes:
             logging.debug("harness: probe start for %s", probe.probename)

garak/harnesses/probewise.py

@@ -58,10 +58,11 @@ def run(self, model, probenames, evaluator, buff_names=[]):
         """
 
         if not probenames:
-            logging.warning("No probes, nothing to do")
+            msg = "No probes, nothing to do"
+            logging.warning(msg)
             if hasattr(_config.system, "verbose") and _config.system.verbose >= 2:
-                print("No probes, nothing to do")
-            return None
+                print(msg)
+            raise ValueError(msg)
 
         self._load_buffs(buff_names)
 

tests/cli/test_cli.py

@@ -77,6 +77,7 @@ def test_run_all_active_detectors(capsys):
             "-g",
             "1",
             "--narrow_output",
+            "--skip_unknown",
         ]
     )
     result = capsys.readouterr()

tests/test_config.py

@@ -439,18 +439,42 @@ def test_blank_generator_instance_loads_cli_config():
 
 # test parsing of probespec
 def test_probespec_loading():
-    assert _config.parse_plugin_spec(None, "detectors") == []
-    assert _config.parse_plugin_spec("Auto", "probes") == []
-    assert _config.parse_plugin_spec("NONE", "probes") == []
-    assert _config.parse_plugin_spec("", "generators") == []
-    assert _config.parse_plugin_spec("atkgen", "probes") == ["probes.atkgen.Tox"]
+    assert _config.parse_plugin_spec(None, "detectors") == ([], [])
+    assert _config.parse_plugin_spec("", "generators") == ([], [])
+    assert _config.parse_plugin_spec("Auto", "probes") == ([], [])
+    assert _config.parse_plugin_spec("NONE", "probes") == ([], [])
+    # reject unmatched spec entires
+    assert _config.parse_plugin_spec("probedoesnotexist", "probes") == (
+        [],
+        ["probedoesnotexist"],
+    )
+    assert _config.parse_plugin_spec("atkgen,probedoesnotexist", "probes") == (
+        ["probes.atkgen.Tox"],
+        ["probedoesnotexist"],
+    )
+    assert _config.parse_plugin_spec("atkgen.Tox,probedoesnotexist", "probes") == (
+        ["probes.atkgen.Tox"],
+        ["probedoesnotexist"],
+    )
+    # reject unmatched spec entires for unknown class
+    assert _config.parse_plugin_spec(
+        "atkgen.Tox,atkgen.ProbeDoesNotExist", "probes"
+    ) == (["probes.atkgen.Tox"], ["atkgen.ProbeDoesNotExist"])
+    # accept known disabled class
+    assert _config.parse_plugin_spec("dan.DanInTheWild", "probes") == (
+        ["probes.dan.DanInTheWild"],
+        [],
+    )
+    # gather all class entires for namespace
+    assert _config.parse_plugin_spec("atkgen", "probes") == (["probes.atkgen.Tox"], [])
+    assert _config.parse_plugin_spec("always", "detectors") == (
+        ["detectors.always.Fail", "detectors.always.Pass"],
+        [],
+    )
+    # reject all unknown class entires for namespace
     assert _config.parse_plugin_spec(
         "long.test.class,another.long.test.class", "probes"
-    ) == ["probes.long.test.class", "probes.another.long.test.class"]
-    assert _config.parse_plugin_spec("always", "detectors") == [
-        "detectors.always.Fail",
-        "detectors.always.Pass",
-    ]
+    ) == ([], ["long.test.class", "another.long.test.class"])
 
 
 def test_buff_config_assertion():
@@ -464,16 +488,18 @@ def test_buff_config_assertion():
 
 
 def test_tag_filter():
-    assert (
-        _config.parse_plugin_spec("atkgen", "probes", probe_tag_filter="LOL NULL") == []
-    )
-    assert _config.parse_plugin_spec("*", "probes", probe_tag_filter="avid") != []
-    assert (
-        _config.parse_plugin_spec("all", "probes", probe_tag_filter="owasp:llm") != []
+    assert _config.parse_plugin_spec(
+        "atkgen", "probes", probe_tag_filter="LOL NULL"
+    ) == ([], [])
+    assert _config.parse_plugin_spec("*", "probes", probe_tag_filter="avid") != ([], [])
+    assert _config.parse_plugin_spec("all", "probes", probe_tag_filter="owasp:llm") != (
+        [],
+        [],
     )
-    assert "probes.lmrc.SexualContent" in _config.parse_plugin_spec(
+    found, rejected = _config.parse_plugin_spec(
         "all", "probes", probe_tag_filter="risk-cards:lmrc:sexual_content"
     )
+    assert "probes.lmrc.SexualContent" in found
 
 
 def test_report_prefix_with_hitlog_no_explode():