tests: fix coverage for eval tests (multiprocessing workaround)

ErikBjare · Aug 14, 2024 · 9ef1ec4 · 9ef1ec4
1 parent 855a46b
commit 9ef1ec4
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 0 deletions.
diff --git a/gptme/eval/main.py b/gptme/eval/main.py
@@ -181,6 +181,15 @@ def run_evals(
     """
     Run evals for a list of tests.
     """
+    # For coverage to work with multiprocessing
+    # https://pytest-cov.readthedocs.io/en/latest/subprocess-support.html
+    try:
+        from pytest_cov.embed import cleanup_on_sigterm  # fmt: skip
+    except ImportError:
+        pass
+    else:
+        cleanup_on_sigterm()
+
     model_results = defaultdict(list)
     with ProcessPoolExecutor(parallel) as executor:
         model_futures_to_test = {

diff --git a/tests/test_eval.py b/tests/test_eval.py
@@ -8,11 +8,13 @@
 @pytest.mark.slow
 def test_eval_cli():
     runner = CliRunner()
+    test_set = ["hello"]
     result = runner.invoke(
         main,
         [
             "--model",
             "openai/gpt-4o",
+            *test_set,
         ],
     )
     assert result