tests: basic test for evals

tests/test_eval.py

@@ -4,7 +4,6 @@
 
 
 @pytest.mark.slow
-@pytest.mark.eval
 def test_eval(test):
     """
     This test will be run for each eval in the tests list.
@@ -18,4 +17,8 @@ def test_eval(test):
 # Hook to generate tests from the tests list
 def pytest_generate_tests(metafunc):
     if "test" in metafunc.fixturenames:
-        metafunc.parametrize("test", tests, ids=[test["name"] for test in tests])
+        allowlist = ["hello"]  # for now, only run the hello test
+        test_set, test_names = zip(
+            *[(test, test["name"]) for test in tests if test["name"] in allowlist]
+        )
+        metafunc.parametrize("test", test_set, ids=test_names)