Add F1 Macro score to qa.multiple_choice.with_topic

openshift-psap · Oct 25, 2024 · 7b02ab7 · 7b02ab7
1 parent c1090ec
commit 7b02ab7
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/llm_eval_test/benchmarks/catalog/metrics/f1_macro.json b/llm_eval_test/benchmarks/catalog/metrics/f1_macro.json
@@ -0,0 +1,3 @@
+{
+    "__type__": "f1_macro"
+}
diff --git a/llm_eval_test/benchmarks/catalog/tasks/qa/multiple_choice/with_topic.json b/llm_eval_test/benchmarks/catalog/tasks/qa/multiple_choice/with_topic.json
@@ -11,6 +11,7 @@
     },
     "prediction_type": "str",
     "metrics": [
-        "metrics.accuracy"
+        "metrics.accuracy",
+        "metrics.f1_macro"
     ]
 }