openshift-psap · sjmonson · Oct 17, 2024 · Oct 17, 2024 · Oct 17, 2024
diff --git a/llm_eval_test/benchmarks/catalog/cards/mmlu/all.json b/llm_eval_test/benchmarks/catalog/cards/mmlu/all.json
@@ -0,0 +1,43 @@
+{
+    "__type__": "task_card",
+    "loader": {
+        "__type__": "load_hf",
+        "path": "cais/mmlu",
+        "name": "all"
+    },
+    "preprocess_steps": [
+        {
+            "__type__": "rename_splits",
+            "mapper": {
+                "dev": "train"
+            }
+        },
+        {
+            "__type__": "set",
+            "fields": {
+                "topic": "multiple"
+            }
+        }
+    ],
+    "task": "tasks.qa.multiple_choice.with_topic",
+    "templates": "templates.qa.multiple_choice.with_topic.all",
+    "__tags__": {
+        "annotations_creators": "no-annotation",
+        "arxiv": [
+            "2009.03300",
+            "2005.00700",
+            "2005.14165",
+            "2008.02275"
+        ],
+        "language": "en",
+        "language_creators": "expert-generated",
+        "license": "mit",
+        "multilinguality": "monolingual",
+        "region": "us",
+        "size_categories": "10K<n<100K",
+        "source_datasets": "original",
+        "task_categories": "question-answering",
+        "task_ids": "multiple-choice-qa"
+    },
+    "__description__": "Measuring Massive Multitask Language Understanding by Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt (ICLR 2021). \nThis is a massive multitask test consisting of multiple-choice questions from various branches of knowledge. The test spans subjects in the humanities, social sciences, hard sciences, and other areas that are important for some people to learn. This covers 57… See the full description on the dataset page: https://huggingface.co/datasets/cais/mmlu."
+}
diff --git a/llm_eval_test/benchmarks/tasks/_common/README.md b/llm_eval_test/benchmarks/tasks/_common/README.md
@@ -0,0 +1,12 @@
+# Common files for Unitxt tasks
+
+Symlink these files into each task that requires Unitxt.
+Ensure that the symlinks are relative to the task directory.
+
+```sh
+mkdir new_task
+cd new_task
+
+ln -s ../_common/task.py ./
+ln -s ../_common/unitxt ./
+```
diff --git a/llm_eval_test/benchmarks/tasks/task.py → ...val_test/benchmarks/tasks/_common/task.py b/llm_eval_test/benchmarks/tasks/task.py → ...val_test/benchmarks/tasks/_common/task.py
diff --git a/llm_eval_test/benchmarks/tasks/unitxt → ...eval_test/benchmarks/tasks/_common/unitxt b/llm_eval_test/benchmarks/tasks/unitxt → ...eval_test/benchmarks/tasks/_common/unitxt
diff --git a/llm_eval_test/benchmarks/tasks/mmlu/mmlu.yaml b/llm_eval_test/benchmarks/tasks/mmlu/mmlu.yaml
@@ -0,0 +1,3 @@
+task: mmlu_all
+include: unitxt
+recipe: card=cards.mmlu.all,template=templates.qa.multiple_choice.with_topic.lm_eval_harness
diff --git a/llm_eval_test/benchmarks/tasks/mmlu/task.py b/llm_eval_test/benchmarks/tasks/mmlu/task.py
@@ -0,0 +1 @@
+../_common/task.py
diff --git a/llm_eval_test/benchmarks/tasks/mmlu/unitxt b/llm_eval_test/benchmarks/tasks/mmlu/unitxt
@@ -0,0 +1 @@
+../_common/unitxt
diff --git a/...eval_test/benchmarks/tasks/_mmlu_pro.yaml → .../benchmarks/tasks/mmlu_pro/_mmlu_pro.yaml b/...eval_test/benchmarks/tasks/_mmlu_pro.yaml → .../benchmarks/tasks/mmlu_pro/_mmlu_pro.yaml
diff --git a/...l_test/benchmarks/tasks/mmlu_pro_all.yaml → ...nchmarks/tasks/mmlu_pro/mmlu_pro_all.yaml b/...l_test/benchmarks/tasks/mmlu_pro_all.yaml → ...nchmarks/tasks/mmlu_pro/mmlu_pro_all.yaml
diff --git a/...st/benchmarks/tasks/mmlu_pro_biology.yaml → ...arks/tasks/mmlu_pro/mmlu_pro_biology.yaml b/...st/benchmarks/tasks/mmlu_pro_biology.yaml → ...arks/tasks/mmlu_pro/mmlu_pro_biology.yaml
diff --git a/...t/benchmarks/tasks/mmlu_pro_business.yaml → ...rks/tasks/mmlu_pro/mmlu_pro_business.yaml b/...t/benchmarks/tasks/mmlu_pro_business.yaml → ...rks/tasks/mmlu_pro/mmlu_pro_business.yaml
diff --git a/.../benchmarks/tasks/mmlu_pro_chemistry.yaml → ...ks/tasks/mmlu_pro/mmlu_pro_chemistry.yaml b/.../benchmarks/tasks/mmlu_pro_chemistry.yaml → ...ks/tasks/mmlu_pro/mmlu_pro_chemistry.yaml
diff --git a/...arks/tasks/mmlu_pro_computer_science.yaml → ...s/mmlu_pro/mmlu_pro_computer_science.yaml b/...arks/tasks/mmlu_pro_computer_science.yaml → ...s/mmlu_pro/mmlu_pro_computer_science.yaml
diff --git a/.../benchmarks/tasks/mmlu_pro_economics.yaml → ...ks/tasks/mmlu_pro/mmlu_pro_economics.yaml b/.../benchmarks/tasks/mmlu_pro_economics.yaml → ...ks/tasks/mmlu_pro/mmlu_pro_economics.yaml
diff --git a/...enchmarks/tasks/mmlu_pro_engineering.yaml → .../tasks/mmlu_pro/mmlu_pro_engineering.yaml b/...enchmarks/tasks/mmlu_pro_engineering.yaml → .../tasks/mmlu_pro/mmlu_pro_engineering.yaml
diff --git a/...est/benchmarks/tasks/mmlu_pro_health.yaml → ...marks/tasks/mmlu_pro/mmlu_pro_health.yaml b/...est/benchmarks/tasks/mmlu_pro_health.yaml → ...marks/tasks/mmlu_pro/mmlu_pro_health.yaml
diff --git a/...st/benchmarks/tasks/mmlu_pro_history.yaml → ...arks/tasks/mmlu_pro/mmlu_pro_history.yaml b/...st/benchmarks/tasks/mmlu_pro_history.yaml → ...arks/tasks/mmlu_pro/mmlu_pro_history.yaml
diff --git a/...l_test/benchmarks/tasks/mmlu_pro_law.yaml → ...nchmarks/tasks/mmlu_pro/mmlu_pro_law.yaml b/...l_test/benchmarks/tasks/mmlu_pro_law.yaml → ...nchmarks/tasks/mmlu_pro/mmlu_pro_law.yaml
diff --git a/..._test/benchmarks/tasks/mmlu_pro_math.yaml → ...chmarks/tasks/mmlu_pro/mmlu_pro_math.yaml b/..._test/benchmarks/tasks/mmlu_pro_math.yaml → ...chmarks/tasks/mmlu_pro/mmlu_pro_math.yaml
diff --git a/...test/benchmarks/tasks/mmlu_pro_other.yaml → ...hmarks/tasks/mmlu_pro/mmlu_pro_other.yaml b/...test/benchmarks/tasks/mmlu_pro_other.yaml → ...hmarks/tasks/mmlu_pro/mmlu_pro_other.yaml
diff --git a/...benchmarks/tasks/mmlu_pro_philosophy.yaml → ...s/tasks/mmlu_pro/mmlu_pro_philosophy.yaml b/...benchmarks/tasks/mmlu_pro_philosophy.yaml → ...s/tasks/mmlu_pro/mmlu_pro_philosophy.yaml
diff --git a/...st/benchmarks/tasks/mmlu_pro_physics.yaml → ...arks/tasks/mmlu_pro/mmlu_pro_physics.yaml b/...st/benchmarks/tasks/mmlu_pro_physics.yaml → ...arks/tasks/mmlu_pro/mmlu_pro_physics.yaml
diff --git a/...benchmarks/tasks/mmlu_pro_psychology.yaml → ...s/tasks/mmlu_pro/mmlu_pro_psychology.yaml b/...benchmarks/tasks/mmlu_pro_psychology.yaml → ...s/tasks/mmlu_pro/mmlu_pro_psychology.yaml
diff --git a/llm_eval_test/benchmarks/tasks/mmlu_pro/task.py b/llm_eval_test/benchmarks/tasks/mmlu_pro/task.py
@@ -0,0 +1 @@
+../_common/task.py
diff --git a/llm_eval_test/benchmarks/tasks/mmlu_pro/unitxt b/llm_eval_test/benchmarks/tasks/mmlu_pro/unitxt
@@ -0,0 +1 @@
+../_common/unitxt