forked from stanford-crfm/helm
-
Notifications
You must be signed in to change notification settings - Fork 0
/
run_moe2.conf
95 lines (79 loc) · 6.26 KB
/
run_moe2.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
entries: [
# {description: "mmlu:model=neurips/local,subject=conceptual_physics,max_eval_instances=19",priority: 1}
# {description: "mmlu:model=neurips/local,subject=elementary_mathematics,max_eval_instances=19",priority: 1}
# medicine_biology
{description: "mmlu:model=neurips/local,subject=anatomy,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=college_medicine,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=college_biology,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=high_school_biology,max_eval_instances=29",priority: 1}
# computer_science
{description: "mmlu:model=neurips/local,subject=college_computer_science,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=high_school_computer_science,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=computer_security,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=electrical_engineering,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=machine_learning,max_eval_instances=29",priority: 1}
# math
{description: "mmlu:model=neurips/local,subject=high_school_mathematics,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=college_mathematics,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=abstract_algebra,max_eval_instances=39",priority: 1}
{description: "mmlu:model=neurips/local,subject=high_school_statistics,max_eval_instances=39",priority: 1}
# physics_chemistry
{description: "mmlu:model=neurips/local,subject=college_chemistry,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=high_school_chemistry,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=high_school_physics,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=college_physics,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=astronomy,max_eval_instances=29",priority: 1}
# formal_reasoning
{description: "mmlu:model=neurips/local,subject=formal_logic,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=logical_fallacies,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=philosophy,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=moral_disputes,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=moral_scenarios,max_eval_instances=29",priority: 1}
# law
{description: "mmlu:model=neurips/local,subject=professional_law,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=international_law,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=jurisprudence,max_eval_instances=39",priority: 1}
# history
{description: "mmlu:model=neurips/local,subject=high_school_european_history,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=high_school_us_history,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=high_school_world_history,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=prehistory,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=world_religions,max_eval_instances=29",priority: 1}
# business
{description: "mmlu:model=neurips/local,subject=business_ethics,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=global_facts,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=management,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=marketing,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=miscellaneous,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=professional_accounting,max_eval_instances=19",priority: 1}
# health
{description: "mmlu:model=neurips/local,subject=nutrition,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=human_aging,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=clinical_knowledge,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=medical_genetics,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=professional_medicine,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=virology,max_eval_instances=19",priority: 1}
# social_studies
{description: "mmlu:model=neurips/local,subject=high_school_government_and_politics,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=high_school_geography,max_eval_instances=19",priority: 1}
{description: "mmlu:model=neurips/local,subject=us_foreign_policy,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=public_relations,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=security_studies,max_eval_instances=29",priority: 1}
# human_behavior
{description: "mmlu:model=neurips/local,subject=high_school_psychology,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=human_sexuality,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=professional_psychology,max_eval_instances=39",priority: 1}
{description: "mmlu:model=neurips/local,subject=sociology,max_eval_instances=39",priority: 1}
# economics
{description: "mmlu:model=neurips/local,subject=high_school_microeconomics,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=econometrics,max_eval_instances=29",priority: 1}
{description: "mmlu:model=neurips/local,subject=high_school_macroeconomics,max_eval_instances=29",priority: 1}
# truthful_qa
{description: "truthful_qa:task=mc_single,model=neurips/local,max_eval_instances=399",priority: 1}
# bbq
{description: "bbq:subject=all,model=neurips/local,max_eval_instances=399",priority: 1}
#cnn
{description: "summarization_cnndm:model=neurips/local,max_eval_instances=99",priority: 1}
# gsm
# {description: "gsm:model=neurips/local,max_eval_instances=50",priority: 1}
]