config.yaml

default: &default
  source: null
  hf_path: null
  hf_name: null
  url: null
  file: null
  paper: null
  annotation: null
  
mmlu:
  <<: *default
  source: huggingface
  hf_path: cais/mmlu
  hf_name: all
  url: https://huggingface.co/datasets/cais/mmlu
  paper: https://arxiv.org/pdf/2009.03300
  
mmlu_pro:
  <<: *default
  source: huggingface
  hf_path: TIGER-Lab/MMLU-Pro
  url: https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro
  paper: https://arxiv.org/pdf/2406.01574
  
agi_eval:
  <<: *default
  source: huggingface
  hf_path:
    - hails/agieval-lsat-rc
    - hails/agieval-logiqa-en
    - hails/agieval-gaokao-english
    - hails/agieval-aqua-rat
    - hails/agieval-sat-math
    - hails/agieval-lsat-ar
    - hails/agieval-lsat-lr
    - hails/agieval-sat-en-without-passage
    - hails/agieval-sat-en
    - hails/agieval-math
  url: https://huggingface.co/hails
  paper: https://arxiv.org/pdf/2304.06364

commonsense_qa:
  <<: *default
  source: huggingface
  hf_path: tau/commonsense_qa
  url: https://huggingface.co/datasets/tau/commonsense_qa
  paper: https://arxiv.org/pdf/1811.00937

winogrande:
  <<: *default
  source: huggingface
  hf_path: allenai/winogrande
  hf_name:
    - winogrande_s
    - winogrande_m
    - winogrande_l
    - winogrande_xl
    - winogrande_debiased
  url: https://huggingface.co/datasets/allenai/winogrande
  paper: https://arxiv.org/pdf/1907.10641

bigbenchhard:
  <<: *default
  source: huggingface
  hf_path: maveriq/bigbenchhard
  hf_name:
    - boolean_expressions
    - causal_judgement
    - date_understanding
    - disambiguation_qa
    - dyck_languages
    - formal_fallacies
    - geometric_shapes
    - hyperbaton
    - logical_deduction_five_objects
    - logical_deduction_seven_objects
    - logical_deduction_three_objects
    - movie_recommendation
    - multistep_arithmetic_two
    - navigate
    - object_counting
    - penguins_in_a_table
    - reasoning_about_colored_objects
    - ruin_names
    - salient_translation_error_detection
    - snarks
    - sports_understanding
    - temporal_sequences
    - tracking_shuffled_objects_five_objects
    - tracking_shuffled_objects_seven_objects
    - tracking_shuffled_objects_three_objects
    - web_of_lies
    - word_sorting
  url: https://huggingface.co/datasets/maveriq/bigbenchhard
  paper: https://arxiv.org/pdf/2210.09261

arc:
  <<: *default
  source: huggingface
  hf_path: allenai/ai2_arc
  hf_name:
    - ARC-Challenge
    - ARC-Easy
  url: https://huggingface.co/datasets/allenai/ai2_arc
  paper: https://arxiv.org/pdf/1803.05457

trivia_qa:
  <<: *default
  source: huggingface
  hf_path: mandarjoshi/trivia_qa
  hf_name:
    - rc
    - rc.nocontext
    - rc.web
    - rc.web.nocontext
    - rc.wikipedia
    - rc.wikipedia.nocontext
    - unfiltered
    - unfiltered.nocontext
  url: https://huggingface.co/datasets/mandarjoshi/trivia_qa
  paper: https://arxiv.org/pdf/1705.03551

squad_v2:
  <<: *default
  source: huggingface
  hf_path: rajpurkar/squad_v2
  url: https://huggingface.co/datasets/rajpurkar/squad_v2
  paper: https://arxiv.org/pdf/1806.03822

quac:
  <<: *default
  source: huggingface
  hf_path: allenai/quac
  url: https://huggingface.co/datasets/allenai/quac
  paper: https://arxiv.org/pdf/1808.07036

boolq:
  <<: *default
  source: huggingface
  hf_path: google/boolq
  url: https://huggingface.co/datasets/google/boolq
  paper: https://arxiv.org/pdf/1905.10044

drop:
  <<: *default
  source: huggingface
  hf_path: ucinlp/drop
  url: https://huggingface.co/datasets/ucinlp/drop
  paper: https://arxiv.org/pdf/1903.00161

haerae:
  <<: *default
  source: huggingface
  hf_path: HAERAE-HUB/HAE_RAE_BENCH_1.1
  hf_name:
  - correct_definition_matching
  - csat_geo
  - csat_law
  - csat_socio
  - date_understanding
  - general_knowledge
  - history
  - loan_words
  - lyrics_denoising
  - proverbs_denoising
  - rare_words
  - standard_nomenclature
  - reading_comprehension
  url: https://huggingface.co/datasets/HAERAE-HUB/HAE_RAE_BENCH_1.1
  paper: https://arxiv.org/pdf/2309.02706

kmmlu:
  <<: *default
  source: huggingface
  hf_path: HAERAE-HUB/KMMLU
  hf_name:
  - Accounting
  - Agricultural-Sciences
  - Aviation-Engineering-and-Maintenance
  - Biology
  - Chemical-Engineering
  - Chemistry
  - Civil-Engineering
  - Computer-Science
  - Construction
  - Criminal-Law
  - Ecology
  - Economics
  - Education
  - Electrical-Engineering
  - Electronics-Engineering
  - Energy-Management
  - Environmental-Science
  - Fashion
  - Food-Processing
  - Gas-Technology-and-Engineering
  - Geomatics
  - Health
  - Industrial-Engineer
  - Information-Technology
  - Interior-Architecture-and-Design
  - Law
  - Machine-Design-and-Manufacturing
  - Management
  - Maritime-Engineering
  - Marketing
  - Materials-Engineering
  - Mechanical-Engineering
  - Nondestructive-Testing
  - Patent
  - Political-Science-and-Sociology
  - Psychology
  - Public-Safety
  - Railway-and-Automotive-Engineering
  - Real-Estate
  - Refrigerating-Machinery
  - Social-Welfare
  - Taxation
  - Telecommunications-and-Wireless-Technology
  - Korean-History
  - Math
  url: https://huggingface.co/datasets/HAERAE-HUB/KMMLU
  paper: https://arxiv.org/pdf/2402.11548

hellaswag:
  <<: *default
  source: huggingface
  hf_path: Rowan/hellaswag
  url: https://huggingface.co/datasets/Rowan/hellaswag
  paper: https://arxiv.org/pdf/1905.07830

glue:
  <<: *default
  source: huggingface
  hf_path: nyu-mll/glue
  hf_name:
    - ax
    - cola
    - mnli_matched
    - mnli_mismatched
    - mnli
    - mrpc
    - qnli
    - qqp
    - rte
    - sst2
    - stsb
    - wnli
  url: https://huggingface.co/datasets/nyu-mll/glue
  paper: https://openreview.net/pdf?id=rJ4km2R5t7

bigbench:
  <<: *default
  source: huggingface
  hf_path: google/bigbench
  hf_name:
    - abstract_narrative_understanding
    - anachronisms
    - analogical_similarity
    - analytic_entailment
    - arithmetic
    - ascii_word_recognition
    - authorship_verification
    - auto_categorization
    - auto_debugging
    - bbq_lite_json
    - bridging_anaphora_resolution_barqa
    - causal_judgment
    - cause_and_effect
    - checkmate_in_one
    - chess_state_tracking
    - chinese_remainder_theorem
    - cifar10_classification
    - code_line_description
    - codenames
    - color
    - common_morpheme
    - conceptual_combinations
    - conlang_translation
    - contextual_parametric_knowledge_conflicts
    - crash_blossom
    - crass_ai
    - cryobiology_spanish
    - cryptonite
    - cs_algorithms
    - dark_humor_detection
    - date_understanding
    - disambiguation_qa
    - discourse_marker_prediction
    - disfl_qa
    - dyck_languages
    - elementary_math_qa
    - emoji_movie
    - emojis_emotion_prediction
    - empirical_judgments
    - english_proverbs
    - english_russian_proverbs
    - entailed_polarity
    - entailed_polarity_hindi
    - epistemic_reasoning
    - evaluating_information_essentiality
    - fact_checker
    - fantasy_reasoning
    - few_shot_nlg
    - figure_of_speech_detection
    - formal_fallacies_syllogisms_negation
    - gem
    - gender_inclusive_sentences_german
    - general_knowledge
    - geometric_shapes
    - goal_step_wikihow
    - gre_reading_comprehension
    - hhh_alignment
    - hindi_question_answering
    - hindu_knowledge
    - hinglish_toxicity
    - human_organs_senses
    - hyperbaton
    - identify_math_theorems
    - identify_odd_metaphor
    - implicatures
    - implicit_relations
    - intent_recognition
    - international_phonetic_alphabet_nli
    - international_phonetic_alphabet_transliterate
    - intersect_geometry
    - irony_identification
    - kanji_ascii
    - kannada
    - key_value_maps
    - known_unknowns
    - language_games
    - language_identification
    - linguistic_mappings
    - linguistics_puzzles
    - list_functions
    - logic_grid_puzzle
    - logical_args
    - logical_deduction
    - logical_fallacy_detection
    - logical_sequence
    - mathematical_induction
    - matrixshapes
    - metaphor_boolean
    - metaphor_understanding
    - minute_mysteries_qa
    - misconceptions
    - misconceptions_russian
    - mnist_ascii
    - modified_arithmetic
    - moral_permissibility
    - movie_dialog_same_or_different
    - movie_recommendation
    - mult_data_wrangling
    - multiemo
    - natural_instructions
    - navigate
    - nonsense_words_grammar
    - novel_concepts
    - object_counting
    - odd_one_out
    - operators
    - paragraph_segmentation
    - parsinlu_qa
    - parsinlu_reading_comprehension
    - penguins_in_a_table
    - periodic_elements
    - persian_idioms
    - phrase_relatedness
    - physical_intuition
    - physics
    - physics_questions
    - play_dialog_same_or_different
    - polish_sequence_labeling
    - presuppositions_as_nli
    - qa_wikidata
    - question_selection
    - real_or_fake_text
    - reasoning_about_colored_objects
    - repeat_copy_logic
    - rephrase
    - riddle_sense
    - ruin_names
    - salient_translation_error_detection
    - scientific_press_release
    - semantic_parsing_in_context_sparc
    - semantic_parsing_spider
    - sentence_ambiguity
    - similarities_abstraction
    - simp_turing_concept
    - simple_arithmetic_json
    - simple_arithmetic_json_multiple_choice
    - simple_arithmetic_json_subtasks
    - simple_arithmetic_multiple_targets_json
    - simple_ethical_questions
    - simple_text_editing
    - snarks
    - social_iqa
    - social_support
    - sports_understanding
    - strange_stories
    - strategyqa
    - sufficient_information
    - suicide_risk
    - swahili_english_proverbs
    - swedish_to_german_proverbs
    - symbol_interpretation
    - temporal_sequences
    - tense
    - timedial
    - topical_chat
    - tracking_shuffled_objects
    - understanding_fables
    - undo_permutation
    - unit_conversion
    - unit_interpretation
    - unnatural_in_context_learning
    - vitaminc_fact_verification
    - what_is_the_tao
    - which_wiki_edit
    - winowhy
    - word_sorting
    - word_unscrambling
  url: https://huggingface.co/datasets/google/bigbench
  paper: https://arxiv.org/pdf/2206.04615

logiqa:
  <<: *default
  source: huggingface
  hf_path: EleutherAI/logiqa
  hf_name:
  url: https://huggingface.co/datasets/EleutherAI/logiqa
  paper: https://arxiv.org/pdf/2007.08124

lambada:
  <<: *default
  source: huggingface
  hf_path: EleutherAI/lambada_openai
  url: https://huggingface.co/datasets/EleutherAI/lambada_openai
  paper: https://arxiv.org/pdf/1606.06031

openbookqa:
  <<: *default
  source: huggingface
  hf_path: allenai/openbookqa
  url: https://huggingface.co/datasets/allenai/openbookqa
  paper: https://arxiv.org/pdf/1809.02789

math:
  <<: *default
  source: huggingface
  hf_path: lighteval/MATH
  url: https://huggingface.co/datasets/lighteval/MATH
  paper: https://arxiv.org/pdf/2103.03874

gpqa:
  <<: *default
  source: huggingface
  hf_path: Idavidrein/gpqa
  hf_name:
    - gpqa_extended
    - gpqa_main
    - gpqa_diamond
    - gpqa_experts
  url: https://huggingface.co/datasets/Idavidrein/gpqa
  paper: https://arxiv.org/pdf/2311.12022

human_eval:
  <<: *default
  source: huggingface
  hf_path: openai/openai_humaneval
  url: https://huggingface.co/datasets/openai/openai_humaneval
  paper: https://arxiv.org/pdf/2107.03374

gsm8k:
  <<: *default
  source: huggingface
  hf_path: openai/gsm8k
  hf_name:
    - main
    - socratic
  url: https://huggingface.co/datasets/openai/gsm8k
  paper: https://arxiv.org/pdf/2110.14168

mbpp:
  <<: *default
  source: huggingface
  hf_path: google-research-datasets/mbpp
  hf_name:
    - sanitized
  url: https://huggingface.co/datasets/google-research-datasets/mbpp
  paper: https://arxiv.org/pdf/2108.07732

race:
  <<: *default
  source: huggingface
  hf_path: EleutherAI/race
  hf_name:
    - high
  url: https://huggingface.co/datasets/EleutherAI/race
  paper: https://arxiv.org/pdf/1704.04683

natural_questions:
  <<: *default
  source: huggingface
  hf_path: google-research-datasets/natural_questions
  hf_name:
    - default
    - dev
  url: https://huggingface.co/datasets/google-research-datasets/natural_questions
  paper: https://direct.mit.edu/tacl/article/doi/10.1162/tacl_a_00276/43518/Natural-Questions-A-Benchmark-for-Question

super_glue:
  <<: *default
  source: huggingface
  hf_path: aps/super_glue
  hf_name:
    - boolq
    - cb
    - copa
    - multirc
    - record
    - rte
    - wic
    - wsc
    - wsc.fixed
    - axb
    - axg
  url: https://huggingface.co/datasets/aps/super_glue
  paper: https://arxiv.org/pdf/1905.00537