{ "results": { "ifeval": { "alias": "ifeval", "prompt_level_strict_acc,none": 0.7338262476894639, "prompt_level_strict_acc_stderr,none": 0.019018766847290668, "inst_level_strict_acc,none": 0.8105515587529976, "inst_level_strict_acc_stderr,none": "N/A", "prompt_level_loose_acc,none": 0.7985212569316081, "prompt_level_loose_acc_stderr,none": 0.017260802262371536, "inst_level_loose_acc,none": 0.8597122302158273, "inst_level_loose_acc_stderr,none": "N/A" }, "mmlu": { "acc,none": 0.7333527188136086, "acc_stderr,none": 0.0031668230487096466, "alias": "mmlu" }, "mmlu_humanities": { "acc,none": 0.6818278427205101, "acc_stderr,none": 0.006410787630866193, "alias": " - humanities" }, "mmlu_formal_logic": { "alias": " - formal_logic", "acc,none": 0.5873015873015873, "acc_stderr,none": 0.04403438954768178 }, "mmlu_high_school_european_history": { "alias": " - high_school_european_history", "acc,none": 0.8727272727272727, "acc_stderr,none": 0.026024657651656246 }, "mmlu_high_school_us_history": { "alias": " - high_school_us_history", "acc,none": 0.9264705882352942, "acc_stderr,none": 0.0183188558500897 }, "mmlu_high_school_world_history": { "alias": " - high_school_world_history", "acc,none": 0.9071729957805907, "acc_stderr,none": 0.018889750550956697 }, "mmlu_international_law": { "alias": " - international_law", "acc,none": 0.8512396694214877, "acc_stderr,none": 0.03248470083807198 }, "mmlu_jurisprudence": { "alias": " - jurisprudence", "acc,none": 0.8611111111111112, "acc_stderr,none": 0.033432700628696244 }, "mmlu_logical_fallacies": { "alias": " - logical_fallacies", "acc,none": 0.8220858895705522, "acc_stderr,none": 0.03004735765580667 }, "mmlu_moral_disputes": { "alias": " - moral_disputes", "acc,none": 0.7976878612716763, "acc_stderr,none": 0.021628077380196193 }, "mmlu_moral_scenarios": { "alias": " - moral_scenarios", "acc,none": 0.47374301675977654, "acc_stderr,none": 0.016699427672784813 }, "mmlu_philosophy": { "alias": " - philosophy", "acc,none": 0.7491961414790996, "acc_stderr,none": 0.024619771956697113 }, "mmlu_prehistory": { "alias": " - prehistory", "acc,none": 0.8518518518518519, "acc_stderr,none": 0.019766459563597284 }, "mmlu_professional_law": { "alias": " - professional_law", "acc,none": 0.5886571056062582, "acc_stderr,none": 0.012567882673803585 }, "mmlu_world_religions": { "alias": " - world_religions", "acc,none": 0.8421052631578947, "acc_stderr,none": 0.027966785859160903 }, "mmlu_other": { "acc,none": 0.8004505954296749, "acc_stderr,none": 0.006858948971573186, "alias": " - other" }, "mmlu_business_ethics": { "alias": " - business_ethics", "acc,none": 0.79, "acc_stderr,none": 0.040936018074033236 }, "mmlu_clinical_knowledge": { "alias": " - clinical_knowledge", "acc,none": 0.8, "acc_stderr,none": 0.024618298195866552 }, "mmlu_college_medicine": { "alias": " - college_medicine", "acc,none": 0.7514450867052023, "acc_stderr,none": 0.03295304696818318 }, "mmlu_global_facts": { "alias": " - global_facts", "acc,none": 0.55, "acc_stderr,none": 0.05 }, "mmlu_human_aging": { "alias": " - human_aging", "acc,none": 0.7892376681614349, "acc_stderr,none": 0.027373095500540158 }, "mmlu_management": { "alias": " - management", "acc,none": 0.883495145631068, "acc_stderr,none": 0.03176683948640407 }, "mmlu_marketing": { "alias": " - marketing", "acc,none": 0.9102564102564102, "acc_stderr,none": 0.018724301741941646 }, "mmlu_medical_genetics": { "alias": " - medical_genetics", "acc,none": 0.84, "acc_stderr,none": 0.03684529491774706 }, "mmlu_miscellaneous": { "alias": " - miscellaneous", "acc,none": 0.896551724137931, "acc_stderr,none": 0.010890452544691464 }, "mmlu_nutrition": { "alias": " - nutrition", "acc,none": 0.8366013071895425, "acc_stderr,none": 0.02117062301121349 }, "mmlu_professional_accounting": { "alias": " - professional_accounting", "acc,none": 0.6063829787234043, "acc_stderr,none": 0.029144544781596123 }, "mmlu_professional_medicine": { "alias": " - professional_medicine", "acc,none": 0.8602941176470589, "acc_stderr,none": 0.021059408919012472 }, "mmlu_virology": { "alias": " - virology", "acc,none": 0.5060240963855421, "acc_stderr,none": 0.03892212195333041 }, "mmlu_social_sciences": { "acc,none": 0.8492037699057523, "acc_stderr,none": 0.006332316434621314, "alias": " - social sciences" }, "mmlu_econometrics": { "alias": " - econometrics", "acc,none": 0.5964912280701754, "acc_stderr,none": 0.04615186962583705 }, "mmlu_high_school_geography": { "alias": " - high_school_geography", "acc,none": 0.8787878787878788, "acc_stderr,none": 0.02325315795194205 }, "mmlu_high_school_government_and_politics": { "alias": " - high_school_government_and_politics", "acc,none": 0.9533678756476683, "acc_stderr,none": 0.015216761819262604 }, "mmlu_high_school_macroeconomics": { "alias": " - high_school_macroeconomics", "acc,none": 0.7948717948717948, "acc_stderr,none": 0.02047323317355194 }, "mmlu_high_school_microeconomics": { "alias": " - high_school_microeconomics", "acc,none": 0.8613445378151261, "acc_stderr,none": 0.022448264476832656 }, "mmlu_high_school_psychology": { "alias": " - high_school_psychology", "acc,none": 0.9137614678899083, "acc_stderr,none": 0.012035597300116307 }, "mmlu_human_sexuality": { "alias": " - human_sexuality", "acc,none": 0.8244274809160306, "acc_stderr,none": 0.03336820338476075 }, "mmlu_professional_psychology": { "alias": " - professional_psychology", "acc,none": 0.8349673202614379, "acc_stderr,none": 0.01501755079924728 }, "mmlu_public_relations": { "alias": " - public_relations", "acc,none": 0.7545454545454545, "acc_stderr,none": 0.04122066502878281 }, "mmlu_security_studies": { "alias": " - security_studies", "acc,none": 0.8081632653061225, "acc_stderr,none": 0.0252069631542254 }, "mmlu_sociology": { "alias": " - sociology", "acc,none": 0.8955223880597015, "acc_stderr,none": 0.021628920516700654 }, "mmlu_us_foreign_policy": { "alias": " - us_foreign_policy", "acc,none": 0.94, "acc_stderr,none": 0.023868325657594204 }, "mmlu_stem": { "acc,none": 0.6822074215033301, "acc_stderr,none": 0.005542901707611365, "alias": "stem" }, "mmlu_abstract_algebra": { "alias": " - abstract_algebra", "acc,none": 0.49, "acc_stderr,none": 0.05024183937956913 }, "mmlu_anatomy": { "alias": " - anatomy", "acc,none": 0.7851851851851852, "acc_stderr,none": 0.03547854198560821 }, "mmlu_astronomy": { "alias": " - astronomy", "acc,none": 0.9013157894736842, "acc_stderr,none": 0.024270227737522694 }, "mmlu_college_biology": { "alias": " - college_biology", "acc,none": 0.8819444444444444, "acc_stderr,none": 0.026983346503309316 }, "mmlu_college_chemistry": { "alias": " - college_chemistry", "acc,none": 0.52, "acc_stderr,none": 0.05021167315686783 }, "mmlu_college_computer_science": { "alias": " - college_computer_science", "acc,none": 0.62, "acc_stderr,none": 0.04878317312145634 }, "mmlu_college_mathematics": { "alias": " - college_mathematics", "acc,none": 0.4, "acc_stderr,none": 0.0492365963917331 }, "mmlu_college_physics": { "alias": " - college_physics", "acc,none": 0.5490196078431373, "acc_stderr,none": 0.04951218252396262 }, "mmlu_computer_security": { "alias": " - computer_security", "acc,none": 0.76, "acc_stderr,none": 0.04292346959909278 }, "mmlu_conceptual_physics": { "alias": " - conceptual_physics", "acc,none": 0.7787234042553192, "acc_stderr,none": 0.027136349602424097 }, "mmlu_electrical_engineering": { "alias": " - electrical_engineering", "acc,none": 0.6413793103448275, "acc_stderr,none": 0.03996629574876715 }, "mmlu_elementary_mathematics": { "alias": " - elementary_mathematics", "acc,none": 0.6587301587301587, "acc_stderr,none": 0.024419234966819043 }, "mmlu_high_school_biology": { "alias": " - high_school_biology", "acc,none": 0.8903225806451613, "acc_stderr,none": 0.017776778700485205 }, "mmlu_high_school_chemistry": { "alias": " - high_school_chemistry", "acc,none": 0.6896551724137931, "acc_stderr,none": 0.032550867699700996 }, "mmlu_high_school_computer_science": { "alias": " - high_school_computer_science", "acc,none": 0.8, "acc_stderr,none": 0.04020151261036849 }, "mmlu_high_school_mathematics": { "alias": " - high_school_mathematics", "acc,none": 0.37407407407407406, "acc_stderr,none": 0.02950286112895523 }, "mmlu_high_school_physics": { "alias": " - high_school_physics", "acc,none": 0.6688741721854304, "acc_stderr,none": 0.038425817186598696 }, "mmlu_high_school_statistics": { "alias": " - high_school_statistics", "acc,none": 0.7314814814814815, "acc_stderr,none": 0.030225226160012438 }, "mmlu_machine_learning": { "alias": " - machine_learning", "acc,none": 0.5803571428571429, "acc_stderr,none": 0.04684099321077108 } }, "groups": { "mmlu": { "acc,none": 0.7333527188136086, "acc_stderr,none": 0.0031668230487096466, "alias": "mmlu" }, "mmlu_humanities": { "acc,none": 0.6818278427205101, "acc_stderr,none": 0.006410787630866193, "alias": " - humanities" }, "mmlu_other": { "acc,none": 0.8004505954296749, "acc_stderr,none": 0.006858948971573186, "alias": " - other" }, "mmlu_social_sciences": { "acc,none": 0.8492037699057523, "acc_stderr,none": 0.006332316434621314, "alias": " - social sciences" }, "mmlu_stem": { "acc,none": 0.6822074215033301, "acc_stderr,none": 0.005542901707611365, "alias": "stem" } }, "group_subtasks": { "ifeval": [], "mmlu_humanities": [ "mmlu_philosophy", "mmlu_high_school_us_history", "mmlu_logical_fallacies", "mmlu_high_school_european_history", "mmlu_high_school_world_history", "mmlu_jurisprudence", "mmlu_prehistory", "mmlu_world_religions", "mmlu_moral_disputes", "mmlu_professional_law", "mmlu_international_law", "mmlu_formal_logic", "mmlu_moral_scenarios" ], "mmlu_social_sciences": [ "mmlu_high_school_psychology", "mmlu_econometrics", "mmlu_professional_psychology", "mmlu_high_school_microeconomics", "mmlu_security_studies", "mmlu_high_school_government_and_politics", "mmlu_human_sexuality", "mmlu_sociology", "mmlu_public_relations", "mmlu_us_foreign_policy", "mmlu_high_school_geography", "mmlu_high_school_macroeconomics" ], "mmlu_other": [ "mmlu_business_ethics", "mmlu_global_facts", "mmlu_professional_accounting", "mmlu_professional_medicine", "mmlu_miscellaneous", "mmlu_marketing", "mmlu_nutrition", "mmlu_virology", "mmlu_clinical_knowledge", "mmlu_management", "mmlu_college_medicine", "mmlu_medical_genetics", "mmlu_human_aging" ], "mmlu": [ "mmlu_stem", "mmlu_other", "mmlu_social_sciences", "mmlu_humanities" ], "mmlu_stem": [ "mmlu_college_mathematics", "mmlu_college_computer_science", "mmlu_electrical_engineering", "mmlu_astronomy", "mmlu_college_biology", "mmlu_elementary_mathematics", "mmlu_high_school_computer_science", "mmlu_college_chemistry", "mmlu_anatomy", "mmlu_computer_security", "mmlu_high_school_chemistry", "mmlu_abstract_algebra", "mmlu_high_school_mathematics", "mmlu_high_school_biology", "mmlu_machine_learning", "mmlu_college_physics", "mmlu_high_school_physics", "mmlu_high_school_statistics", "mmlu_conceptual_physics" ] }, "configs": { "ifeval": { "task": "ifeval", "dataset_path": "google/IFEval", "test_split": "train", "doc_to_text": "prompt", "doc_to_target": 0, "unsafe_code": false, "process_results": "def process_results(doc, results):\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "prompt_level_strict_acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "inst_level_strict_acc", "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", "higher_is_better": true }, { "metric": "prompt_level_loose_acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "inst_level_loose_acc", "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", "higher_is_better": true } ], "output_type": "generate_until", "generation_kwargs": { "until": [], "do_sample": false, "temperature": 0.0, "max_gen_toks": 1280 }, "repeats": 1, "should_decontaminate": false, "metadata": { "version": 4.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_abstract_algebra": { "task": "mmlu_abstract_algebra", "task_alias": "abstract_algebra", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "abstract_algebra", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_anatomy": { "task": "mmlu_anatomy", "task_alias": "anatomy", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "anatomy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about anatomy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_astronomy": { "task": "mmlu_astronomy", "task_alias": "astronomy", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "astronomy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about astronomy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_business_ethics": { "task": "mmlu_business_ethics", "task_alias": "business_ethics", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "business_ethics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about business ethics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_clinical_knowledge": { "task": "mmlu_clinical_knowledge", "task_alias": "clinical_knowledge", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "clinical_knowledge", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_college_biology": { "task": "mmlu_college_biology", "task_alias": "college_biology", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_biology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college biology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_college_chemistry": { "task": "mmlu_college_chemistry", "task_alias": "college_chemistry", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_chemistry", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_college_computer_science": { "task": "mmlu_college_computer_science", "task_alias": "college_computer_science", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_computer_science", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college computer science.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_college_mathematics": { "task": "mmlu_college_mathematics", "task_alias": "college_mathematics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_mathematics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_college_medicine": { "task": "mmlu_college_medicine", "task_alias": "college_medicine", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_medicine", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college medicine.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_college_physics": { "task": "mmlu_college_physics", "task_alias": "college_physics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "college_physics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about college physics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_computer_security": { "task": "mmlu_computer_security", "task_alias": "computer_security", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "computer_security", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about computer security.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_conceptual_physics": { "task": "mmlu_conceptual_physics", "task_alias": "conceptual_physics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "conceptual_physics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_econometrics": { "task": "mmlu_econometrics", "task_alias": "econometrics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "econometrics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about econometrics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_electrical_engineering": { "task": "mmlu_electrical_engineering", "task_alias": "electrical_engineering", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "electrical_engineering", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_elementary_mathematics": { "task": "mmlu_elementary_mathematics", "task_alias": "elementary_mathematics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "elementary_mathematics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_formal_logic": { "task": "mmlu_formal_logic", "task_alias": "formal_logic", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "formal_logic", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about formal logic.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_global_facts": { "task": "mmlu_global_facts", "task_alias": "global_facts", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "global_facts", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about global facts.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_biology": { "task": "mmlu_high_school_biology", "task_alias": "high_school_biology", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_biology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school biology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_chemistry": { "task": "mmlu_high_school_chemistry", "task_alias": "high_school_chemistry", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_chemistry", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_computer_science": { "task": "mmlu_high_school_computer_science", "task_alias": "high_school_computer_science", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_computer_science", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_european_history": { "task": "mmlu_high_school_european_history", "task_alias": "high_school_european_history", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_european_history", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school european history.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_geography": { "task": "mmlu_high_school_geography", "task_alias": "high_school_geography", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_geography", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school geography.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_government_and_politics": { "task": "mmlu_high_school_government_and_politics", "task_alias": "high_school_government_and_politics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_government_and_politics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_macroeconomics": { "task": "mmlu_high_school_macroeconomics", "task_alias": "high_school_macroeconomics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_macroeconomics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_mathematics": { "task": "mmlu_high_school_mathematics", "task_alias": "high_school_mathematics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_mathematics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_microeconomics": { "task": "mmlu_high_school_microeconomics", "task_alias": "high_school_microeconomics", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_microeconomics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_physics": { "task": "mmlu_high_school_physics", "task_alias": "high_school_physics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_physics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school physics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_psychology": { "task": "mmlu_high_school_psychology", "task_alias": "high_school_psychology", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_psychology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_statistics": { "task": "mmlu_high_school_statistics", "task_alias": "high_school_statistics", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_statistics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_us_history": { "task": "mmlu_high_school_us_history", "task_alias": "high_school_us_history", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_us_history", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school us history.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_high_school_world_history": { "task": "mmlu_high_school_world_history", "task_alias": "high_school_world_history", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "high_school_world_history", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about high school world history.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_human_aging": { "task": "mmlu_human_aging", "task_alias": "human_aging", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "human_aging", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about human aging.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_human_sexuality": { "task": "mmlu_human_sexuality", "task_alias": "human_sexuality", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "human_sexuality", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_international_law": { "task": "mmlu_international_law", "task_alias": "international_law", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "international_law", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about international law.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_jurisprudence": { "task": "mmlu_jurisprudence", "task_alias": "jurisprudence", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "jurisprudence", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_logical_fallacies": { "task": "mmlu_logical_fallacies", "task_alias": "logical_fallacies", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "logical_fallacies", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_machine_learning": { "task": "mmlu_machine_learning", "task_alias": "machine_learning", "tag": "mmlu_stem_tasks", "dataset_path": "cais/mmlu", "dataset_name": "machine_learning", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about machine learning.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_management": { "task": "mmlu_management", "task_alias": "management", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "management", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about management.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_marketing": { "task": "mmlu_marketing", "task_alias": "marketing", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "marketing", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about marketing.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_medical_genetics": { "task": "mmlu_medical_genetics", "task_alias": "medical_genetics", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "medical_genetics", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_miscellaneous": { "task": "mmlu_miscellaneous", "task_alias": "miscellaneous", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "miscellaneous", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_moral_disputes": { "task": "mmlu_moral_disputes", "task_alias": "moral_disputes", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "moral_disputes", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_moral_scenarios": { "task": "mmlu_moral_scenarios", "task_alias": "moral_scenarios", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "moral_scenarios", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_nutrition": { "task": "mmlu_nutrition", "task_alias": "nutrition", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "nutrition", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about nutrition.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_philosophy": { "task": "mmlu_philosophy", "task_alias": "philosophy", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "philosophy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about philosophy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_prehistory": { "task": "mmlu_prehistory", "task_alias": "prehistory", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "prehistory", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about prehistory.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_professional_accounting": { "task": "mmlu_professional_accounting", "task_alias": "professional_accounting", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_accounting", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_professional_law": { "task": "mmlu_professional_law", "task_alias": "professional_law", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_law", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional law.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_professional_medicine": { "task": "mmlu_professional_medicine", "task_alias": "professional_medicine", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_medicine", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_professional_psychology": { "task": "mmlu_professional_psychology", "task_alias": "professional_psychology", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "professional_psychology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_public_relations": { "task": "mmlu_public_relations", "task_alias": "public_relations", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "public_relations", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about public relations.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_security_studies": { "task": "mmlu_security_studies", "task_alias": "security_studies", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "security_studies", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about security studies.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_sociology": { "task": "mmlu_sociology", "task_alias": "sociology", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "sociology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about sociology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_us_foreign_policy": { "task": "mmlu_us_foreign_policy", "task_alias": "us_foreign_policy", "tag": "mmlu_social_sciences_tasks", "dataset_path": "cais/mmlu", "dataset_name": "us_foreign_policy", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_virology": { "task": "mmlu_virology", "task_alias": "virology", "tag": "mmlu_other_tasks", "dataset_path": "cais/mmlu", "dataset_name": "virology", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about virology.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } }, "mmlu_world_religions": { "task": "mmlu_world_religions", "task_alias": "world_religions", "tag": "mmlu_humanities_tasks", "dataset_path": "cais/mmlu", "dataset_name": "world_religions", "test_split": "test", "fewshot_split": "dev", "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", "doc_to_target": "answer", "unsafe_code": false, "doc_to_choice": [ "A", "B", "C", "D" ], "description": "The following are multiple choice questions (with answers) about world religions.\n\n", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "first_n" }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "pretrained": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "tokenizer": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "max_gen_toks": 4096, "max_model_len": 8192, "enable_prefix_caching": true, "enable_chunked_prefill": true, "disable_custom_all_reduce": true, "tensor_parallel_size": 4 } } }, "versions": { "ifeval": 4.0, "mmlu": 2, "mmlu_abstract_algebra": 1.0, "mmlu_anatomy": 1.0, "mmlu_astronomy": 1.0, "mmlu_business_ethics": 1.0, "mmlu_clinical_knowledge": 1.0, "mmlu_college_biology": 1.0, "mmlu_college_chemistry": 1.0, "mmlu_college_computer_science": 1.0, "mmlu_college_mathematics": 1.0, "mmlu_college_medicine": 1.0, "mmlu_college_physics": 1.0, "mmlu_computer_security": 1.0, "mmlu_conceptual_physics": 1.0, "mmlu_econometrics": 1.0, "mmlu_electrical_engineering": 1.0, "mmlu_elementary_mathematics": 1.0, "mmlu_formal_logic": 1.0, "mmlu_global_facts": 1.0, "mmlu_high_school_biology": 1.0, "mmlu_high_school_chemistry": 1.0, "mmlu_high_school_computer_science": 1.0, "mmlu_high_school_european_history": 1.0, "mmlu_high_school_geography": 1.0, "mmlu_high_school_government_and_politics": 1.0, "mmlu_high_school_macroeconomics": 1.0, "mmlu_high_school_mathematics": 1.0, "mmlu_high_school_microeconomics": 1.0, "mmlu_high_school_physics": 1.0, "mmlu_high_school_psychology": 1.0, "mmlu_high_school_statistics": 1.0, "mmlu_high_school_us_history": 1.0, "mmlu_high_school_world_history": 1.0, "mmlu_human_aging": 1.0, "mmlu_human_sexuality": 1.0, "mmlu_humanities": 2, "mmlu_international_law": 1.0, "mmlu_jurisprudence": 1.0, "mmlu_logical_fallacies": 1.0, "mmlu_machine_learning": 1.0, "mmlu_management": 1.0, "mmlu_marketing": 1.0, "mmlu_medical_genetics": 1.0, "mmlu_miscellaneous": 1.0, "mmlu_moral_disputes": 1.0, "mmlu_moral_scenarios": 1.0, "mmlu_nutrition": 1.0, "mmlu_other": 2, "mmlu_philosophy": 1.0, "mmlu_prehistory": 1.0, "mmlu_professional_accounting": 1.0, "mmlu_professional_law": 1.0, "mmlu_professional_medicine": 1.0, "mmlu_professional_psychology": 1.0, "mmlu_public_relations": 1.0, "mmlu_security_studies": 1.0, "mmlu_social_sciences": 2, "mmlu_sociology": 1.0, "mmlu_stem": 2, "mmlu_us_foreign_policy": 1.0, "mmlu_virology": 1.0, "mmlu_world_religions": 1.0 }, "n-shot": { "ifeval": 0, "mmlu_abstract_algebra": 0, "mmlu_anatomy": 0, "mmlu_astronomy": 0, "mmlu_business_ethics": 0, "mmlu_clinical_knowledge": 0, "mmlu_college_biology": 0, "mmlu_college_chemistry": 0, "mmlu_college_computer_science": 0, "mmlu_college_mathematics": 0, "mmlu_college_medicine": 0, "mmlu_college_physics": 0, "mmlu_computer_security": 0, "mmlu_conceptual_physics": 0, "mmlu_econometrics": 0, "mmlu_electrical_engineering": 0, "mmlu_elementary_mathematics": 0, "mmlu_formal_logic": 0, "mmlu_global_facts": 0, "mmlu_high_school_biology": 0, "mmlu_high_school_chemistry": 0, "mmlu_high_school_computer_science": 0, "mmlu_high_school_european_history": 0, "mmlu_high_school_geography": 0, "mmlu_high_school_government_and_politics": 0, "mmlu_high_school_macroeconomics": 0, "mmlu_high_school_mathematics": 0, "mmlu_high_school_microeconomics": 0, "mmlu_high_school_physics": 0, "mmlu_high_school_psychology": 0, "mmlu_high_school_statistics": 0, "mmlu_high_school_us_history": 0, "mmlu_high_school_world_history": 0, "mmlu_human_aging": 0, "mmlu_human_sexuality": 0, "mmlu_international_law": 0, "mmlu_jurisprudence": 0, "mmlu_logical_fallacies": 0, "mmlu_machine_learning": 0, "mmlu_management": 0, "mmlu_marketing": 0, "mmlu_medical_genetics": 0, "mmlu_miscellaneous": 0, "mmlu_moral_disputes": 0, "mmlu_moral_scenarios": 0, "mmlu_nutrition": 0, "mmlu_philosophy": 0, "mmlu_prehistory": 0, "mmlu_professional_accounting": 0, "mmlu_professional_law": 0, "mmlu_professional_medicine": 0, "mmlu_professional_psychology": 0, "mmlu_public_relations": 0, "mmlu_security_studies": 0, "mmlu_sociology": 0, "mmlu_us_foreign_policy": 0, "mmlu_virology": 0, "mmlu_world_religions": 0 }, "higher_is_better": { "ifeval": { "prompt_level_strict_acc": true, "inst_level_strict_acc": true, "prompt_level_loose_acc": true, "inst_level_loose_acc": true }, "mmlu": { "acc": true }, "mmlu_abstract_algebra": { "acc": true }, "mmlu_anatomy": { "acc": true }, "mmlu_astronomy": { "acc": true }, "mmlu_business_ethics": { "acc": true }, "mmlu_clinical_knowledge": { "acc": true }, "mmlu_college_biology": { "acc": true }, "mmlu_college_chemistry": { "acc": true }, "mmlu_college_computer_science": { "acc": true }, "mmlu_college_mathematics": { "acc": true }, "mmlu_college_medicine": { "acc": true }, "mmlu_college_physics": { "acc": true }, "mmlu_computer_security": { "acc": true }, "mmlu_conceptual_physics": { "acc": true }, "mmlu_econometrics": { "acc": true }, "mmlu_electrical_engineering": { "acc": true }, "mmlu_elementary_mathematics": { "acc": true }, "mmlu_formal_logic": { "acc": true }, "mmlu_global_facts": { "acc": true }, "mmlu_high_school_biology": { "acc": true }, "mmlu_high_school_chemistry": { "acc": true }, "mmlu_high_school_computer_science": { "acc": true }, "mmlu_high_school_european_history": { "acc": true }, "mmlu_high_school_geography": { "acc": true }, "mmlu_high_school_government_and_politics": { "acc": true }, "mmlu_high_school_macroeconomics": { "acc": true }, "mmlu_high_school_mathematics": { "acc": true }, "mmlu_high_school_microeconomics": { "acc": true }, "mmlu_high_school_physics": { "acc": true }, "mmlu_high_school_psychology": { "acc": true }, "mmlu_high_school_statistics": { "acc": true }, "mmlu_high_school_us_history": { "acc": true }, "mmlu_high_school_world_history": { "acc": true }, "mmlu_human_aging": { "acc": true }, "mmlu_human_sexuality": { "acc": true }, "mmlu_humanities": { "acc": true }, "mmlu_international_law": { "acc": true }, "mmlu_jurisprudence": { "acc": true }, "mmlu_logical_fallacies": { "acc": true }, "mmlu_machine_learning": { "acc": true }, "mmlu_management": { "acc": true }, "mmlu_marketing": { "acc": true }, "mmlu_medical_genetics": { "acc": true }, "mmlu_miscellaneous": { "acc": true }, "mmlu_moral_disputes": { "acc": true }, "mmlu_moral_scenarios": { "acc": true }, "mmlu_nutrition": { "acc": true }, "mmlu_other": { "acc": true }, "mmlu_philosophy": { "acc": true }, "mmlu_prehistory": { "acc": true }, "mmlu_professional_accounting": { "acc": true }, "mmlu_professional_law": { "acc": true }, "mmlu_professional_medicine": { "acc": true }, "mmlu_professional_psychology": { "acc": true }, "mmlu_public_relations": { "acc": true }, "mmlu_security_studies": { "acc": true }, "mmlu_social_sciences": { "acc": true }, "mmlu_sociology": { "acc": true }, "mmlu_stem": { "acc": true }, "mmlu_us_foreign_policy": { "acc": true }, "mmlu_virology": { "acc": true }, "mmlu_world_religions": { "acc": true } }, "n-samples": { "mmlu_college_mathematics": { "original": 100, "effective": 100 }, "mmlu_college_computer_science": { "original": 100, "effective": 100 }, "mmlu_electrical_engineering": { "original": 145, "effective": 145 }, "mmlu_astronomy": { "original": 152, "effective": 152 }, "mmlu_college_biology": { "original": 144, "effective": 144 }, "mmlu_elementary_mathematics": { "original": 378, "effective": 378 }, "mmlu_high_school_computer_science": { "original": 100, "effective": 100 }, "mmlu_college_chemistry": { "original": 100, "effective": 100 }, "mmlu_anatomy": { "original": 135, "effective": 135 }, "mmlu_computer_security": { "original": 100, "effective": 100 }, "mmlu_high_school_chemistry": { "original": 203, "effective": 203 }, "mmlu_abstract_algebra": { "original": 100, "effective": 100 }, "mmlu_high_school_mathematics": { "original": 270, "effective": 270 }, "mmlu_high_school_biology": { "original": 310, "effective": 310 }, "mmlu_machine_learning": { "original": 112, "effective": 112 }, "mmlu_college_physics": { "original": 102, "effective": 102 }, "mmlu_high_school_physics": { "original": 151, "effective": 151 }, "mmlu_high_school_statistics": { "original": 216, "effective": 216 }, "mmlu_conceptual_physics": { "original": 235, "effective": 235 }, "mmlu_business_ethics": { "original": 100, "effective": 100 }, "mmlu_global_facts": { "original": 100, "effective": 100 }, "mmlu_professional_accounting": { "original": 282, "effective": 282 }, "mmlu_professional_medicine": { "original": 272, "effective": 272 }, "mmlu_miscellaneous": { "original": 783, "effective": 783 }, "mmlu_marketing": { "original": 234, "effective": 234 }, "mmlu_nutrition": { "original": 306, "effective": 306 }, "mmlu_virology": { "original": 166, "effective": 166 }, "mmlu_clinical_knowledge": { "original": 265, "effective": 265 }, "mmlu_management": { "original": 103, "effective": 103 }, "mmlu_college_medicine": { "original": 173, "effective": 173 }, "mmlu_medical_genetics": { "original": 100, "effective": 100 }, "mmlu_human_aging": { "original": 223, "effective": 223 }, "mmlu_high_school_psychology": { "original": 545, "effective": 545 }, "mmlu_econometrics": { "original": 114, "effective": 114 }, "mmlu_professional_psychology": { "original": 612, "effective": 612 }, "mmlu_high_school_microeconomics": { "original": 238, "effective": 238 }, "mmlu_security_studies": { "original": 245, "effective": 245 }, "mmlu_high_school_government_and_politics": { "original": 193, "effective": 193 }, "mmlu_human_sexuality": { "original": 131, "effective": 131 }, "mmlu_sociology": { "original": 201, "effective": 201 }, "mmlu_public_relations": { "original": 110, "effective": 110 }, "mmlu_us_foreign_policy": { "original": 100, "effective": 100 }, "mmlu_high_school_geography": { "original": 198, "effective": 198 }, "mmlu_high_school_macroeconomics": { "original": 390, "effective": 390 }, "mmlu_philosophy": { "original": 311, "effective": 311 }, "mmlu_high_school_us_history": { "original": 204, "effective": 204 }, "mmlu_logical_fallacies": { "original": 163, "effective": 163 }, "mmlu_high_school_european_history": { "original": 165, "effective": 165 }, "mmlu_high_school_world_history": { "original": 237, "effective": 237 }, "mmlu_jurisprudence": { "original": 108, "effective": 108 }, "mmlu_prehistory": { "original": 324, "effective": 324 }, "mmlu_world_religions": { "original": 171, "effective": 171 }, "mmlu_moral_disputes": { "original": 346, "effective": 346 }, "mmlu_professional_law": { "original": 1534, "effective": 1534 }, "mmlu_international_law": { "original": 121, "effective": 121 }, "mmlu_formal_logic": { "original": 126, "effective": 126 }, "mmlu_moral_scenarios": { "original": 895, "effective": 895 }, "ifeval": { "original": 541, "effective": 541 } }, "config": { "model": "vllm", "model_args": "pretrained=/home/dgxuser/workspace/Mango/models/ms3.2-rc,tokenizer=/home/dgxuser/workspace/Mango/models/ms3.2-rc,max_gen_toks=4096,max_model_len=8192,enable_prefix_caching=True,enable_chunked_prefill=True,disable_custom_all_reduce=True,tensor_parallel_size=4", "batch_size": "auto", "batch_sizes": [], "device": null, "use_cache": null, "limit": null, "bootstrap_iters": 100000, "gen_kwargs": null, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": "c0fc7172", "date": 1762263282.7718267, "pretty_env_info": "PyTorch version: 2.8.0+cu128\nIs debug build: False\nCUDA used to build PyTorch: 12.8\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.5 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.22.1\nLibc version: glibc-2.35\n\nPython version: 3.12.10 (main, Apr 9 2025, 04:03:51) [Clang 20.1.0 ] (64-bit runtime)\nPython platform: Linux-5.15.0-1082-nvidia-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.8.93\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.247.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 256\nOn-line CPU(s) list: 0-255\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7742 64-Core Processor\nCPU family: 23\nModel: 49\nThread(s) per core: 2\nCore(s) per socket: 64\nSocket(s): 2\nStepping: 0\nFrequency boost: enabled\nCPU max MHz: 2250.0000\nCPU min MHz: 1500.0000\nBogoMIPS: 4491.47\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sme sev sev_es\nVirtualization: AMD-V\nL1d cache: 4 MiB (128 instances)\nL1i cache: 4 MiB (128 instances)\nL2 cache: 64 MiB (128 instances)\nL3 cache: 512 MiB (32 instances)\nNUMA node(s): 8\nNUMA node0 CPU(s): 0-15,128-143\nNUMA node1 CPU(s): 16-31,144-159\nNUMA node2 CPU(s): 32-47,160-175\nNUMA node3 CPU(s): 48-63,176-191\nNUMA node4 CPU(s): 64-79,192-207\nNUMA node5 CPU(s): 80-95,208-223\nNUMA node6 CPU(s): 96-111,224-239\nNUMA node7 CPU(s): 112-127,240-255\nVulnerability Gather data sampling: Not affected\nVulnerability Indirect target selection: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow: Mitigation; safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.2.6\n[pip3] nvidia-cublas-cu12==12.8.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.8.90\n[pip3] nvidia-cuda-nvrtc-cu12==12.8.93\n[pip3] nvidia-cuda-runtime-cu12==12.8.90\n[pip3] nvidia-cudnn-cu12==9.10.2.21\n[pip3] nvidia-cufft-cu12==11.3.3.83\n[pip3] nvidia-curand-cu12==10.3.9.90\n[pip3] nvidia-cusolver-cu12==11.7.3.90\n[pip3] nvidia-cusparse-cu12==12.5.8.93\n[pip3] nvidia-cusparselt-cu12==0.7.1\n[pip3] nvidia-nccl-cu12==2.27.3\n[pip3] nvidia-nvjitlink-cu12==12.8.93\n[pip3] nvidia-nvtx-cu12==12.8.90\n[pip3] torch==2.8.0\n[pip3] torchaudio==2.8.0\n[pip3] torchvision==0.23.0\n[pip3] triton==3.4.0\n[conda] Could not collect", "transformers_version": "4.57.0", "lm_eval_version": "0.4.9.1", "upper_git_hash": null, "tokenizer_pad_token": [ "", "11" ], "tokenizer_eos_token": [ "", "2" ], "tokenizer_bos_token": [ "", "1" ], "eot_token_id": 2, "max_length": 8192, "task_hashes": {}, "model_source": "vllm", "model_name": "/home/dgxuser/workspace/Mango/models/ms3.2-rc", "model_name_sanitized": "__home__dgxuser__workspace__Mango__models__ms3.2-rc", "system_instruction": null, "system_instruction_sha": null, "fewshot_as_multiturn": false, "chat_template": "{%- set today = strftime_now(\"%Y-%m-%d\") %}\n{%- set default_system_message = \"You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is \" + today + \".\\n\\nWhen you're not sure about some information, you say that you don't have the information and don't make up anything.\\nIf the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \\\"What are some good restaurants around me?\\\" => \\\"Where are you?\\\" or \\\"When is the next flight to Tokyo\\\" => \\\"Where do you travel from?\\\")\" %}\n\n{{- bos_token }}\n\n{%- if messages[0]['role'] == 'system' %}\n {%- if messages[0]['content'] is string %}\n {%- set system_message = messages[0]['content'] %}\n {%- else %}\n {%- set system_message = messages[0]['content'][0]['text'] %}\n {%- endif %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set system_message = default_system_message %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\n\n{%- for message in loop_messages %}\n {%- if message['role'] == 'user' %}\n {%- if message['content'] is string %}\n {{- '[INST]' + message['content'] + '[/INST]' }}\n {%- else %}\n {{- '[INST]' }}\n {%- for block in message['content'] %}\n {%- if block['type'] == 'text' %}\n {{- block['text'] }}\n {%- elif block['type'] in ['image', 'image_url'] %}\n {{- '[IMG]' }}\n {%- else %}\n {{- raise_exception('Only text and image blocks are supported in message content!') }}\n {%- endif %}\n {%- endfor %}\n {{- '[/INST]' }}\n {%- endif %}\n {%- elif message['role'] == 'system' %}\n {%- if message['content'] is string %}\n {{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }}\n {%- else %}\n {{- '[SYSTEM_PROMPT]' + message['content'][0]['text'] + '[/SYSTEM_PROMPT]' }}\n {%- endif %}\n {%- elif message['role'] == 'assistant' %}\n {%- if message['content'] is string %}\n {{- message['content'] + eos_token }}\n {%- else %}\n {{- message['content'][0]['text'] + eos_token }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Only user, system and assistant roles are supported!') }}\n {%- endif %}\n{%- endfor %}", "chat_template_sha": "5f42291cf1b71a8eeae031eafd5bdc669586dbbbebba344a5d641322af152aa7", "start_time": 1278750.799285267, "end_time": 1279587.389985953, "total_evaluation_time_seconds": "836.5907006859779" }