diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..52373fe24473b1aa44333d318f578ae6bf04b49b 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..f4685f7c06f99f5057f1b4ce9a33dbba9fe4991c
--- /dev/null
+++ b/config.json
@@ -0,0 +1,35 @@
+{
+ "architectures": [
+ "LlamaForCausalLM"
+ ],
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "bos_token_id": 128000,
+ "eos_token_id": 128001,
+ "head_dim": 128,
+ "hidden_act": "silu",
+ "hidden_size": 4096,
+ "initializer_range": 0.02,
+ "intermediate_size": 14336,
+ "max_position_embeddings": 131072,
+ "mlp_bias": false,
+ "model_type": "llama",
+ "num_attention_heads": 32,
+ "num_hidden_layers": 32,
+ "num_key_value_heads": 8,
+ "pretraining_tp": 1,
+ "rms_norm_eps": 1e-05,
+ "rope_scaling": {
+ "factor": 8.0,
+ "high_freq_factor": 4.0,
+ "low_freq_factor": 1.0,
+ "original_max_position_embeddings": 8192,
+ "rope_type": "llama3"
+ },
+ "rope_theta": 500000.0,
+ "tie_word_embeddings": false,
+ "torch_dtype": "bfloat16",
+ "transformers_version": "4.51.3",
+ "use_cache": true,
+ "vocab_size": 128256
+}
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5b213ecb30558d26009ee82be81660a93a6ac37d
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,9 @@
+{
+ "_from_model_config": true,
+ "bos_token_id": 128000,
+ "do_sample": true,
+ "eos_token_id": 128001,
+ "temperature": 0.6,
+ "top_p": 0.95,
+ "transformers_version": "4.51.3"
+}
diff --git a/mmlu.json/models__v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1__checkpoint_499/results_2025-05-11T21-19-23.629779.json b/mmlu.json/models__v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1__checkpoint_499/results_2025-05-11T21-19-23.629779.json
new file mode 100644
index 0000000000000000000000000000000000000000..e700b99d349e6b7fd9af86a31901969d33a77aea
--- /dev/null
+++ b/mmlu.json/models__v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1__checkpoint_499/results_2025-05-11T21-19-23.629779.json
@@ -0,0 +1,3575 @@
+{
+ "results": {
+ "mmlu": {
+ "acc,none": 0.46360917248255235,
+ "acc_stderr,none": 0.004092816565152735,
+ "alias": "mmlu"
+ },
+ "mmlu_humanities": {
+ "acc,none": 0.41381509032943675,
+ "acc_stderr,none": 0.00698135175083635,
+ "alias": " - humanities"
+ },
+ "mmlu_formal_logic": {
+ "alias": " - formal_logic",
+ "acc,none": 0.4444444444444444,
+ "acc_stderr,none": 0.044444444444444495
+ },
+ "mmlu_high_school_european_history": {
+ "alias": " - high_school_european_history",
+ "acc,none": 0.4666666666666667,
+ "acc_stderr,none": 0.03895658065271846
+ },
+ "mmlu_high_school_us_history": {
+ "alias": " - high_school_us_history",
+ "acc,none": 0.49019607843137253,
+ "acc_stderr,none": 0.03508637358630572
+ },
+ "mmlu_high_school_world_history": {
+ "alias": " - high_school_world_history",
+ "acc,none": 0.5274261603375527,
+ "acc_stderr,none": 0.03249822718301304
+ },
+ "mmlu_international_law": {
+ "alias": " - international_law",
+ "acc,none": 0.5537190082644629,
+ "acc_stderr,none": 0.0453793517794788
+ },
+ "mmlu_jurisprudence": {
+ "alias": " - jurisprudence",
+ "acc,none": 0.5462962962962963,
+ "acc_stderr,none": 0.048129173245368216
+ },
+ "mmlu_logical_fallacies": {
+ "alias": " - logical_fallacies",
+ "acc,none": 0.5214723926380368,
+ "acc_stderr,none": 0.03924746876751129
+ },
+ "mmlu_moral_disputes": {
+ "alias": " - moral_disputes",
+ "acc,none": 0.4653179190751445,
+ "acc_stderr,none": 0.026854257928258886
+ },
+ "mmlu_moral_scenarios": {
+ "alias": " - moral_scenarios",
+ "acc,none": 0.23910614525139665,
+ "acc_stderr,none": 0.014265554192331154
+ },
+ "mmlu_philosophy": {
+ "alias": " - philosophy",
+ "acc,none": 0.4790996784565916,
+ "acc_stderr,none": 0.028373270961069414
+ },
+ "mmlu_prehistory": {
+ "alias": " - prehistory",
+ "acc,none": 0.5648148148148148,
+ "acc_stderr,none": 0.02758600622160771
+ },
+ "mmlu_professional_law": {
+ "alias": " - professional_law",
+ "acc,none": 0.3604954367666232,
+ "acc_stderr,none": 0.012263110237299235
+ },
+ "mmlu_world_religions": {
+ "alias": " - world_religions",
+ "acc,none": 0.6900584795321637,
+ "acc_stderr,none": 0.03546976959393163
+ },
+ "mmlu_other": {
+ "acc,none": 0.502092050209205,
+ "acc_stderr,none": 0.008570757102961374,
+ "alias": " - other"
+ },
+ "mmlu_business_ethics": {
+ "alias": " - business_ethics",
+ "acc,none": 0.53,
+ "acc_stderr,none": 0.05016135580465919
+ },
+ "mmlu_clinical_knowledge": {
+ "alias": " - clinical_knowledge",
+ "acc,none": 0.3584905660377358,
+ "acc_stderr,none": 0.029514703583981765
+ },
+ "mmlu_college_medicine": {
+ "alias": " - college_medicine",
+ "acc,none": 0.3815028901734104,
+ "acc_stderr,none": 0.037038511930995194
+ },
+ "mmlu_global_facts": {
+ "alias": " - global_facts",
+ "acc,none": 0.38,
+ "acc_stderr,none": 0.048783173121456316
+ },
+ "mmlu_human_aging": {
+ "alias": " - human_aging",
+ "acc,none": 0.49327354260089684,
+ "acc_stderr,none": 0.033554765962343545
+ },
+ "mmlu_management": {
+ "alias": " - management",
+ "acc,none": 0.6019417475728155,
+ "acc_stderr,none": 0.04846748253977239
+ },
+ "mmlu_marketing": {
+ "alias": " - marketing",
+ "acc,none": 0.6965811965811965,
+ "acc_stderr,none": 0.030118210106942656
+ },
+ "mmlu_medical_genetics": {
+ "alias": " - medical_genetics",
+ "acc,none": 0.37,
+ "acc_stderr,none": 0.04852365870939099
+ },
+ "mmlu_miscellaneous": {
+ "alias": " - miscellaneous",
+ "acc,none": 0.6998722860791826,
+ "acc_stderr,none": 0.016389249691317418
+ },
+ "mmlu_nutrition": {
+ "alias": " - nutrition",
+ "acc,none": 0.4477124183006536,
+ "acc_stderr,none": 0.028472938478033522
+ },
+ "mmlu_professional_accounting": {
+ "alias": " - professional_accounting",
+ "acc,none": 0.35106382978723405,
+ "acc_stderr,none": 0.028473501272963768
+ },
+ "mmlu_professional_medicine": {
+ "alias": " - professional_medicine",
+ "acc,none": 0.38235294117647056,
+ "acc_stderr,none": 0.02952009569768775
+ },
+ "mmlu_virology": {
+ "alias": " - virology",
+ "acc,none": 0.2891566265060241,
+ "acc_stderr,none": 0.035294868015111155
+ },
+ "mmlu_social_sciences": {
+ "acc,none": 0.5141371465713357,
+ "acc_stderr,none": 0.008870645111287658,
+ "alias": " - social sciences"
+ },
+ "mmlu_econometrics": {
+ "alias": " - econometrics",
+ "acc,none": 0.2631578947368421,
+ "acc_stderr,none": 0.041424397194893596
+ },
+ "mmlu_high_school_geography": {
+ "alias": " - high_school_geography",
+ "acc,none": 0.5353535353535354,
+ "acc_stderr,none": 0.035534363688280626
+ },
+ "mmlu_high_school_government_and_politics": {
+ "alias": " - high_school_government_and_politics",
+ "acc,none": 0.5751295336787565,
+ "acc_stderr,none": 0.035674713352125395
+ },
+ "mmlu_high_school_macroeconomics": {
+ "alias": " - high_school_macroeconomics",
+ "acc,none": 0.4717948717948718,
+ "acc_stderr,none": 0.025310639254933893
+ },
+ "mmlu_high_school_microeconomics": {
+ "alias": " - high_school_microeconomics",
+ "acc,none": 0.4411764705882353,
+ "acc_stderr,none": 0.0322529423239964
+ },
+ "mmlu_high_school_psychology": {
+ "alias": " - high_school_psychology",
+ "acc,none": 0.618348623853211,
+ "acc_stderr,none": 0.020828148517022596
+ },
+ "mmlu_human_sexuality": {
+ "alias": " - human_sexuality",
+ "acc,none": 0.5190839694656488,
+ "acc_stderr,none": 0.043820947055509867
+ },
+ "mmlu_professional_psychology": {
+ "alias": " - professional_psychology",
+ "acc,none": 0.4591503267973856,
+ "acc_stderr,none": 0.020160213617222516
+ },
+ "mmlu_public_relations": {
+ "alias": " - public_relations",
+ "acc,none": 0.4727272727272727,
+ "acc_stderr,none": 0.04782001791380063
+ },
+ "mmlu_security_studies": {
+ "alias": " - security_studies",
+ "acc,none": 0.4448979591836735,
+ "acc_stderr,none": 0.031814251181977865
+ },
+ "mmlu_sociology": {
+ "alias": " - sociology",
+ "acc,none": 0.6218905472636815,
+ "acc_stderr,none": 0.034288678487786564
+ },
+ "mmlu_us_foreign_policy": {
+ "alias": " - us_foreign_policy",
+ "acc,none": 0.74,
+ "acc_stderr,none": 0.0440844002276808
+ },
+ "mmlu_stem": {
+ "acc,none": 0.45068189026324135,
+ "acc_stderr,none": 0.008800554304211464,
+ "alias": " - stem"
+ },
+ "mmlu_abstract_algebra": {
+ "alias": " - abstract_algebra",
+ "acc,none": 0.4,
+ "acc_stderr,none": 0.04923659639173309
+ },
+ "mmlu_anatomy": {
+ "alias": " - anatomy",
+ "acc,none": 0.5407407407407407,
+ "acc_stderr,none": 0.04304979692464241
+ },
+ "mmlu_astronomy": {
+ "alias": " - astronomy",
+ "acc,none": 0.5657894736842105,
+ "acc_stderr,none": 0.0403356566784832
+ },
+ "mmlu_college_biology": {
+ "alias": " - college_biology",
+ "acc,none": 0.4166666666666667,
+ "acc_stderr,none": 0.04122728707651282
+ },
+ "mmlu_college_chemistry": {
+ "alias": " - college_chemistry",
+ "acc,none": 0.39,
+ "acc_stderr,none": 0.04902071300001975
+ },
+ "mmlu_college_computer_science": {
+ "alias": " - college_computer_science",
+ "acc,none": 0.46,
+ "acc_stderr,none": 0.05009082659620332
+ },
+ "mmlu_college_mathematics": {
+ "alias": " - college_mathematics",
+ "acc,none": 0.38,
+ "acc_stderr,none": 0.04878317312145633
+ },
+ "mmlu_college_physics": {
+ "alias": " - college_physics",
+ "acc,none": 0.37254901960784315,
+ "acc_stderr,none": 0.048108401480826346
+ },
+ "mmlu_computer_security": {
+ "alias": " - computer_security",
+ "acc,none": 0.62,
+ "acc_stderr,none": 0.048783173121456316
+ },
+ "mmlu_conceptual_physics": {
+ "alias": " - conceptual_physics",
+ "acc,none": 0.4127659574468085,
+ "acc_stderr,none": 0.03218471141400351
+ },
+ "mmlu_electrical_engineering": {
+ "alias": " - electrical_engineering",
+ "acc,none": 0.5448275862068965,
+ "acc_stderr,none": 0.04149886942192117
+ },
+ "mmlu_elementary_mathematics": {
+ "alias": " - elementary_mathematics",
+ "acc,none": 0.43915343915343913,
+ "acc_stderr,none": 0.025559920550531013
+ },
+ "mmlu_high_school_biology": {
+ "alias": " - high_school_biology",
+ "acc,none": 0.5193548387096775,
+ "acc_stderr,none": 0.02842268740431211
+ },
+ "mmlu_high_school_chemistry": {
+ "alias": " - high_school_chemistry",
+ "acc,none": 0.43842364532019706,
+ "acc_stderr,none": 0.03491207857486519
+ },
+ "mmlu_high_school_computer_science": {
+ "alias": " - high_school_computer_science",
+ "acc,none": 0.47,
+ "acc_stderr,none": 0.050161355804659205
+ },
+ "mmlu_high_school_mathematics": {
+ "alias": " - high_school_mathematics",
+ "acc,none": 0.3925925925925926,
+ "acc_stderr,none": 0.02977384701253297
+ },
+ "mmlu_high_school_physics": {
+ "alias": " - high_school_physics",
+ "acc,none": 0.3708609271523179,
+ "acc_stderr,none": 0.03943966699183629
+ },
+ "mmlu_high_school_statistics": {
+ "alias": " - high_school_statistics",
+ "acc,none": 0.47685185185185186,
+ "acc_stderr,none": 0.03406315360711507
+ },
+ "mmlu_machine_learning": {
+ "alias": " - machine_learning",
+ "acc,none": 0.3125,
+ "acc_stderr,none": 0.043994650575715215
+ }
+ },
+ "groups": {
+ "mmlu": {
+ "acc,none": 0.46360917248255235,
+ "acc_stderr,none": 0.004092816565152735,
+ "alias": "mmlu"
+ },
+ "mmlu_humanities": {
+ "acc,none": 0.41381509032943675,
+ "acc_stderr,none": 0.00698135175083635,
+ "alias": " - humanities"
+ },
+ "mmlu_other": {
+ "acc,none": 0.502092050209205,
+ "acc_stderr,none": 0.008570757102961374,
+ "alias": " - other"
+ },
+ "mmlu_social_sciences": {
+ "acc,none": 0.5141371465713357,
+ "acc_stderr,none": 0.008870645111287658,
+ "alias": " - social sciences"
+ },
+ "mmlu_stem": {
+ "acc,none": 0.45068189026324135,
+ "acc_stderr,none": 0.008800554304211464,
+ "alias": " - stem"
+ }
+ },
+ "group_subtasks": {
+ "mmlu_humanities": [
+ "mmlu_professional_law",
+ "mmlu_high_school_world_history",
+ "mmlu_jurisprudence",
+ "mmlu_philosophy",
+ "mmlu_international_law",
+ "mmlu_logical_fallacies",
+ "mmlu_high_school_us_history",
+ "mmlu_moral_disputes",
+ "mmlu_world_religions",
+ "mmlu_prehistory",
+ "mmlu_formal_logic",
+ "mmlu_moral_scenarios",
+ "mmlu_high_school_european_history"
+ ],
+ "mmlu_social_sciences": [
+ "mmlu_security_studies",
+ "mmlu_sociology",
+ "mmlu_human_sexuality",
+ "mmlu_professional_psychology",
+ "mmlu_high_school_government_and_politics",
+ "mmlu_high_school_macroeconomics",
+ "mmlu_econometrics",
+ "mmlu_high_school_geography",
+ "mmlu_high_school_psychology",
+ "mmlu_us_foreign_policy",
+ "mmlu_high_school_microeconomics",
+ "mmlu_public_relations"
+ ],
+ "mmlu_other": [
+ "mmlu_business_ethics",
+ "mmlu_college_medicine",
+ "mmlu_human_aging",
+ "mmlu_professional_medicine",
+ "mmlu_professional_accounting",
+ "mmlu_global_facts",
+ "mmlu_nutrition",
+ "mmlu_marketing",
+ "mmlu_virology",
+ "mmlu_medical_genetics",
+ "mmlu_management",
+ "mmlu_clinical_knowledge",
+ "mmlu_miscellaneous"
+ ],
+ "mmlu_stem": [
+ "mmlu_elementary_mathematics",
+ "mmlu_high_school_statistics",
+ "mmlu_machine_learning",
+ "mmlu_electrical_engineering",
+ "mmlu_abstract_algebra",
+ "mmlu_high_school_physics",
+ "mmlu_college_computer_science",
+ "mmlu_high_school_biology",
+ "mmlu_high_school_mathematics",
+ "mmlu_college_mathematics",
+ "mmlu_anatomy",
+ "mmlu_conceptual_physics",
+ "mmlu_high_school_computer_science",
+ "mmlu_high_school_chemistry",
+ "mmlu_college_physics",
+ "mmlu_college_chemistry",
+ "mmlu_computer_security",
+ "mmlu_college_biology",
+ "mmlu_astronomy"
+ ],
+ "mmlu": [
+ "mmlu_stem",
+ "mmlu_other",
+ "mmlu_social_sciences",
+ "mmlu_humanities"
+ ]
+ },
+ "configs": {
+ "mmlu_abstract_algebra": {
+ "task": "mmlu_abstract_algebra",
+ "task_alias": "abstract_algebra",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "abstract_algebra",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_anatomy": {
+ "task": "mmlu_anatomy",
+ "task_alias": "anatomy",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "anatomy",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about anatomy.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_astronomy": {
+ "task": "mmlu_astronomy",
+ "task_alias": "astronomy",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "astronomy",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about astronomy.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_business_ethics": {
+ "task": "mmlu_business_ethics",
+ "task_alias": "business_ethics",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "business_ethics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about business ethics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_clinical_knowledge": {
+ "task": "mmlu_clinical_knowledge",
+ "task_alias": "clinical_knowledge",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "clinical_knowledge",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_college_biology": {
+ "task": "mmlu_college_biology",
+ "task_alias": "college_biology",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "college_biology",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about college biology.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_college_chemistry": {
+ "task": "mmlu_college_chemistry",
+ "task_alias": "college_chemistry",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "college_chemistry",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_college_computer_science": {
+ "task": "mmlu_college_computer_science",
+ "task_alias": "college_computer_science",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "college_computer_science",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about college computer science.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_college_mathematics": {
+ "task": "mmlu_college_mathematics",
+ "task_alias": "college_mathematics",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "college_mathematics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_college_medicine": {
+ "task": "mmlu_college_medicine",
+ "task_alias": "college_medicine",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "college_medicine",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about college medicine.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_college_physics": {
+ "task": "mmlu_college_physics",
+ "task_alias": "college_physics",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "college_physics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about college physics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_computer_security": {
+ "task": "mmlu_computer_security",
+ "task_alias": "computer_security",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "computer_security",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about computer security.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_conceptual_physics": {
+ "task": "mmlu_conceptual_physics",
+ "task_alias": "conceptual_physics",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "conceptual_physics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_econometrics": {
+ "task": "mmlu_econometrics",
+ "task_alias": "econometrics",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "econometrics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about econometrics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_electrical_engineering": {
+ "task": "mmlu_electrical_engineering",
+ "task_alias": "electrical_engineering",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "electrical_engineering",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_elementary_mathematics": {
+ "task": "mmlu_elementary_mathematics",
+ "task_alias": "elementary_mathematics",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "elementary_mathematics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_formal_logic": {
+ "task": "mmlu_formal_logic",
+ "task_alias": "formal_logic",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "formal_logic",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about formal logic.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_global_facts": {
+ "task": "mmlu_global_facts",
+ "task_alias": "global_facts",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "global_facts",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about global facts.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_biology": {
+ "task": "mmlu_high_school_biology",
+ "task_alias": "high_school_biology",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_biology",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school biology.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_chemistry": {
+ "task": "mmlu_high_school_chemistry",
+ "task_alias": "high_school_chemistry",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_chemistry",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_computer_science": {
+ "task": "mmlu_high_school_computer_science",
+ "task_alias": "high_school_computer_science",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_computer_science",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_european_history": {
+ "task": "mmlu_high_school_european_history",
+ "task_alias": "high_school_european_history",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_european_history",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school european history.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_geography": {
+ "task": "mmlu_high_school_geography",
+ "task_alias": "high_school_geography",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_geography",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school geography.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_government_and_politics": {
+ "task": "mmlu_high_school_government_and_politics",
+ "task_alias": "high_school_government_and_politics",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_government_and_politics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_macroeconomics": {
+ "task": "mmlu_high_school_macroeconomics",
+ "task_alias": "high_school_macroeconomics",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_macroeconomics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_mathematics": {
+ "task": "mmlu_high_school_mathematics",
+ "task_alias": "high_school_mathematics",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_mathematics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_microeconomics": {
+ "task": "mmlu_high_school_microeconomics",
+ "task_alias": "high_school_microeconomics",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_microeconomics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_physics": {
+ "task": "mmlu_high_school_physics",
+ "task_alias": "high_school_physics",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_physics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school physics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_psychology": {
+ "task": "mmlu_high_school_psychology",
+ "task_alias": "high_school_psychology",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_psychology",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_statistics": {
+ "task": "mmlu_high_school_statistics",
+ "task_alias": "high_school_statistics",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_statistics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_us_history": {
+ "task": "mmlu_high_school_us_history",
+ "task_alias": "high_school_us_history",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_us_history",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school us history.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_high_school_world_history": {
+ "task": "mmlu_high_school_world_history",
+ "task_alias": "high_school_world_history",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "high_school_world_history",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about high school world history.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_human_aging": {
+ "task": "mmlu_human_aging",
+ "task_alias": "human_aging",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "human_aging",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about human aging.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_human_sexuality": {
+ "task": "mmlu_human_sexuality",
+ "task_alias": "human_sexuality",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "human_sexuality",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_international_law": {
+ "task": "mmlu_international_law",
+ "task_alias": "international_law",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "international_law",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about international law.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_jurisprudence": {
+ "task": "mmlu_jurisprudence",
+ "task_alias": "jurisprudence",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "jurisprudence",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_logical_fallacies": {
+ "task": "mmlu_logical_fallacies",
+ "task_alias": "logical_fallacies",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "logical_fallacies",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_machine_learning": {
+ "task": "mmlu_machine_learning",
+ "task_alias": "machine_learning",
+ "tag": "mmlu_stem_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "machine_learning",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about machine learning.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_management": {
+ "task": "mmlu_management",
+ "task_alias": "management",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "management",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about management.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_marketing": {
+ "task": "mmlu_marketing",
+ "task_alias": "marketing",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "marketing",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about marketing.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_medical_genetics": {
+ "task": "mmlu_medical_genetics",
+ "task_alias": "medical_genetics",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "medical_genetics",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_miscellaneous": {
+ "task": "mmlu_miscellaneous",
+ "task_alias": "miscellaneous",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "miscellaneous",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_moral_disputes": {
+ "task": "mmlu_moral_disputes",
+ "task_alias": "moral_disputes",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "moral_disputes",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_moral_scenarios": {
+ "task": "mmlu_moral_scenarios",
+ "task_alias": "moral_scenarios",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "moral_scenarios",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_nutrition": {
+ "task": "mmlu_nutrition",
+ "task_alias": "nutrition",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "nutrition",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about nutrition.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_philosophy": {
+ "task": "mmlu_philosophy",
+ "task_alias": "philosophy",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "philosophy",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about philosophy.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_prehistory": {
+ "task": "mmlu_prehistory",
+ "task_alias": "prehistory",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "prehistory",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about prehistory.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_professional_accounting": {
+ "task": "mmlu_professional_accounting",
+ "task_alias": "professional_accounting",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "professional_accounting",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_professional_law": {
+ "task": "mmlu_professional_law",
+ "task_alias": "professional_law",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "professional_law",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about professional law.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_professional_medicine": {
+ "task": "mmlu_professional_medicine",
+ "task_alias": "professional_medicine",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "professional_medicine",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_professional_psychology": {
+ "task": "mmlu_professional_psychology",
+ "task_alias": "professional_psychology",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "professional_psychology",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_public_relations": {
+ "task": "mmlu_public_relations",
+ "task_alias": "public_relations",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "public_relations",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about public relations.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_security_studies": {
+ "task": "mmlu_security_studies",
+ "task_alias": "security_studies",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "security_studies",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about security studies.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_sociology": {
+ "task": "mmlu_sociology",
+ "task_alias": "sociology",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "sociology",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about sociology.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_us_foreign_policy": {
+ "task": "mmlu_us_foreign_policy",
+ "task_alias": "us_foreign_policy",
+ "tag": "mmlu_social_sciences_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "us_foreign_policy",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_virology": {
+ "task": "mmlu_virology",
+ "task_alias": "virology",
+ "tag": "mmlu_other_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "virology",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about virology.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ },
+ "mmlu_world_religions": {
+ "task": "mmlu_world_religions",
+ "task_alias": "world_religions",
+ "tag": "mmlu_humanities_tasks",
+ "dataset_path": "cais/mmlu",
+ "dataset_name": "world_religions",
+ "dataset_kwargs": {
+ "trust_remote_code": true
+ },
+ "test_split": "test",
+ "fewshot_split": "dev",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about world religions.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "fewshot_config": {
+ "sampler": "first_n"
+ },
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1.0,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ }
+ },
+ "versions": {
+ "mmlu": 2,
+ "mmlu_abstract_algebra": 1.0,
+ "mmlu_anatomy": 1.0,
+ "mmlu_astronomy": 1.0,
+ "mmlu_business_ethics": 1.0,
+ "mmlu_clinical_knowledge": 1.0,
+ "mmlu_college_biology": 1.0,
+ "mmlu_college_chemistry": 1.0,
+ "mmlu_college_computer_science": 1.0,
+ "mmlu_college_mathematics": 1.0,
+ "mmlu_college_medicine": 1.0,
+ "mmlu_college_physics": 1.0,
+ "mmlu_computer_security": 1.0,
+ "mmlu_conceptual_physics": 1.0,
+ "mmlu_econometrics": 1.0,
+ "mmlu_electrical_engineering": 1.0,
+ "mmlu_elementary_mathematics": 1.0,
+ "mmlu_formal_logic": 1.0,
+ "mmlu_global_facts": 1.0,
+ "mmlu_high_school_biology": 1.0,
+ "mmlu_high_school_chemistry": 1.0,
+ "mmlu_high_school_computer_science": 1.0,
+ "mmlu_high_school_european_history": 1.0,
+ "mmlu_high_school_geography": 1.0,
+ "mmlu_high_school_government_and_politics": 1.0,
+ "mmlu_high_school_macroeconomics": 1.0,
+ "mmlu_high_school_mathematics": 1.0,
+ "mmlu_high_school_microeconomics": 1.0,
+ "mmlu_high_school_physics": 1.0,
+ "mmlu_high_school_psychology": 1.0,
+ "mmlu_high_school_statistics": 1.0,
+ "mmlu_high_school_us_history": 1.0,
+ "mmlu_high_school_world_history": 1.0,
+ "mmlu_human_aging": 1.0,
+ "mmlu_human_sexuality": 1.0,
+ "mmlu_humanities": 2,
+ "mmlu_international_law": 1.0,
+ "mmlu_jurisprudence": 1.0,
+ "mmlu_logical_fallacies": 1.0,
+ "mmlu_machine_learning": 1.0,
+ "mmlu_management": 1.0,
+ "mmlu_marketing": 1.0,
+ "mmlu_medical_genetics": 1.0,
+ "mmlu_miscellaneous": 1.0,
+ "mmlu_moral_disputes": 1.0,
+ "mmlu_moral_scenarios": 1.0,
+ "mmlu_nutrition": 1.0,
+ "mmlu_other": 2,
+ "mmlu_philosophy": 1.0,
+ "mmlu_prehistory": 1.0,
+ "mmlu_professional_accounting": 1.0,
+ "mmlu_professional_law": 1.0,
+ "mmlu_professional_medicine": 1.0,
+ "mmlu_professional_psychology": 1.0,
+ "mmlu_public_relations": 1.0,
+ "mmlu_security_studies": 1.0,
+ "mmlu_social_sciences": 2,
+ "mmlu_sociology": 1.0,
+ "mmlu_stem": 2,
+ "mmlu_us_foreign_policy": 1.0,
+ "mmlu_virology": 1.0,
+ "mmlu_world_religions": 1.0
+ },
+ "n-shot": {
+ "mmlu_abstract_algebra": 0,
+ "mmlu_anatomy": 0,
+ "mmlu_astronomy": 0,
+ "mmlu_business_ethics": 0,
+ "mmlu_clinical_knowledge": 0,
+ "mmlu_college_biology": 0,
+ "mmlu_college_chemistry": 0,
+ "mmlu_college_computer_science": 0,
+ "mmlu_college_mathematics": 0,
+ "mmlu_college_medicine": 0,
+ "mmlu_college_physics": 0,
+ "mmlu_computer_security": 0,
+ "mmlu_conceptual_physics": 0,
+ "mmlu_econometrics": 0,
+ "mmlu_electrical_engineering": 0,
+ "mmlu_elementary_mathematics": 0,
+ "mmlu_formal_logic": 0,
+ "mmlu_global_facts": 0,
+ "mmlu_high_school_biology": 0,
+ "mmlu_high_school_chemistry": 0,
+ "mmlu_high_school_computer_science": 0,
+ "mmlu_high_school_european_history": 0,
+ "mmlu_high_school_geography": 0,
+ "mmlu_high_school_government_and_politics": 0,
+ "mmlu_high_school_macroeconomics": 0,
+ "mmlu_high_school_mathematics": 0,
+ "mmlu_high_school_microeconomics": 0,
+ "mmlu_high_school_physics": 0,
+ "mmlu_high_school_psychology": 0,
+ "mmlu_high_school_statistics": 0,
+ "mmlu_high_school_us_history": 0,
+ "mmlu_high_school_world_history": 0,
+ "mmlu_human_aging": 0,
+ "mmlu_human_sexuality": 0,
+ "mmlu_international_law": 0,
+ "mmlu_jurisprudence": 0,
+ "mmlu_logical_fallacies": 0,
+ "mmlu_machine_learning": 0,
+ "mmlu_management": 0,
+ "mmlu_marketing": 0,
+ "mmlu_medical_genetics": 0,
+ "mmlu_miscellaneous": 0,
+ "mmlu_moral_disputes": 0,
+ "mmlu_moral_scenarios": 0,
+ "mmlu_nutrition": 0,
+ "mmlu_philosophy": 0,
+ "mmlu_prehistory": 0,
+ "mmlu_professional_accounting": 0,
+ "mmlu_professional_law": 0,
+ "mmlu_professional_medicine": 0,
+ "mmlu_professional_psychology": 0,
+ "mmlu_public_relations": 0,
+ "mmlu_security_studies": 0,
+ "mmlu_sociology": 0,
+ "mmlu_us_foreign_policy": 0,
+ "mmlu_virology": 0,
+ "mmlu_world_religions": 0
+ },
+ "higher_is_better": {
+ "mmlu": {
+ "acc": true
+ },
+ "mmlu_abstract_algebra": {
+ "acc": true
+ },
+ "mmlu_anatomy": {
+ "acc": true
+ },
+ "mmlu_astronomy": {
+ "acc": true
+ },
+ "mmlu_business_ethics": {
+ "acc": true
+ },
+ "mmlu_clinical_knowledge": {
+ "acc": true
+ },
+ "mmlu_college_biology": {
+ "acc": true
+ },
+ "mmlu_college_chemistry": {
+ "acc": true
+ },
+ "mmlu_college_computer_science": {
+ "acc": true
+ },
+ "mmlu_college_mathematics": {
+ "acc": true
+ },
+ "mmlu_college_medicine": {
+ "acc": true
+ },
+ "mmlu_college_physics": {
+ "acc": true
+ },
+ "mmlu_computer_security": {
+ "acc": true
+ },
+ "mmlu_conceptual_physics": {
+ "acc": true
+ },
+ "mmlu_econometrics": {
+ "acc": true
+ },
+ "mmlu_electrical_engineering": {
+ "acc": true
+ },
+ "mmlu_elementary_mathematics": {
+ "acc": true
+ },
+ "mmlu_formal_logic": {
+ "acc": true
+ },
+ "mmlu_global_facts": {
+ "acc": true
+ },
+ "mmlu_high_school_biology": {
+ "acc": true
+ },
+ "mmlu_high_school_chemistry": {
+ "acc": true
+ },
+ "mmlu_high_school_computer_science": {
+ "acc": true
+ },
+ "mmlu_high_school_european_history": {
+ "acc": true
+ },
+ "mmlu_high_school_geography": {
+ "acc": true
+ },
+ "mmlu_high_school_government_and_politics": {
+ "acc": true
+ },
+ "mmlu_high_school_macroeconomics": {
+ "acc": true
+ },
+ "mmlu_high_school_mathematics": {
+ "acc": true
+ },
+ "mmlu_high_school_microeconomics": {
+ "acc": true
+ },
+ "mmlu_high_school_physics": {
+ "acc": true
+ },
+ "mmlu_high_school_psychology": {
+ "acc": true
+ },
+ "mmlu_high_school_statistics": {
+ "acc": true
+ },
+ "mmlu_high_school_us_history": {
+ "acc": true
+ },
+ "mmlu_high_school_world_history": {
+ "acc": true
+ },
+ "mmlu_human_aging": {
+ "acc": true
+ },
+ "mmlu_human_sexuality": {
+ "acc": true
+ },
+ "mmlu_humanities": {
+ "acc": true
+ },
+ "mmlu_international_law": {
+ "acc": true
+ },
+ "mmlu_jurisprudence": {
+ "acc": true
+ },
+ "mmlu_logical_fallacies": {
+ "acc": true
+ },
+ "mmlu_machine_learning": {
+ "acc": true
+ },
+ "mmlu_management": {
+ "acc": true
+ },
+ "mmlu_marketing": {
+ "acc": true
+ },
+ "mmlu_medical_genetics": {
+ "acc": true
+ },
+ "mmlu_miscellaneous": {
+ "acc": true
+ },
+ "mmlu_moral_disputes": {
+ "acc": true
+ },
+ "mmlu_moral_scenarios": {
+ "acc": true
+ },
+ "mmlu_nutrition": {
+ "acc": true
+ },
+ "mmlu_other": {
+ "acc": true
+ },
+ "mmlu_philosophy": {
+ "acc": true
+ },
+ "mmlu_prehistory": {
+ "acc": true
+ },
+ "mmlu_professional_accounting": {
+ "acc": true
+ },
+ "mmlu_professional_law": {
+ "acc": true
+ },
+ "mmlu_professional_medicine": {
+ "acc": true
+ },
+ "mmlu_professional_psychology": {
+ "acc": true
+ },
+ "mmlu_public_relations": {
+ "acc": true
+ },
+ "mmlu_security_studies": {
+ "acc": true
+ },
+ "mmlu_social_sciences": {
+ "acc": true
+ },
+ "mmlu_sociology": {
+ "acc": true
+ },
+ "mmlu_stem": {
+ "acc": true
+ },
+ "mmlu_us_foreign_policy": {
+ "acc": true
+ },
+ "mmlu_virology": {
+ "acc": true
+ },
+ "mmlu_world_religions": {
+ "acc": true
+ }
+ },
+ "n-samples": {
+ "mmlu_elementary_mathematics": {
+ "original": 378,
+ "effective": 378
+ },
+ "mmlu_high_school_statistics": {
+ "original": 216,
+ "effective": 216
+ },
+ "mmlu_machine_learning": {
+ "original": 112,
+ "effective": 112
+ },
+ "mmlu_electrical_engineering": {
+ "original": 145,
+ "effective": 145
+ },
+ "mmlu_abstract_algebra": {
+ "original": 100,
+ "effective": 100
+ },
+ "mmlu_high_school_physics": {
+ "original": 151,
+ "effective": 151
+ },
+ "mmlu_college_computer_science": {
+ "original": 100,
+ "effective": 100
+ },
+ "mmlu_high_school_biology": {
+ "original": 310,
+ "effective": 310
+ },
+ "mmlu_high_school_mathematics": {
+ "original": 270,
+ "effective": 270
+ },
+ "mmlu_college_mathematics": {
+ "original": 100,
+ "effective": 100
+ },
+ "mmlu_anatomy": {
+ "original": 135,
+ "effective": 135
+ },
+ "mmlu_conceptual_physics": {
+ "original": 235,
+ "effective": 235
+ },
+ "mmlu_high_school_computer_science": {
+ "original": 100,
+ "effective": 100
+ },
+ "mmlu_high_school_chemistry": {
+ "original": 203,
+ "effective": 203
+ },
+ "mmlu_college_physics": {
+ "original": 102,
+ "effective": 102
+ },
+ "mmlu_college_chemistry": {
+ "original": 100,
+ "effective": 100
+ },
+ "mmlu_computer_security": {
+ "original": 100,
+ "effective": 100
+ },
+ "mmlu_college_biology": {
+ "original": 144,
+ "effective": 144
+ },
+ "mmlu_astronomy": {
+ "original": 152,
+ "effective": 152
+ },
+ "mmlu_business_ethics": {
+ "original": 100,
+ "effective": 100
+ },
+ "mmlu_college_medicine": {
+ "original": 173,
+ "effective": 173
+ },
+ "mmlu_human_aging": {
+ "original": 223,
+ "effective": 223
+ },
+ "mmlu_professional_medicine": {
+ "original": 272,
+ "effective": 272
+ },
+ "mmlu_professional_accounting": {
+ "original": 282,
+ "effective": 282
+ },
+ "mmlu_global_facts": {
+ "original": 100,
+ "effective": 100
+ },
+ "mmlu_nutrition": {
+ "original": 306,
+ "effective": 306
+ },
+ "mmlu_marketing": {
+ "original": 234,
+ "effective": 234
+ },
+ "mmlu_virology": {
+ "original": 166,
+ "effective": 166
+ },
+ "mmlu_medical_genetics": {
+ "original": 100,
+ "effective": 100
+ },
+ "mmlu_management": {
+ "original": 103,
+ "effective": 103
+ },
+ "mmlu_clinical_knowledge": {
+ "original": 265,
+ "effective": 265
+ },
+ "mmlu_miscellaneous": {
+ "original": 783,
+ "effective": 783
+ },
+ "mmlu_security_studies": {
+ "original": 245,
+ "effective": 245
+ },
+ "mmlu_sociology": {
+ "original": 201,
+ "effective": 201
+ },
+ "mmlu_human_sexuality": {
+ "original": 131,
+ "effective": 131
+ },
+ "mmlu_professional_psychology": {
+ "original": 612,
+ "effective": 612
+ },
+ "mmlu_high_school_government_and_politics": {
+ "original": 193,
+ "effective": 193
+ },
+ "mmlu_high_school_macroeconomics": {
+ "original": 390,
+ "effective": 390
+ },
+ "mmlu_econometrics": {
+ "original": 114,
+ "effective": 114
+ },
+ "mmlu_high_school_geography": {
+ "original": 198,
+ "effective": 198
+ },
+ "mmlu_high_school_psychology": {
+ "original": 545,
+ "effective": 545
+ },
+ "mmlu_us_foreign_policy": {
+ "original": 100,
+ "effective": 100
+ },
+ "mmlu_high_school_microeconomics": {
+ "original": 238,
+ "effective": 238
+ },
+ "mmlu_public_relations": {
+ "original": 110,
+ "effective": 110
+ },
+ "mmlu_professional_law": {
+ "original": 1534,
+ "effective": 1534
+ },
+ "mmlu_high_school_world_history": {
+ "original": 237,
+ "effective": 237
+ },
+ "mmlu_jurisprudence": {
+ "original": 108,
+ "effective": 108
+ },
+ "mmlu_philosophy": {
+ "original": 311,
+ "effective": 311
+ },
+ "mmlu_international_law": {
+ "original": 121,
+ "effective": 121
+ },
+ "mmlu_logical_fallacies": {
+ "original": 163,
+ "effective": 163
+ },
+ "mmlu_high_school_us_history": {
+ "original": 204,
+ "effective": 204
+ },
+ "mmlu_moral_disputes": {
+ "original": 346,
+ "effective": 346
+ },
+ "mmlu_world_religions": {
+ "original": 171,
+ "effective": 171
+ },
+ "mmlu_prehistory": {
+ "original": 324,
+ "effective": 324
+ },
+ "mmlu_formal_logic": {
+ "original": 126,
+ "effective": 126
+ },
+ "mmlu_moral_scenarios": {
+ "original": 895,
+ "effective": 895
+ },
+ "mmlu_high_school_european_history": {
+ "original": 165,
+ "effective": 165
+ }
+ },
+ "config": {
+ "model": "hf",
+ "model_args": "pretrained=models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499,cache_dir=./.cache,device_map=auto,parallelize=True",
+ "model_num_parameters": 8030261248,
+ "model_dtype": "torch.bfloat16",
+ "model_revision": "main",
+ "model_sha": "",
+ "batch_size": "auto",
+ "batch_sizes": [
+ 32
+ ],
+ "device": null,
+ "use_cache": null,
+ "limit": null,
+ "bootstrap_iters": 100000,
+ "gen_kwargs": null,
+ "random_seed": 0,
+ "numpy_seed": 1234,
+ "torch_seed": 1234,
+ "fewshot_seed": 1234
+ },
+ "git_hash": "bc5e1ba",
+ "date": 1747012301.3390298,
+ "pretty_env_info": "PyTorch version: 2.7.0+cu126\nIs debug build: False\nCUDA used to build PyTorch: 12.6\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 20.04.6 LTS (x86_64)\nGCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0\nClang version: Could not collect\nCMake version: version 3.16.3\nLibc version: glibc-2.31\n\nPython version: 3.9.21 (main, Dec 11 2024, 16:24:11) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.15.0-67-generic-x86_64-with-glibc2.31\nIs CUDA available: True\nCUDA runtime version: 11.8.89\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A6000\nGPU 1: NVIDIA RTX A6000\nGPU 2: NVIDIA RTX A6000\nGPU 3: NVIDIA RTX A6000\nGPU 4: NVIDIA RTX A6000\nGPU 5: NVIDIA RTX A6000\nGPU 6: NVIDIA RTX A6000\nGPU 7: NVIDIA RTX A6000\n\nNvidia driver version: 525.85.12\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.8.1\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_adv_infer.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_adv_train.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_cnn_infer.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_cnn_train.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_ops_infer.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_ops_train.so.8.6.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_adv_infer.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_adv_train.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_cnn_infer.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_cnn_train.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_ops_infer.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_ops_train.so.8.8.0\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 48 bits physical, 48 bits virtual\nCPU(s): 192\nOn-line CPU(s) list: 0-191\nThread(s) per core: 2\nCore(s) per socket: 48\nSocket(s): 2\nNUMA node(s): 2\nVendor ID: AuthenticAMD\nCPU family: 25\nModel: 1\nModel name: AMD EPYC 7643 48-Core Processor\nStepping: 1\nFrequency boost: enabled\nCPU MHz: 1500.000\nCPU max MHz: 3640.9170\nCPU min MHz: 1500.0000\nBogoMIPS: 4591.71\nVirtualization: AMD-V\nL1d cache: 3 MiB\nL1i cache: 3 MiB\nL2 cache: 48 MiB\nL3 cache: 512 MiB\nNUMA node0 CPU(s): 0-47,96-143\nNUMA node1 CPU(s): 48-95,144-191\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, IBPB conditional, IBRS_FW, STIBP always-on, RSB filling, PBRSB-eIBRS Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 invpcid_single hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload vgif v_spec_ctrl umip pku ospke vaes vpclmulqdq rdpid overflow_recov succor smca fsrm\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.2\n[pip3] nvidia-cublas-cu12==12.6.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.6.80\n[pip3] nvidia-cuda-nvrtc-cu12==12.6.77\n[pip3] nvidia-cuda-runtime-cu12==12.6.77\n[pip3] nvidia-cudnn-cu12==9.5.1.17\n[pip3] nvidia-cufft-cu12==11.3.0.4\n[pip3] nvidia-curand-cu12==10.3.7.77\n[pip3] nvidia-cusolver-cu12==11.7.1.2\n[pip3] nvidia-cusparse-cu12==12.5.4.2\n[pip3] nvidia-cusparselt-cu12==0.6.3\n[pip3] nvidia-nccl-cu12==2.26.2\n[pip3] nvidia-nvjitlink-cu12==12.6.85\n[pip3] nvidia-nvtx-cu12==12.6.77\n[pip3] torch==2.7.0\n[pip3] triton==3.3.0\n[conda] numpy 2.0.2 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.6.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.6.80 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.6.77 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.6.77 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.5.1.17 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.0.4 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.7.77 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.1.2 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.4.2 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.3 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.26.2 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.6.85 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.6.77 pypi_0 pypi\n[conda] torch 2.7.0 pypi_0 pypi\n[conda] triton 3.3.0 pypi_0 pypi",
+ "transformers_version": "4.51.3",
+ "lm_eval_version": "0.4.8",
+ "upper_git_hash": null,
+ "tokenizer_pad_token": [
+ "<｜end▁of▁sentence｜>",
+ "128001"
+ ],
+ "tokenizer_eos_token": [
+ "<｜end▁of▁sentence｜>",
+ "128001"
+ ],
+ "tokenizer_bos_token": [
+ "<｜begin▁of▁sentence｜>",
+ "128000"
+ ],
+ "eot_token_id": 128001,
+ "max_length": 131072,
+ "task_hashes": {},
+ "model_source": "hf",
+ "model_name": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "model_name_sanitized": "models__v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1__checkpoint_499",
+ "system_instruction": null,
+ "system_instruction_sha": null,
+ "fewshot_as_multiturn": false,
+ "chat_template": null,
+ "chat_template_sha": null,
+ "start_time": 2701288.578731254,
+ "end_time": 2701800.111117421,
+ "total_evaluation_time_seconds": "511.5323861669749"
+}
\ No newline at end of file
diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0654c2f478ba734d633b5198759b00d9aa32bba1
--- /dev/null
+++ b/model-00001-of-00004.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad17d25b077e39d1791622d0f7e08934beaddec1587967c636f54d8babf546eb
+size 4976698672
diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..51bd0f41c9f82408efe9ca4dc39e7c83ea4a435b
--- /dev/null
+++ b/model-00002-of-00004.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21b8ca8f9ab09417c124d32ba5b9a59bcd417c4594e41a48d3e869a8a328a021
+size 4999802720
diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7c999d2369130a33bc1304de0c5383684e14b479
--- /dev/null
+++ b/model-00003-of-00004.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c15378b8bf8af3ceaa5e7a81372996b5080fe2035fd304b491064f95b8625e2
+size 4915916176
diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ef7942d49abf9498820c5762eaa953dd01dd43f9
--- /dev/null
+++ b/model-00004-of-00004.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13263c27b6e1c82a791559fc2fe27af0748060180c559220d45b93b5fffe239e
+size 1168138808
diff --git a/model.safetensors.index.json b/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..0fd8120f1c6acddc268ebc2583058efaf699a771
--- /dev/null
+++ b/model.safetensors.index.json
@@ -0,0 +1,298 @@
+{
+ "metadata": {
+ "total_size": 16060522496
+ },
+ "weight_map": {
+ "lm_head.weight": "model-00004-of-00004.safetensors",
+ "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+ "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+ "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+ "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+ "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+ "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+ "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+ "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+ "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+ "model.norm.weight": "model-00004-of-00004.safetensors"
+ }
+}
diff --git a/special_tokens_map.json b/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9c5f940affc91084126f072ae65de3f63cc1705
--- /dev/null
+++ b/special_tokens_map.json
@@ -0,0 +1,20 @@
+{
+ "bos_token": {
+ "content": "<｜begin▁of▁sentence｜>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "cls_token": "<｜end▁of▁sentence｜>",
+ "eos_token": {
+ "content": "<｜end▁of▁sentence｜>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "mask_token": "<｜end▁of▁sentence｜>",
+ "pad_token": "<｜end▁of▁sentence｜>",
+ "sep_token": "<｜end▁of▁sentence｜>"
+}
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..7aa75b71f325cefd964f16e028f58c9b26e50042
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff986a623d6d3225e5e6aab1095376967f35557f063e61f1d1cfbb7f487cc0f6
+size 17209809
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..4d190fdc8c6b68bbdb28ab474ef5ce65a8c640d0
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,2070 @@
+{
+ "add_bos_token": true,
+ "add_eos_token": false,
+ "add_prefix_space": null,
+ "added_tokens_decoder": {
+ "128000": {
+ "content": "<｜begin▁of▁sentence｜>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128001": {
+ "content": "<｜end▁of▁sentence｜>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128002": {
+ "content": "<|reserved_special_token_0|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128003": {
+ "content": "<|reserved_special_token_1|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128004": {
+ "content": "<|finetune_right_pad_id|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128005": {
+ "content": "<|reserved_special_token_2|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128006": {
+ "content": "<|start_header_id|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128007": {
+ "content": "<|end_header_id|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128008": {
+ "content": "<|eom_id|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128009": {
+ "content": "<|eot_id|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128010": {
+ "content": "<|python_tag|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128011": {
+ "content": "<｜User｜>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": false
+ },
+ "128012": {
+ "content": "<｜Assistant｜>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": false
+ },
+ "128013": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": false
+ },
+ "128014": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": false
+ },
+ "128015": {
+ "content": "<｜▁pad▁｜>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128016": {
+ "content": "<|reserved_special_token_8|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128017": {
+ "content": "<|reserved_special_token_9|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128018": {
+ "content": "<|reserved_special_token_10|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128019": {
+ "content": "<|reserved_special_token_11|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128020": {
+ "content": "<|reserved_special_token_12|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128021": {
+ "content": "<|reserved_special_token_13|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128022": {
+ "content": "<|reserved_special_token_14|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128023": {
+ "content": "<|reserved_special_token_15|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128024": {
+ "content": "<|reserved_special_token_16|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128025": {
+ "content": "<|reserved_special_token_17|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128026": {
+ "content": "<|reserved_special_token_18|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128027": {
+ "content": "<|reserved_special_token_19|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128028": {
+ "content": "<|reserved_special_token_20|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128029": {
+ "content": "<|reserved_special_token_21|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128030": {
+ "content": "<|reserved_special_token_22|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128031": {
+ "content": "<|reserved_special_token_23|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128032": {
+ "content": "<|reserved_special_token_24|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128033": {
+ "content": "<|reserved_special_token_25|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128034": {
+ "content": "<|reserved_special_token_26|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128035": {
+ "content": "<|reserved_special_token_27|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128036": {
+ "content": "<|reserved_special_token_28|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128037": {
+ "content": "<|reserved_special_token_29|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128038": {
+ "content": "<|reserved_special_token_30|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128039": {
+ "content": "<|reserved_special_token_31|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128040": {
+ "content": "<|reserved_special_token_32|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128041": {
+ "content": "<|reserved_special_token_33|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128042": {
+ "content": "<|reserved_special_token_34|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128043": {
+ "content": "<|reserved_special_token_35|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128044": {
+ "content": "<|reserved_special_token_36|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128045": {
+ "content": "<|reserved_special_token_37|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128046": {
+ "content": "<|reserved_special_token_38|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128047": {
+ "content": "<|reserved_special_token_39|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128048": {
+ "content": "<|reserved_special_token_40|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128049": {
+ "content": "<|reserved_special_token_41|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128050": {
+ "content": "<|reserved_special_token_42|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128051": {
+ "content": "<|reserved_special_token_43|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128052": {
+ "content": "<|reserved_special_token_44|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128053": {
+ "content": "<|reserved_special_token_45|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128054": {
+ "content": "<|reserved_special_token_46|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128055": {
+ "content": "<|reserved_special_token_47|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128056": {
+ "content": "<|reserved_special_token_48|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128057": {
+ "content": "<|reserved_special_token_49|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128058": {
+ "content": "<|reserved_special_token_50|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128059": {
+ "content": "<|reserved_special_token_51|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128060": {
+ "content": "<|reserved_special_token_52|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128061": {
+ "content": "<|reserved_special_token_53|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128062": {
+ "content": "<|reserved_special_token_54|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128063": {
+ "content": "<|reserved_special_token_55|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128064": {
+ "content": "<|reserved_special_token_56|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128065": {
+ "content": "<|reserved_special_token_57|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128066": {
+ "content": "<|reserved_special_token_58|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128067": {
+ "content": "<|reserved_special_token_59|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128068": {
+ "content": "<|reserved_special_token_60|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128069": {
+ "content": "<|reserved_special_token_61|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128070": {
+ "content": "<|reserved_special_token_62|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128071": {
+ "content": "<|reserved_special_token_63|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128072": {
+ "content": "<|reserved_special_token_64|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128073": {
+ "content": "<|reserved_special_token_65|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128074": {
+ "content": "<|reserved_special_token_66|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128075": {
+ "content": "<|reserved_special_token_67|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128076": {
+ "content": "<|reserved_special_token_68|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128077": {
+ "content": "<|reserved_special_token_69|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128078": {
+ "content": "<|reserved_special_token_70|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128079": {
+ "content": "<|reserved_special_token_71|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128080": {
+ "content": "<|reserved_special_token_72|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128081": {
+ "content": "<|reserved_special_token_73|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128082": {
+ "content": "<|reserved_special_token_74|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128083": {
+ "content": "<|reserved_special_token_75|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128084": {
+ "content": "<|reserved_special_token_76|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128085": {
+ "content": "<|reserved_special_token_77|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128086": {
+ "content": "<|reserved_special_token_78|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128087": {
+ "content": "<|reserved_special_token_79|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128088": {
+ "content": "<|reserved_special_token_80|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128089": {
+ "content": "<|reserved_special_token_81|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128090": {
+ "content": "<|reserved_special_token_82|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128091": {
+ "content": "<|reserved_special_token_83|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128092": {
+ "content": "<|reserved_special_token_84|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128093": {
+ "content": "<|reserved_special_token_85|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128094": {
+ "content": "<|reserved_special_token_86|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128095": {
+ "content": "<|reserved_special_token_87|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128096": {
+ "content": "<|reserved_special_token_88|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128097": {
+ "content": "<|reserved_special_token_89|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128098": {
+ "content": "<|reserved_special_token_90|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128099": {
+ "content": "<|reserved_special_token_91|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128100": {
+ "content": "<|reserved_special_token_92|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128101": {
+ "content": "<|reserved_special_token_93|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128102": {
+ "content": "<|reserved_special_token_94|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128103": {
+ "content": "<|reserved_special_token_95|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128104": {
+ "content": "<|reserved_special_token_96|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128105": {
+ "content": "<|reserved_special_token_97|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128106": {
+ "content": "<|reserved_special_token_98|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128107": {
+ "content": "<|reserved_special_token_99|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128108": {
+ "content": "<|reserved_special_token_100|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128109": {
+ "content": "<|reserved_special_token_101|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128110": {
+ "content": "<|reserved_special_token_102|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128111": {
+ "content": "<|reserved_special_token_103|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128112": {
+ "content": "<|reserved_special_token_104|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128113": {
+ "content": "<|reserved_special_token_105|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128114": {
+ "content": "<|reserved_special_token_106|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128115": {
+ "content": "<|reserved_special_token_107|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128116": {
+ "content": "<|reserved_special_token_108|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128117": {
+ "content": "<|reserved_special_token_109|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128118": {
+ "content": "<|reserved_special_token_110|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128119": {
+ "content": "<|reserved_special_token_111|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128120": {
+ "content": "<|reserved_special_token_112|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128121": {
+ "content": "<|reserved_special_token_113|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128122": {
+ "content": "<|reserved_special_token_114|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128123": {
+ "content": "<|reserved_special_token_115|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128124": {
+ "content": "<|reserved_special_token_116|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128125": {
+ "content": "<|reserved_special_token_117|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128126": {
+ "content": "<|reserved_special_token_118|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128127": {
+ "content": "<|reserved_special_token_119|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128128": {
+ "content": "<|reserved_special_token_120|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128129": {
+ "content": "<|reserved_special_token_121|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128130": {
+ "content": "<|reserved_special_token_122|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128131": {
+ "content": "<|reserved_special_token_123|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128132": {
+ "content": "<|reserved_special_token_124|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128133": {
+ "content": "<|reserved_special_token_125|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128134": {
+ "content": "<|reserved_special_token_126|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128135": {
+ "content": "<|reserved_special_token_127|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128136": {
+ "content": "<|reserved_special_token_128|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128137": {
+ "content": "<|reserved_special_token_129|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128138": {
+ "content": "<|reserved_special_token_130|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128139": {
+ "content": "<|reserved_special_token_131|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128140": {
+ "content": "<|reserved_special_token_132|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128141": {
+ "content": "<|reserved_special_token_133|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128142": {
+ "content": "<|reserved_special_token_134|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128143": {
+ "content": "<|reserved_special_token_135|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128144": {
+ "content": "<|reserved_special_token_136|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128145": {
+ "content": "<|reserved_special_token_137|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128146": {
+ "content": "<|reserved_special_token_138|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128147": {
+ "content": "<|reserved_special_token_139|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128148": {
+ "content": "<|reserved_special_token_140|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128149": {
+ "content": "<|reserved_special_token_141|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128150": {
+ "content": "<|reserved_special_token_142|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128151": {
+ "content": "<|reserved_special_token_143|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128152": {
+ "content": "<|reserved_special_token_144|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128153": {
+ "content": "<|reserved_special_token_145|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128154": {
+ "content": "<|reserved_special_token_146|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128155": {
+ "content": "<|reserved_special_token_147|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128156": {
+ "content": "<|reserved_special_token_148|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128157": {
+ "content": "<|reserved_special_token_149|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128158": {
+ "content": "<|reserved_special_token_150|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128159": {
+ "content": "<|reserved_special_token_151|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128160": {
+ "content": "<|reserved_special_token_152|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128161": {
+ "content": "<|reserved_special_token_153|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128162": {
+ "content": "<|reserved_special_token_154|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128163": {
+ "content": "<|reserved_special_token_155|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128164": {
+ "content": "<|reserved_special_token_156|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128165": {
+ "content": "<|reserved_special_token_157|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128166": {
+ "content": "<|reserved_special_token_158|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128167": {
+ "content": "<|reserved_special_token_159|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128168": {
+ "content": "<|reserved_special_token_160|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128169": {
+ "content": "<|reserved_special_token_161|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128170": {
+ "content": "<|reserved_special_token_162|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128171": {
+ "content": "<|reserved_special_token_163|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128172": {
+ "content": "<|reserved_special_token_164|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128173": {
+ "content": "<|reserved_special_token_165|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128174": {
+ "content": "<|reserved_special_token_166|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128175": {
+ "content": "<|reserved_special_token_167|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128176": {
+ "content": "<|reserved_special_token_168|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128177": {
+ "content": "<|reserved_special_token_169|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128178": {
+ "content": "<|reserved_special_token_170|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128179": {
+ "content": "<|reserved_special_token_171|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128180": {
+ "content": "<|reserved_special_token_172|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128181": {
+ "content": "<|reserved_special_token_173|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128182": {
+ "content": "<|reserved_special_token_174|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128183": {
+ "content": "<|reserved_special_token_175|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128184": {
+ "content": "<|reserved_special_token_176|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128185": {
+ "content": "<|reserved_special_token_177|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128186": {
+ "content": "<|reserved_special_token_178|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128187": {
+ "content": "<|reserved_special_token_179|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128188": {
+ "content": "<|reserved_special_token_180|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128189": {
+ "content": "<|reserved_special_token_181|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128190": {
+ "content": "<|reserved_special_token_182|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128191": {
+ "content": "<|reserved_special_token_183|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128192": {
+ "content": "<|reserved_special_token_184|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128193": {
+ "content": "<|reserved_special_token_185|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128194": {
+ "content": "<|reserved_special_token_186|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128195": {
+ "content": "<|reserved_special_token_187|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128196": {
+ "content": "<|reserved_special_token_188|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128197": {
+ "content": "<|reserved_special_token_189|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128198": {
+ "content": "<|reserved_special_token_190|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128199": {
+ "content": "<|reserved_special_token_191|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128200": {
+ "content": "<|reserved_special_token_192|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128201": {
+ "content": "<|reserved_special_token_193|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128202": {
+ "content": "<|reserved_special_token_194|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128203": {
+ "content": "<|reserved_special_token_195|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128204": {
+ "content": "<|reserved_special_token_196|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128205": {
+ "content": "<|reserved_special_token_197|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128206": {
+ "content": "<|reserved_special_token_198|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128207": {
+ "content": "<|reserved_special_token_199|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128208": {
+ "content": "<|reserved_special_token_200|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128209": {
+ "content": "<|reserved_special_token_201|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128210": {
+ "content": "<|reserved_special_token_202|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128211": {
+ "content": "<|reserved_special_token_203|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128212": {
+ "content": "<|reserved_special_token_204|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128213": {
+ "content": "<|reserved_special_token_205|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128214": {
+ "content": "<|reserved_special_token_206|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128215": {
+ "content": "<|reserved_special_token_207|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128216": {
+ "content": "<|reserved_special_token_208|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128217": {
+ "content": "<|reserved_special_token_209|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128218": {
+ "content": "<|reserved_special_token_210|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128219": {
+ "content": "<|reserved_special_token_211|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128220": {
+ "content": "<|reserved_special_token_212|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128221": {
+ "content": "<|reserved_special_token_213|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128222": {
+ "content": "<|reserved_special_token_214|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128223": {
+ "content": "<|reserved_special_token_215|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128224": {
+ "content": "<|reserved_special_token_216|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128225": {
+ "content": "<|reserved_special_token_217|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128226": {
+ "content": "<|reserved_special_token_218|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128227": {
+ "content": "<|reserved_special_token_219|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128228": {
+ "content": "<|reserved_special_token_220|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128229": {
+ "content": "<|reserved_special_token_221|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128230": {
+ "content": "<|reserved_special_token_222|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128231": {
+ "content": "<|reserved_special_token_223|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128232": {
+ "content": "<|reserved_special_token_224|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128233": {
+ "content": "<|reserved_special_token_225|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128234": {
+ "content": "<|reserved_special_token_226|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128235": {
+ "content": "<|reserved_special_token_227|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128236": {
+ "content": "<|reserved_special_token_228|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128237": {
+ "content": "<|reserved_special_token_229|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128238": {
+ "content": "<|reserved_special_token_230|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128239": {
+ "content": "<|reserved_special_token_231|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128240": {
+ "content": "<|reserved_special_token_232|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128241": {
+ "content": "<|reserved_special_token_233|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128242": {
+ "content": "<|reserved_special_token_234|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128243": {
+ "content": "<|reserved_special_token_235|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128244": {
+ "content": "<|reserved_special_token_236|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128245": {
+ "content": "<|reserved_special_token_237|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128246": {
+ "content": "<|reserved_special_token_238|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128247": {
+ "content": "<|reserved_special_token_239|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128248": {
+ "content": "<|reserved_special_token_240|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128249": {
+ "content": "<|reserved_special_token_241|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128250": {
+ "content": "<|reserved_special_token_242|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128251": {
+ "content": "<|reserved_special_token_243|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128252": {
+ "content": "<|reserved_special_token_244|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128253": {
+ "content": "<|reserved_special_token_245|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128254": {
+ "content": "<|reserved_special_token_246|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "128255": {
+ "content": "<|reserved_special_token_247|>",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "bos_token": "<｜begin▁of▁sentence｜>",
+ "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '' in content %}{% set content = content.split('')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>\\n'}}{% endif %}",
+ "clean_up_tokenization_spaces": false,
+ "cls_token": "<｜end▁of▁sentence｜>",
+ "eos_token": "<｜end▁of▁sentence｜>",
+ "extra_special_tokens": {},
+ "legacy": true,
+ "mask_token": "<｜end▁of▁sentence｜>",
+ "model_max_length": 16384,
+ "pad_token": "<｜end▁of▁sentence｜>",
+ "sep_token": "<｜end▁of▁sentence｜>",
+ "sp_model_kwargs": {},
+ "tokenizer_class": "LlamaTokenizerFast",
+ "unk_token": null,
+ "use_default_system_prompt": false
+}
diff --git a/wmdp.json/models__v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1__checkpoint_499/results_2025-05-11T21-10-22.626237.json b/wmdp.json/models__v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1__checkpoint_499/results_2025-05-11T21-10-22.626237.json
new file mode 100644
index 0000000000000000000000000000000000000000..da0fea2108374d4d91a9cbbc27b43e3cdd4ae3fe
--- /dev/null
+++ b/wmdp.json/models__v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1__checkpoint_499/results_2025-05-11T21-10-22.626237.json
@@ -0,0 +1,120 @@
+{
+ "results": {
+ "wmdp_bio": {
+ "alias": "wmdp_bio",
+ "acc,none": 0.3087195600942655,
+ "acc_stderr,none": 0.012952859416638277
+ }
+ },
+ "group_subtasks": {
+ "wmdp_bio": []
+ },
+ "configs": {
+ "wmdp_bio": {
+ "task": "wmdp_bio",
+ "dataset_path": "cais/wmdp",
+ "dataset_name": "wmdp-bio",
+ "test_split": "test",
+ "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:",
+ "doc_to_target": "answer",
+ "unsafe_code": false,
+ "doc_to_choice": [
+ "A",
+ "B",
+ "C",
+ "D"
+ ],
+ "description": "The following are multiple choice questions (with answers) about biology.\n\n",
+ "target_delimiter": " ",
+ "fewshot_delimiter": "\n\n",
+ "num_fewshot": 0,
+ "metric_list": [
+ {
+ "metric": "acc",
+ "aggregation": "mean",
+ "higher_is_better": true
+ }
+ ],
+ "output_type": "multiple_choice",
+ "repeats": 1,
+ "should_decontaminate": false,
+ "metadata": {
+ "version": 1,
+ "pretrained": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "cache_dir": "./.cache",
+ "device_map": "auto",
+ "parallelize": true
+ }
+ }
+ },
+ "versions": {
+ "wmdp_bio": 1
+ },
+ "n-shot": {
+ "wmdp_bio": 0
+ },
+ "higher_is_better": {
+ "wmdp_bio": {
+ "acc": true
+ }
+ },
+ "n-samples": {
+ "wmdp_bio": {
+ "original": 1273,
+ "effective": 1273
+ }
+ },
+ "config": {
+ "model": "hf",
+ "model_args": "pretrained=models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499,cache_dir=./.cache,device_map=auto,parallelize=True",
+ "model_num_parameters": 8030261248,
+ "model_dtype": "torch.bfloat16",
+ "model_revision": "main",
+ "model_sha": "",
+ "batch_size": "auto",
+ "batch_sizes": [
+ 64
+ ],
+ "device": null,
+ "use_cache": null,
+ "limit": null,
+ "bootstrap_iters": 100000,
+ "gen_kwargs": null,
+ "random_seed": 0,
+ "numpy_seed": 1234,
+ "torch_seed": 1234,
+ "fewshot_seed": 1234
+ },
+ "git_hash": "bc5e1ba",
+ "date": 1747012102.8421698,
+ "pretty_env_info": "PyTorch version: 2.7.0+cu126\nIs debug build: False\nCUDA used to build PyTorch: 12.6\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 20.04.6 LTS (x86_64)\nGCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0\nClang version: Could not collect\nCMake version: version 3.16.3\nLibc version: glibc-2.31\n\nPython version: 3.9.21 (main, Dec 11 2024, 16:24:11) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-5.15.0-67-generic-x86_64-with-glibc2.31\nIs CUDA available: True\nCUDA runtime version: 11.8.89\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A6000\nGPU 1: NVIDIA RTX A6000\nGPU 2: NVIDIA RTX A6000\nGPU 3: NVIDIA RTX A6000\nGPU 4: NVIDIA RTX A6000\nGPU 5: NVIDIA RTX A6000\nGPU 6: NVIDIA RTX A6000\nGPU 7: NVIDIA RTX A6000\n\nNvidia driver version: 525.85.12\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.8.1\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.8.1\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_adv_infer.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_adv_train.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_cnn_infer.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_cnn_train.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_ops_infer.so.8.6.0\n/usr/local/cuda-11.8/targets/x86_64-linux/lib/libcudnn_ops_train.so.8.6.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_adv_infer.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_adv_train.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_cnn_infer.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_cnn_train.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_ops_infer.so.8.8.0\n/usr/local/cuda-12.0/targets/x86_64-linux/lib/libcudnn_ops_train.so.8.8.0\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 48 bits physical, 48 bits virtual\nCPU(s): 192\nOn-line CPU(s) list: 0-191\nThread(s) per core: 2\nCore(s) per socket: 48\nSocket(s): 2\nNUMA node(s): 2\nVendor ID: AuthenticAMD\nCPU family: 25\nModel: 1\nModel name: AMD EPYC 7643 48-Core Processor\nStepping: 1\nFrequency boost: enabled\nCPU MHz: 1500.000\nCPU max MHz: 3640.9170\nCPU min MHz: 1500.0000\nBogoMIPS: 4591.71\nVirtualization: AMD-V\nL1d cache: 3 MiB\nL1i cache: 3 MiB\nL2 cache: 48 MiB\nL3 cache: 512 MiB\nNUMA node0 CPU(s): 0-47,96-143\nNUMA node1 CPU(s): 48-95,144-191\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, IBPB conditional, IBRS_FW, STIBP always-on, RSB filling, PBRSB-eIBRS Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 invpcid_single hw_pstate ssbd mba ibrs ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd amd_ppin arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold v_vmsave_vmload vgif v_spec_ctrl umip pku ospke vaes vpclmulqdq rdpid overflow_recov succor smca fsrm\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.2\n[pip3] nvidia-cublas-cu12==12.6.4.1\n[pip3] nvidia-cuda-cupti-cu12==12.6.80\n[pip3] nvidia-cuda-nvrtc-cu12==12.6.77\n[pip3] nvidia-cuda-runtime-cu12==12.6.77\n[pip3] nvidia-cudnn-cu12==9.5.1.17\n[pip3] nvidia-cufft-cu12==11.3.0.4\n[pip3] nvidia-curand-cu12==10.3.7.77\n[pip3] nvidia-cusolver-cu12==11.7.1.2\n[pip3] nvidia-cusparse-cu12==12.5.4.2\n[pip3] nvidia-cusparselt-cu12==0.6.3\n[pip3] nvidia-nccl-cu12==2.26.2\n[pip3] nvidia-nvjitlink-cu12==12.6.85\n[pip3] nvidia-nvtx-cu12==12.6.77\n[pip3] torch==2.7.0\n[pip3] triton==3.3.0\n[conda] numpy 2.0.2 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.6.4.1 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.6.80 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.6.77 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.6.77 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.5.1.17 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.3.0.4 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.7.77 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.7.1.2 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.5.4.2 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.3 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.26.2 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.6.85 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.6.77 pypi_0 pypi\n[conda] torch 2.7.0 pypi_0 pypi\n[conda] triton 3.3.0 pypi_0 pypi",
+ "transformers_version": "4.51.3",
+ "lm_eval_version": "0.4.8",
+ "upper_git_hash": null,
+ "tokenizer_pad_token": [
+ "<｜end▁of▁sentence｜>",
+ "128001"
+ ],
+ "tokenizer_eos_token": [
+ "<｜end▁of▁sentence｜>",
+ "128001"
+ ],
+ "tokenizer_bos_token": [
+ "<｜begin▁of▁sentence｜>",
+ "128000"
+ ],
+ "eot_token_id": 128001,
+ "max_length": 131072,
+ "task_hashes": {},
+ "model_source": "hf",
+ "model_name": "models/v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1/checkpoint_499",
+ "model_name_sanitized": "models__v11_alpha1.4x1.4_lr7.5e-5_data500_reasoning_assistant_assist_loss_1__checkpoint_499",
+ "system_instruction": null,
+ "system_instruction_sha": null,
+ "fewshot_as_multiturn": false,
+ "chat_template": null,
+ "chat_template_sha": null,
+ "start_time": 2701118.729477107,
+ "end_time": 2701259.126140049,
+ "total_evaluation_time_seconds": "140.39666294213384"
+}
\ No newline at end of file