Upload 8 files

config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 256,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 16,
+  "pad_token_id": 1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.32.1",
+  "use_cache": true,
+  "vocab_size": 16000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": 1,
+  "transformers_version": "4.32.1"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d14ec41b42d9451cb10fac16f4fb4921325319f5a5b935787b04a1a72835d17
+size 62211542

results.json

@@ -0,0 +1,2247 @@
+{
+  "results": {
+    "blimp": {
+      "acc,none": 0.659910447761194,
+      "acc_stderr,none": 0.16415593005353757,
+      "alias": "blimp"
+    },
+    "blimp_adjunct_island": {
+      "acc,none": 0.609,
+      "acc_stderr,none": 0.015438826294681782,
+      "alias": " - blimp_adjunct_island"
+    },
+    "blimp_anaphor_gender_agreement": {
+      "acc,none": 0.68,
+      "acc_stderr,none": 0.01475865230357487,
+      "alias": " - blimp_anaphor_gender_agreement"
+    },
+    "blimp_anaphor_number_agreement": {
+      "acc,none": 0.854,
+      "acc_stderr,none": 0.011171786285496496,
+      "alias": " - blimp_anaphor_number_agreement"
+    },
+    "blimp_animate_subject_passive": {
+      "acc,none": 0.67,
+      "acc_stderr,none": 0.014876872027456732,
+      "alias": " - blimp_animate_subject_passive"
+    },
+    "blimp_animate_subject_trans": {
+      "acc,none": 0.707,
+      "acc_stderr,none": 0.014399942998441275,
+      "alias": " - blimp_animate_subject_trans"
+    },
+    "blimp_causative": {
+      "acc,none": 0.613,
+      "acc_stderr,none": 0.015410011955493932,
+      "alias": " - blimp_causative"
+    },
+    "blimp_complex_NP_island": {
+      "acc,none": 0.532,
+      "acc_stderr,none": 0.015786868759359005,
+      "alias": " - blimp_complex_NP_island"
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "acc,none": 0.443,
+      "acc_stderr,none": 0.0157161699532041,
+      "alias": " - blimp_coordinate_structure_constraint_complex_left_branch"
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "acc,none": 0.591,
+      "acc_stderr,none": 0.015555094373257942,
+      "alias": " - blimp_coordinate_structure_constraint_object_extraction"
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "acc,none": 0.917,
+      "acc_stderr,none": 0.00872852720607479,
+      "alias": " - blimp_determiner_noun_agreement_1"
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "acc,none": 0.929,
+      "acc_stderr,none": 0.008125578442487909,
+      "alias": " - blimp_determiner_noun_agreement_2"
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "acc,none": 0.74,
+      "acc_stderr,none": 0.013877773329774166,
+      "alias": " - blimp_determiner_noun_agreement_irregular_1"
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "acc,none": 0.838,
+      "acc_stderr,none": 0.011657267771304415,
+      "alias": " - blimp_determiner_noun_agreement_irregular_2"
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "acc,none": 0.855,
+      "acc_stderr,none": 0.011139977517890145,
+      "alias": " - blimp_determiner_noun_agreement_with_adj_2"
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "acc,none": 0.795,
+      "acc_stderr,none": 0.01277255409611311,
+      "alias": " - blimp_determiner_noun_agreement_with_adj_irregular_1"
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "acc,none": 0.803,
+      "acc_stderr,none": 0.012583693787968132,
+      "alias": " - blimp_determiner_noun_agreement_with_adj_irregular_2"
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "acc,none": 0.886,
+      "acc_stderr,none": 0.010055103435823335,
+      "alias": " - blimp_determiner_noun_agreement_with_adjective_1"
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "acc,none": 0.506,
+      "acc_stderr,none": 0.015818160898606715,
+      "alias": " - blimp_distractor_agreement_relational_noun"
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "acc,none": 0.477,
+      "acc_stderr,none": 0.0158025542467261,
+      "alias": " - blimp_distractor_agreement_relative_clause"
+    },
+    "blimp_drop_argument": {
+      "acc,none": 0.71,
+      "acc_stderr,none": 0.014356395999905685,
+      "alias": " - blimp_drop_argument"
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "acc,none": 0.588,
+      "acc_stderr,none": 0.015572363292015095,
+      "alias": " - blimp_ellipsis_n_bar_1"
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "acc,none": 0.543,
+      "acc_stderr,none": 0.01576069159013638,
+      "alias": " - blimp_ellipsis_n_bar_2"
+    },
+    "blimp_existential_there_object_raising": {
+      "acc,none": 0.673,
+      "acc_stderr,none": 0.014842213153411244,
+      "alias": " - blimp_existential_there_object_raising"
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "acc,none": 0.912,
+      "acc_stderr,none": 0.00896305396259209,
+      "alias": " - blimp_existential_there_quantifiers_1"
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "acc,none": 0.724,
+      "acc_stderr,none": 0.014142984975740663,
+      "alias": " - blimp_existential_there_quantifiers_2"
+    },
+    "blimp_existential_there_subject_raising": {
+      "acc,none": 0.765,
+      "acc_stderr,none": 0.01341472903024712,
+      "alias": " - blimp_existential_there_subject_raising"
+    },
+    "blimp_expletive_it_object_raising": {
+      "acc,none": 0.64,
+      "acc_stderr,none": 0.015186527932040122,
+      "alias": " - blimp_expletive_it_object_raising"
+    },
+    "blimp_inchoative": {
+      "acc,none": 0.551,
+      "acc_stderr,none": 0.01573679276875202,
+      "alias": " - blimp_inchoative"
+    },
+    "blimp_intransitive": {
+      "acc,none": 0.664,
+      "acc_stderr,none": 0.014944140233795021,
+      "alias": " - blimp_intransitive"
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "acc,none": 0.76,
+      "acc_stderr,none": 0.013512312258920826,
+      "alias": " - blimp_irregular_past_participle_adjectives"
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "acc,none": 0.799,
+      "acc_stderr,none": 0.01267910721461733,
+      "alias": " - blimp_irregular_past_participle_verbs"
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "acc,none": 0.681,
+      "acc_stderr,none": 0.014746404865473477,
+      "alias": " - blimp_irregular_plural_subject_verb_agreement_1"
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "acc,none": 0.783,
+      "acc_stderr,none": 0.01304151375727071,
+      "alias": " - blimp_irregular_plural_subject_verb_agreement_2"
+    },
+    "blimp_left_branch_island_echo_question": {
+      "acc,none": 0.434,
+      "acc_stderr,none": 0.015680876566375054,
+      "alias": " - blimp_left_branch_island_echo_question"
+    },
+    "blimp_left_branch_island_simple_question": {
+      "acc,none": 0.559,
+      "acc_stderr,none": 0.01570877989424268,
+      "alias": " - blimp_left_branch_island_simple_question"
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "acc,none": 0.271,
+      "acc_stderr,none": 0.014062601350986186,
+      "alias": " - blimp_matrix_question_npi_licensor_present"
+    },
+    "blimp_npi_present_1": {
+      "acc,none": 0.472,
+      "acc_stderr,none": 0.015794475789511476,
+      "alias": " - blimp_npi_present_1"
+    },
+    "blimp_npi_present_2": {
+      "acc,none": 0.526,
+      "acc_stderr,none": 0.015797897758042766,
+      "alias": " - blimp_npi_present_2"
+    },
+    "blimp_only_npi_licensor_present": {
+      "acc,none": 0.576,
+      "acc_stderr,none": 0.01563548747140519,
+      "alias": " - blimp_only_npi_licensor_present"
+    },
+    "blimp_only_npi_scope": {
+      "acc,none": 0.621,
+      "acc_stderr,none": 0.015349091002225349,
+      "alias": " - blimp_only_npi_scope"
+    },
+    "blimp_passive_1": {
+      "acc,none": 0.797,
+      "acc_stderr,none": 0.012726073744598288,
+      "alias": " - blimp_passive_1"
+    },
+    "blimp_passive_2": {
+      "acc,none": 0.756,
+      "acc_stderr,none": 0.01358854843788143,
+      "alias": " - blimp_passive_2"
+    },
+    "blimp_principle_A_c_command": {
+      "acc,none": 0.561,
+      "acc_stderr,none": 0.01570113134540077,
+      "alias": " - blimp_principle_A_c_command"
+    },
+    "blimp_principle_A_case_1": {
+      "acc,none": 0.997,
+      "acc_stderr,none": 0.001730316154346938,
+      "alias": " - blimp_principle_A_case_1"
+    },
+    "blimp_principle_A_case_2": {
+      "acc,none": 0.745,
+      "acc_stderr,none": 0.013790038620872845,
+      "alias": " - blimp_principle_A_case_2"
+    },
+    "blimp_principle_A_domain_1": {
+      "acc,none": 0.932,
+      "acc_stderr,none": 0.007964887911291603,
+      "alias": " - blimp_principle_A_domain_1"
+    },
+    "blimp_principle_A_domain_2": {
+      "acc,none": 0.615,
+      "acc_stderr,none": 0.015395194445410805,
+      "alias": " - blimp_principle_A_domain_2"
+    },
+    "blimp_principle_A_domain_3": {
+      "acc,none": 0.484,
+      "acc_stderr,none": 0.015811198373114878,
+      "alias": " - blimp_principle_A_domain_3"
+    },
+    "blimp_principle_A_reconstruction": {
+      "acc,none": 0.392,
+      "acc_stderr,none": 0.015445859463771302,
+      "alias": " - blimp_principle_A_reconstruction"
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "acc,none": 0.765,
+      "acc_stderr,none": 0.013414729030247116,
+      "alias": " - blimp_regular_plural_subject_verb_agreement_1"
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "acc,none": 0.729,
+      "acc_stderr,none": 0.014062601350986182,
+      "alias": " - blimp_regular_plural_subject_verb_agreement_2"
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "acc,none": 0.848,
+      "acc_stderr,none": 0.01135891830347528,
+      "alias": " - blimp_sentential_negation_npi_licensor_present"
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "acc,none": 0.426,
+      "acc_stderr,none": 0.01564508768811381,
+      "alias": " - blimp_sentential_negation_npi_scope"
+    },
+    "blimp_sentential_subject_island": {
+      "acc,none": 0.391,
+      "acc_stderr,none": 0.015438826294681787,
+      "alias": " - blimp_sentential_subject_island"
+    },
+    "blimp_superlative_quantifiers_1": {
+      "acc,none": 0.517,
+      "acc_stderr,none": 0.015810153729833427,
+      "alias": " - blimp_superlative_quantifiers_1"
+    },
+    "blimp_superlative_quantifiers_2": {
+      "acc,none": 0.58,
+      "acc_stderr,none": 0.015615500115072956,
+      "alias": " - blimp_superlative_quantifiers_2"
+    },
+    "blimp_tough_vs_raising_1": {
+      "acc,none": 0.336,
+      "acc_stderr,none": 0.014944140233795021,
+      "alias": " - blimp_tough_vs_raising_1"
+    },
+    "blimp_tough_vs_raising_2": {
+      "acc,none": 0.774,
+      "acc_stderr,none": 0.013232501619085348,
+      "alias": " - blimp_tough_vs_raising_2"
+    },
+    "blimp_transitive": {
+      "acc,none": 0.691,
+      "acc_stderr,none": 0.0146196009772065,
+      "alias": " - blimp_transitive"
+    },
+    "blimp_wh_island": {
+      "acc,none": 0.636,
+      "acc_stderr,none": 0.015222868840522022,
+      "alias": " - blimp_wh_island"
+    },
+    "blimp_wh_questions_object_gap": {
+      "acc,none": 0.612,
+      "acc_stderr,none": 0.015417317979911076,
+      "alias": " - blimp_wh_questions_object_gap"
+    },
+    "blimp_wh_questions_subject_gap": {
+      "acc,none": 0.784,
+      "acc_stderr,none": 0.013019735539307804,
+      "alias": " - blimp_wh_questions_subject_gap"
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "acc,none": 0.806,
+      "acc_stderr,none": 0.012510816141264354,
+      "alias": " - blimp_wh_questions_subject_gap_long_distance"
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "acc,none": 0.864,
+      "acc_stderr,none": 0.010845350230472988,
+      "alias": " - blimp_wh_vs_that_no_gap"
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "acc,none": 0.902,
+      "acc_stderr,none": 0.009406619184621238,
+      "alias": " - blimp_wh_vs_that_no_gap_long_distance"
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "acc,none": 0.381,
+      "acc_stderr,none": 0.015364734787007436,
+      "alias": " - blimp_wh_vs_that_with_gap"
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "acc,none": 0.196,
+      "acc_stderr,none": 0.012559527926707387,
+      "alias": " - blimp_wh_vs_that_with_gap_long_distance"
+    }
+  },
+  "groups": {
+    "blimp": {
+      "acc,none": 0.659910447761194,
+      "acc_stderr,none": 0.16415593005353757,
+      "alias": "blimp"
+    }
+  },
+  "configs": {
+    "blimp_adjunct_island": {
+      "task": "blimp_adjunct_island",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "adjunct_island",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_anaphor_gender_agreement": {
+      "task": "blimp_anaphor_gender_agreement",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "anaphor_gender_agreement",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_anaphor_number_agreement": {
+      "task": "blimp_anaphor_number_agreement",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "anaphor_number_agreement",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_animate_subject_passive": {
+      "task": "blimp_animate_subject_passive",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "animate_subject_passive",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_animate_subject_trans": {
+      "task": "blimp_animate_subject_trans",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "animate_subject_trans",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_causative": {
+      "task": "blimp_causative",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "causative",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_complex_NP_island": {
+      "task": "blimp_complex_NP_island",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "complex_NP_island",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "task": "blimp_coordinate_structure_constraint_complex_left_branch",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "coordinate_structure_constraint_complex_left_branch",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "task": "blimp_coordinate_structure_constraint_object_extraction",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "coordinate_structure_constraint_object_extraction",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "task": "blimp_determiner_noun_agreement_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "task": "blimp_determiner_noun_agreement_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "task": "blimp_determiner_noun_agreement_irregular_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_irregular_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "task": "blimp_determiner_noun_agreement_irregular_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_irregular_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "task": "blimp_determiner_noun_agreement_with_adj_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_with_adj_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "task": "blimp_determiner_noun_agreement_with_adj_irregular_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_with_adj_irregular_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "task": "blimp_determiner_noun_agreement_with_adj_irregular_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_with_adj_irregular_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "task": "blimp_determiner_noun_agreement_with_adjective_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "determiner_noun_agreement_with_adjective_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "task": "blimp_distractor_agreement_relational_noun",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "distractor_agreement_relational_noun",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "task": "blimp_distractor_agreement_relative_clause",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "distractor_agreement_relative_clause",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_drop_argument": {
+      "task": "blimp_drop_argument",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "drop_argument",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "task": "blimp_ellipsis_n_bar_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "ellipsis_n_bar_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "task": "blimp_ellipsis_n_bar_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "ellipsis_n_bar_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_existential_there_object_raising": {
+      "task": "blimp_existential_there_object_raising",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "existential_there_object_raising",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "task": "blimp_existential_there_quantifiers_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "existential_there_quantifiers_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "task": "blimp_existential_there_quantifiers_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "existential_there_quantifiers_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_existential_there_subject_raising": {
+      "task": "blimp_existential_there_subject_raising",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "existential_there_subject_raising",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_expletive_it_object_raising": {
+      "task": "blimp_expletive_it_object_raising",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "expletive_it_object_raising",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_inchoative": {
+      "task": "blimp_inchoative",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "inchoative",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_intransitive": {
+      "task": "blimp_intransitive",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "intransitive",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "task": "blimp_irregular_past_participle_adjectives",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "irregular_past_participle_adjectives",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "task": "blimp_irregular_past_participle_verbs",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "irregular_past_participle_verbs",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "task": "blimp_irregular_plural_subject_verb_agreement_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "irregular_plural_subject_verb_agreement_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "task": "blimp_irregular_plural_subject_verb_agreement_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "irregular_plural_subject_verb_agreement_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_left_branch_island_echo_question": {
+      "task": "blimp_left_branch_island_echo_question",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "left_branch_island_echo_question",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_left_branch_island_simple_question": {
+      "task": "blimp_left_branch_island_simple_question",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "left_branch_island_simple_question",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "task": "blimp_matrix_question_npi_licensor_present",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "matrix_question_npi_licensor_present",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_npi_present_1": {
+      "task": "blimp_npi_present_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "npi_present_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_npi_present_2": {
+      "task": "blimp_npi_present_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "npi_present_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_only_npi_licensor_present": {
+      "task": "blimp_only_npi_licensor_present",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "only_npi_licensor_present",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_only_npi_scope": {
+      "task": "blimp_only_npi_scope",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "only_npi_scope",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_passive_1": {
+      "task": "blimp_passive_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "passive_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_passive_2": {
+      "task": "blimp_passive_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "passive_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_c_command": {
+      "task": "blimp_principle_A_c_command",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_c_command",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_case_1": {
+      "task": "blimp_principle_A_case_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_case_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_case_2": {
+      "task": "blimp_principle_A_case_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_case_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_domain_1": {
+      "task": "blimp_principle_A_domain_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_domain_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_domain_2": {
+      "task": "blimp_principle_A_domain_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_domain_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_domain_3": {
+      "task": "blimp_principle_A_domain_3",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_domain_3",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_principle_A_reconstruction": {
+      "task": "blimp_principle_A_reconstruction",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "principle_A_reconstruction",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "task": "blimp_regular_plural_subject_verb_agreement_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "regular_plural_subject_verb_agreement_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "task": "blimp_regular_plural_subject_verb_agreement_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "regular_plural_subject_verb_agreement_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "task": "blimp_sentential_negation_npi_licensor_present",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "sentential_negation_npi_licensor_present",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "task": "blimp_sentential_negation_npi_scope",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "sentential_negation_npi_scope",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_sentential_subject_island": {
+      "task": "blimp_sentential_subject_island",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "sentential_subject_island",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_superlative_quantifiers_1": {
+      "task": "blimp_superlative_quantifiers_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "superlative_quantifiers_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_superlative_quantifiers_2": {
+      "task": "blimp_superlative_quantifiers_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "superlative_quantifiers_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_tough_vs_raising_1": {
+      "task": "blimp_tough_vs_raising_1",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "tough_vs_raising_1",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_tough_vs_raising_2": {
+      "task": "blimp_tough_vs_raising_2",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "tough_vs_raising_2",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_transitive": {
+      "task": "blimp_transitive",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "transitive",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_island": {
+      "task": "blimp_wh_island",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_island",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_questions_object_gap": {
+      "task": "blimp_wh_questions_object_gap",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_questions_object_gap",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_questions_subject_gap": {
+      "task": "blimp_wh_questions_subject_gap",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_questions_subject_gap",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "task": "blimp_wh_questions_subject_gap_long_distance",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_questions_subject_gap_long_distance",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "task": "blimp_wh_vs_that_no_gap",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_vs_that_no_gap",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "task": "blimp_wh_vs_that_no_gap_long_distance",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_vs_that_no_gap_long_distance",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "task": "blimp_wh_vs_that_with_gap",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_vs_that_with_gap",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "task": "blimp_wh_vs_that_with_gap_long_distance",
+      "group": "blimp",
+      "dataset_path": "blimp",
+      "dataset_name": "wh_vs_that_with_gap_long_distance",
+      "validation_split": "train",
+      "doc_to_text": "",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{[sentence_good, sentence_bad]}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "blimp": "N/A",
+    "blimp_adjunct_island": 1.0,
+    "blimp_anaphor_gender_agreement": 1.0,
+    "blimp_anaphor_number_agreement": 1.0,
+    "blimp_animate_subject_passive": 1.0,
+    "blimp_animate_subject_trans": 1.0,
+    "blimp_causative": 1.0,
+    "blimp_complex_NP_island": 1.0,
+    "blimp_coordinate_structure_constraint_complex_left_branch": 1.0,
+    "blimp_coordinate_structure_constraint_object_extraction": 1.0,
+    "blimp_determiner_noun_agreement_1": 1.0,
+    "blimp_determiner_noun_agreement_2": 1.0,
+    "blimp_determiner_noun_agreement_irregular_1": 1.0,
+    "blimp_determiner_noun_agreement_irregular_2": 1.0,
+    "blimp_determiner_noun_agreement_with_adj_2": 1.0,
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": 1.0,
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": 1.0,
+    "blimp_determiner_noun_agreement_with_adjective_1": 1.0,
+    "blimp_distractor_agreement_relational_noun": 1.0,
+    "blimp_distractor_agreement_relative_clause": 1.0,
+    "blimp_drop_argument": 1.0,
+    "blimp_ellipsis_n_bar_1": 1.0,
+    "blimp_ellipsis_n_bar_2": 1.0,
+    "blimp_existential_there_object_raising": 1.0,
+    "blimp_existential_there_quantifiers_1": 1.0,
+    "blimp_existential_there_quantifiers_2": 1.0,
+    "blimp_existential_there_subject_raising": 1.0,
+    "blimp_expletive_it_object_raising": 1.0,
+    "blimp_inchoative": 1.0,
+    "blimp_intransitive": 1.0,
+    "blimp_irregular_past_participle_adjectives": 1.0,
+    "blimp_irregular_past_participle_verbs": 1.0,
+    "blimp_irregular_plural_subject_verb_agreement_1": 1.0,
+    "blimp_irregular_plural_subject_verb_agreement_2": 1.0,
+    "blimp_left_branch_island_echo_question": 1.0,
+    "blimp_left_branch_island_simple_question": 1.0,
+    "blimp_matrix_question_npi_licensor_present": 1.0,
+    "blimp_npi_present_1": 1.0,
+    "blimp_npi_present_2": 1.0,
+    "blimp_only_npi_licensor_present": 1.0,
+    "blimp_only_npi_scope": 1.0,
+    "blimp_passive_1": 1.0,
+    "blimp_passive_2": 1.0,
+    "blimp_principle_A_c_command": 1.0,
+    "blimp_principle_A_case_1": 1.0,
+    "blimp_principle_A_case_2": 1.0,
+    "blimp_principle_A_domain_1": 1.0,
+    "blimp_principle_A_domain_2": 1.0,
+    "blimp_principle_A_domain_3": 1.0,
+    "blimp_principle_A_reconstruction": 1.0,
+    "blimp_regular_plural_subject_verb_agreement_1": 1.0,
+    "blimp_regular_plural_subject_verb_agreement_2": 1.0,
+    "blimp_sentential_negation_npi_licensor_present": 1.0,
+    "blimp_sentential_negation_npi_scope": 1.0,
+    "blimp_sentential_subject_island": 1.0,
+    "blimp_superlative_quantifiers_1": 1.0,
+    "blimp_superlative_quantifiers_2": 1.0,
+    "blimp_tough_vs_raising_1": 1.0,
+    "blimp_tough_vs_raising_2": 1.0,
+    "blimp_transitive": 1.0,
+    "blimp_wh_island": 1.0,
+    "blimp_wh_questions_object_gap": 1.0,
+    "blimp_wh_questions_subject_gap": 1.0,
+    "blimp_wh_questions_subject_gap_long_distance": 1.0,
+    "blimp_wh_vs_that_no_gap": 1.0,
+    "blimp_wh_vs_that_no_gap_long_distance": 1.0,
+    "blimp_wh_vs_that_with_gap": 1.0,
+    "blimp_wh_vs_that_with_gap_long_distance": 1.0
+  },
+  "n-shot": {
+    "blimp": 0,
+    "blimp_adjunct_island": 0,
+    "blimp_anaphor_gender_agreement": 0,
+    "blimp_anaphor_number_agreement": 0,
+    "blimp_animate_subject_passive": 0,
+    "blimp_animate_subject_trans": 0,
+    "blimp_causative": 0,
+    "blimp_complex_NP_island": 0,
+    "blimp_coordinate_structure_constraint_complex_left_branch": 0,
+    "blimp_coordinate_structure_constraint_object_extraction": 0,
+    "blimp_determiner_noun_agreement_1": 0,
+    "blimp_determiner_noun_agreement_2": 0,
+    "blimp_determiner_noun_agreement_irregular_1": 0,
+    "blimp_determiner_noun_agreement_irregular_2": 0,
+    "blimp_determiner_noun_agreement_with_adj_2": 0,
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": 0,
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": 0,
+    "blimp_determiner_noun_agreement_with_adjective_1": 0,
+    "blimp_distractor_agreement_relational_noun": 0,
+    "blimp_distractor_agreement_relative_clause": 0,
+    "blimp_drop_argument": 0,
+    "blimp_ellipsis_n_bar_1": 0,
+    "blimp_ellipsis_n_bar_2": 0,
+    "blimp_existential_there_object_raising": 0,
+    "blimp_existential_there_quantifiers_1": 0,
+    "blimp_existential_there_quantifiers_2": 0,
+    "blimp_existential_there_subject_raising": 0,
+    "blimp_expletive_it_object_raising": 0,
+    "blimp_inchoative": 0,
+    "blimp_intransitive": 0,
+    "blimp_irregular_past_participle_adjectives": 0,
+    "blimp_irregular_past_participle_verbs": 0,
+    "blimp_irregular_plural_subject_verb_agreement_1": 0,
+    "blimp_irregular_plural_subject_verb_agreement_2": 0,
+    "blimp_left_branch_island_echo_question": 0,
+    "blimp_left_branch_island_simple_question": 0,
+    "blimp_matrix_question_npi_licensor_present": 0,
+    "blimp_npi_present_1": 0,
+    "blimp_npi_present_2": 0,
+    "blimp_only_npi_licensor_present": 0,
+    "blimp_only_npi_scope": 0,
+    "blimp_passive_1": 0,
+    "blimp_passive_2": 0,
+    "blimp_principle_A_c_command": 0,
+    "blimp_principle_A_case_1": 0,
+    "blimp_principle_A_case_2": 0,
+    "blimp_principle_A_domain_1": 0,
+    "blimp_principle_A_domain_2": 0,
+    "blimp_principle_A_domain_3": 0,
+    "blimp_principle_A_reconstruction": 0,
+    "blimp_regular_plural_subject_verb_agreement_1": 0,
+    "blimp_regular_plural_subject_verb_agreement_2": 0,
+    "blimp_sentential_negation_npi_licensor_present": 0,
+    "blimp_sentential_negation_npi_scope": 0,
+    "blimp_sentential_subject_island": 0,
+    "blimp_superlative_quantifiers_1": 0,
+    "blimp_superlative_quantifiers_2": 0,
+    "blimp_tough_vs_raising_1": 0,
+    "blimp_tough_vs_raising_2": 0,
+    "blimp_transitive": 0,
+    "blimp_wh_island": 0,
+    "blimp_wh_questions_object_gap": 0,
+    "blimp_wh_questions_subject_gap": 0,
+    "blimp_wh_questions_subject_gap_long_distance": 0,
+    "blimp_wh_vs_that_no_gap": 0,
+    "blimp_wh_vs_that_no_gap_long_distance": 0,
+    "blimp_wh_vs_that_with_gap": 0,
+    "blimp_wh_vs_that_with_gap_long_distance": 0
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/home/bastian/Dokumente/weenie_llamas/models/final_20",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": "cuda",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null
+  },
+  "git_hash": null
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<pad>"
+}

tokenizer_config.json

@@ -0,0 +1,8 @@
+{
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b57c8c6441a9f4bedee2336ae581d8f8499d13d0ce15169f9e94c4daeb3186f
+size 4536