add first Yi chat

evals/arc/arc_nl_Yi-6B-Chat.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_nl": {
+      "acc": 0.2711719418306245,
+      "acc_stderr": 0.01300808888870032,
+      "acc_norm": 0.29597946963216426,
+      "acc_norm_stderr": 0.013356788048643684
+    }
+  },
+  "versions": {
+    "arc_nl": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=01-ai/Yi-6B-Chat,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 2,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/hellaswag/hellaswag_nl_Yi-6B-Chat.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "hellaswag_nl": {
+      "acc": 0.34830005396654073,
+      "acc_stderr": 0.004949956019124193,
+      "acc_norm": 0.4175930922827847,
+      "acc_norm_stderr": 0.005123781939169409
+    }
+  },
+  "versions": {
+    "hellaswag_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=01-ai/Yi-6B-Chat,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 2,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/models.json

@@ -135,6 +135,14 @@
         "num_parameters": 6738440192,
         "quantization": "8-bit"
     },
+    "yi-6b-chat": {
+        "compute_dtype": "auto",
+        "dutch_coverage": "none",
+        "model_name": "01-ai/Yi-6B-Chat",
+        "model_type": "instruction-tuned",
+        "num_parameters": 6061035520,
+        "quantization": "8-bit"
+    },
     "zephyr-7b-beta": {
         "compute_dtype": "bfloat16",
         "dutch_coverage": "none",
@@ -143,4 +151,4 @@
         "num_parameters": 7241732096,
         "quantization": "8-bit"
     }
-}
+}

evals/truthfulqa/truthfulqa_nl_Yi-6B-Chat.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "truthfulqa_nl": {
+      "mc1": 0.27770700636942675,
+      "mc1_stderr": 0.015995288886407874,
+      "mc2": 0.4363133415772647,
+      "mc2_stderr": 0.016027103264630135
+    }
+  },
+  "versions": {
+    "truthfulqa_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=01-ai/Yi-6B-Chat,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 2,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}