openchat, nous hermes results

IFEval.csv

@@ -9,12 +9,14 @@ Llama 3 8B,8,NeverSleep/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF,Q8_0,0.7395,0.6747,0.7
 LLama 3 8B,8,perplexity/llama-3-sonar-small-32k-chat (Openrouter),FP16,0.7392,0.6600,0.7470,0.7357,0.8141,https://openrouter.ai/models/perplexity/llama-3-sonar-small-32k-chat
 Llama 3 8B,8,MaziyarPanahi/Meta-Llama-3-8B-Instruct-GGUF,Q8_0,0.7366,0.6765,0.7614,0.7172,0.7914,https://huggingface.co/MaziyarPanahi/Meta-Llama-3-8B-Instruct-GGUF
 Mixtral 8x22B,176,mistralai/mixtral-8x22b-instruct (Openrouter),FP16,0.7308,0.6710,0.7542,0.7079,0.7902,https://openrouter.ai/models/mistralai/mixtral-8x22b-instruct
-Command R,35,mradermacher/c4ai-command-r-v01-GGUF,Q6_K,0.6971,0.6230,0.7218,0.6784,0.7650,https://huggingface.co/mradermacher/c4ai-command-r-v01-GGUF
 Llama 3 8B,8,bartowski/Llama-3-SauerkrautLM-8b-Instruct-GGUF,Q8_0,0.7141,0.6562,0.7290,0.7024,0.7686,https://huggingface.co/bartowski/Llama-3-SauerkrautLM-8b-Instruct-GGUF
+Command R,35,mradermacher/c4ai-command-r-v01-GGUF,Q6_K,0.6971,0.6230,0.7218,0.6784,0.7650,https://huggingface.co/mradermacher/c4ai-command-r-v01-GGUF
 Phi 3 Medium 4K,14,bartowski/Phi-3-medium-4k-instruct-GGUF,Q8_0,0.6673,0.6100,0.7014,0.6322,0.7254,https://huggingface.co/bartowski/Phi-3-medium-4k-instruct-GGUF
 Mistral 7B v0.1,7,TheBloke/CapybaraHermes-2.5-Mistral-7B-GGUF,Q8_0,0.6096,0.5287,0.6367,0.5860,0.6871,https://huggingface.co/TheBloke/CapybaraHermes-2.5-Mistral-7B-GGUF
 Codestral 22B v0.1,22,bartowski/Codestral-22B-v0.1-GGUF,Q6_K,0.6074,0.5305,0.6415,0.5730,0.6847,https://huggingface.co/bartowski/Codestral-22B-v0.1-GGUF
+Mistral 7B v0.1,7,nousresearch/nous-hermes-2-mistral-7b-dpo (Openrouter),FP16,0.5968,0.5194,0.6343,0.5619,0.6715,https://openrouter.ai/models/nousresearch/nous-hermes-2-mistral-7b-dpo
 Mixtral 8x7B v0.1,56,mixtral-8x7b-32768 (Groq),FP16,0.5887,0.5028,0.6247,0.5545,0.6727,https://console.groq.com/docs/models
+Llama 3 8B,8,bartowski/openchat-3.6-8b-20240522-GGUF,Q8_0,0.5791,0.5139,0.6139,0.5434,0.6451,https://huggingface.co/bartowski/openchat-3.6-8b-20240522-GGUF
 Mistral 7B v0.3,7,MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF,Q8_0,0.5689,0.4972,0.5983,0.5397,0.6403,https://huggingface.co/MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF
 Llama 3 8B,8,bartowski/Hermes-2-Pro-Llama-3-8B-GGUF,Q8_0,0.5686,0.5010,0.6079,0.5287,0.6367,https://huggingface.co/bartowski/Hermes-2-Pro-Llama-3-8B-GGUF
 Mistral 7B v0.1,7,mlabonne/AlphaMonarch-7B,Q8_0,0.5095,0.4214,0.5432,0.4787,0.5947,https://huggingface.co/mlabonne/AlphaMonarch-7B

app.py

@@ -80,7 +80,7 @@ def build_leaderboard_tab(leaderboard_table_file, show_plot=False):
             # arena table
             arena_table_vals = get_arena_table(model_table_df)
             with gr.Tab("IFEval", id=0):
-                md = "Leaderboard for various Large Language Models measured with IFEval benchmark.\n\n[IFEval](https://github.com/google-research/google-research/tree/master/instruction_following_eval) is a straightforward and easy-to-reproduce evaluation benchmark. It focuses on a set of \"verifiable instructions\" such as \"write in more than 400 words\" and \"mention the keyword of AI at least 3 times\". We identified 25 types of those verifiable instructions and constructed around 500 prompts, with each prompt containing one or more verifiable instructions. \n\nTest ran with `lm-evaluation-harness`. Raw results can be found in the `results` directory. Made by [Kristian Polso](https://polso.info)\n\n\**Changelog**\n\n8.6.2024 - Changed CapybaraHermes result, was using the wrong prompt template."
+                md = "Leaderboard for various Large Language Models measured with IFEval benchmark.\n\n[IFEval](https://github.com/google-research/google-research/tree/master/instruction_following_eval) is a straightforward and easy-to-reproduce evaluation benchmark. It focuses on a set of \"verifiable instructions\" such as \"write in more than 400 words\" and \"mention the keyword of AI at least 3 times\". We identified 25 types of those verifiable instructions and constructed around 500 prompts, with each prompt containing one or more verifiable instructions. \n\nTest ran with `lm-evaluation-harness`. Raw results can be found in the `results` directory. Made by [Kristian Polso](https://polso.info)\n\n**Changelog**\n\n8.6.2024 - Changed CapybaraHermes result, was using the wrong prompt template."
                 gr.Markdown(md, elem_id="leaderboard_markdown")
                 gr.Dataframe(
                     headers=[

results/openchat-3.6-8b-20240522-GGUF.json

@@ -1,13 +1,13 @@
 {
   "results": {
     "ifeval": {
-      "prompt_level_strict_acc,none": 0.3752310536044362,
-      "prompt_level_strict_acc_stderr,none": 0.02083589806503567,
-      "inst_level_strict_acc,none": 0.48081534772182255,
+      "prompt_level_strict_acc,none": 0.5138632162661737,
+      "prompt_level_strict_acc_stderr,none": 0.0215083020678561,
+      "inst_level_strict_acc,none": 0.6139088729016786,
       "inst_level_strict_acc_stderr,none": "N/A",
-      "prompt_level_loose_acc,none": 0.39926062846580407,
-      "prompt_level_loose_acc_stderr,none": 0.021075331332701258,
-      "inst_level_loose_acc,none": 0.5047961630695443,
+      "prompt_level_loose_acc,none": 0.5434380776340111,
+      "prompt_level_loose_acc_stderr,none": 0.021435222545538937,
+      "inst_level_loose_acc,none": 0.645083932853717,
       "inst_level_loose_acc_stderr,none": "N/A",
       "alias": "ifeval"
     }
@@ -99,8 +99,8 @@
     "fewshot_seed": 1234
   },
   "git_hash": "e39df01c",
-  "date": 1717777026.8398135,
-  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Arch Linux (x86_64)\nGCC version: (GCC) 14.1.1 20240522\nClang version: Could not collect\nCMake version: version 3.29.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Apr 23 2024, 09:16:07) [GCC 13.2.1 20240417] (64-bit runtime)\nPython platform: Linux-6.9.2-arch1-1-x86_64-with-glibc2.39\nIs CUDA available: False\nCUDA runtime version: No CUDA\nCUDA_MODULE_LOADING set to: N/A\nGPU models and configuration: No CUDA\nNvidia driver version: No CUDA\ncuDNN version: No CUDA\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        43 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               16\nOn-line CPU(s) list:                  0-15\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 3700X 8-Core Processor\nCPU family:                           23\nModel:                                113\nThread(s) per core:                   2\nCore(s) per socket:                   8\nSocket(s):                            1\nStepping:                             0\nFrequency boost:                      enabled\nCPU(s) scaling MHz:                   74%\nCPU max MHz:                          4426.1709\nCPU min MHz:                          2200.0000\nBogoMIPS:                             7189.76\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization:                       AMD-V\nL1d cache:                            256 KiB (8 instances)\nL1i cache:                            256 KiB (8 instances)\nL2 cache:                             4 MiB (8 instances)\nL3 cache:                             32 MiB (2 instances)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-15\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow:   Mitigation; Safe RET\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] torch==2.3.1\n[conda] Could not collect",
+  "date": 1717833446.674556,
+  "pretty_env_info": "PyTorch version: 2.3.1+rocm6.0\nIs debug build: False\nCUDA used to build PyTorch: N/A\nROCM used to build PyTorch: 6.0.32830-d62f6a171\n\nOS: Arch Linux (x86_64)\nGCC version: (GCC) 14.1.1 20240522\nClang version: Could not collect\nCMake version: version 3.29.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Apr 23 2024, 09:16:07) [GCC 13.2.1 20240417] (64-bit runtime)\nPython platform: Linux-6.9.2-arch1-1-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: AMD Radeon RX 7900 XTX (gfx1100)\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nHIP runtime version: 6.0.32830\nMIOpen runtime version: 3.0.0\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        43 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               16\nOn-line CPU(s) list:                  0-15\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 3700X 8-Core Processor\nCPU family:                           23\nModel:                                113\nThread(s) per core:                   2\nCore(s) per socket:                   8\nSocket(s):                            1\nStepping:                             0\nFrequency boost:                      enabled\nCPU(s) scaling MHz:                   74%\nCPU max MHz:                          4426.1709\nCPU min MHz:                          2200.0000\nBogoMIPS:                             7189.76\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization:                       AMD-V\nL1d cache:                            256 KiB (8 instances)\nL1i cache:                            256 KiB (8 instances)\nL2 cache:                             4 MiB (8 instances)\nL3 cache:                             32 MiB (2 instances)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-15\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow:   Mitigation; Safe RET\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.3\n[pip3] torch==2.3.1+rocm6.0\n[pip3] torchaudio==2.3.1+rocm6.0\n[pip3] torchvision==0.18.1+rocm6.0\n[conda] Could not collect",
   "transformers_version": "4.41.2",
   "upper_git_hash": null,
   "task_hashes": {},
@@ -111,7 +111,7 @@
   "system_instruction_sha": null,
   "chat_template": null,
   "chat_template_sha": null,
-  "start_time": 720282.438291442,
-  "end_time": 722334.057655391,
-  "total_evaluation_time_seconds": "2051.619363949052"
+  "start_time": 776702.200045851,
+  "end_time": 779142.741448573,
+  "total_evaluation_time_seconds": "2440.5414027220104"
 }

results/openrouter-nousresearch-nous-hermes-2-mistral-7b-dpo.json

@@ -0,0 +1,117 @@
+{
+  "results": {
+    "ifeval": {
+      "prompt_level_strict_acc,none": 0.5194085027726433,
+      "prompt_level_strict_acc_stderr,none": 0.021500357879025083,
+      "inst_level_strict_acc,none": 0.6342925659472423,
+      "inst_level_strict_acc_stderr,none": "N/A",
+      "prompt_level_loose_acc,none": 0.5619223659889094,
+      "prompt_level_loose_acc_stderr,none": 0.021350931135490865,
+      "inst_level_loose_acc,none": 0.6714628297362111,
+      "inst_level_loose_acc_stderr,none": "N/A",
+      "alias": "ifeval"
+    }
+  },
+  "group_subtasks": {
+    "ifeval": []
+  },
+  "configs": {
+    "ifeval": {
+      "task": "ifeval",
+      "dataset_path": "wis-k/instruction-following-eval",
+      "test_split": "train",
+      "doc_to_text": "prompt",
+      "doc_to_target": 0,
+      "process_results": "def process_results(doc, results):\n    eval_logger.warning(\n        \"This task is meant for chat-finetuned models, and may not give meaningful results for models other than `openai` or `anthropic` if `doc_to_text` in its YAML is not wrapped in the appropriate chat template string. This warning will be removed when chat templating support is added natively to local models\"\n    )\n\n    inp = InputExample(\n        key=doc[\"key\"],\n        instruction_id_list=doc[\"instruction_id_list\"],\n        prompt=doc[\"prompt\"],\n        kwargs=doc[\"kwargs\"],\n    )\n    response = results[0]\n\n    out_strict = test_instruction_following_strict(inp, response)\n    out_loose = test_instruction_following_loose(inp, response)\n\n    return {\n        \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n        \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n        \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n        \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "prompt_level_strict_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_strict_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "prompt_level_loose_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_loose_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 1280
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "ifeval": 2.0
+  },
+  "n-shot": {
+    "ifeval": 0
+  },
+  "higher_is_better": {
+    "ifeval": {
+      "prompt_level_strict_acc": true,
+      "inst_level_strict_acc": true,
+      "prompt_level_loose_acc": true,
+      "inst_level_loose_acc": true
+    }
+  },
+  "n-samples": {
+    "ifeval": {
+      "original": 541,
+      "effective": 541
+    }
+  },
+  "config": {
+    "model": "local-chat-completions",
+    "model_args": "base_url=https://openrouter.ai/api/v1,model=nousresearch/nous-hermes-2-mistral-7b-dpo",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "e39df01c",
+  "date": 1717834283.3541698,
+  "pretty_env_info": "PyTorch version: 2.3.1+rocm6.0\nIs debug build: False\nCUDA used to build PyTorch: N/A\nROCM used to build PyTorch: 6.0.32830-d62f6a171\n\nOS: Arch Linux (x86_64)\nGCC version: (GCC) 14.1.1 20240522\nClang version: Could not collect\nCMake version: version 3.29.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Apr 23 2024, 09:16:07) [GCC 13.2.1 20240417] (64-bit runtime)\nPython platform: Linux-6.9.2-arch1-1-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: AMD Radeon RX 7900 XTX (gfx1100)\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nHIP runtime version: 6.0.32830\nMIOpen runtime version: 3.0.0\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                         x86_64\nCPU op-mode(s):                       32-bit, 64-bit\nAddress sizes:                        43 bits physical, 48 bits virtual\nByte Order:                           Little Endian\nCPU(s):                               16\nOn-line CPU(s) list:                  0-15\nVendor ID:                            AuthenticAMD\nModel name:                           AMD Ryzen 7 3700X 8-Core Processor\nCPU family:                           23\nModel:                                113\nThread(s) per core:                   2\nCore(s) per socket:                   8\nSocket(s):                            1\nStepping:                             0\nFrequency boost:                      enabled\nCPU(s) scaling MHz:                   64%\nCPU max MHz:                          4426.1709\nCPU min MHz:                          2200.0000\nBogoMIPS:                             7189.76\nFlags:                                fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization:                       AMD-V\nL1d cache:                            256 KiB (8 instances)\nL1i cache:                            256 KiB (8 instances)\nL2 cache:                             4 MiB (8 instances)\nL3 cache:                             32 MiB (2 instances)\nNUMA node(s):                         1\nNUMA node0 CPU(s):                    0-15\nVulnerability Gather data sampling:   Not affected\nVulnerability Itlb multihit:          Not affected\nVulnerability L1tf:                   Not affected\nVulnerability Mds:                    Not affected\nVulnerability Meltdown:               Not affected\nVulnerability Mmio stale data:        Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed:               Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow:   Mitigation; Safe RET\nVulnerability Spec store bypass:      Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:             Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:             Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                  Not affected\nVulnerability Tsx async abort:        Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.3\n[pip3] torch==2.3.1+rocm6.0\n[pip3] torchaudio==2.3.1+rocm6.0\n[pip3] torchvision==0.18.1+rocm6.0\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "task_hashes": {},
+  "model_source": "local-chat-completions",
+  "model_name": "nousresearch/nous-hermes-2-mistral-7b-dpo",
+  "model_name_sanitized": "nousresearch__nous-hermes-2-mistral-7b-dpo",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 777538.783488651,
+  "end_time": 779170.783773116,
+  "total_evaluation_time_seconds": "1632.0002844649134"
+}