Spaces:
Running
Running
| { | |
| "results": { | |
| "ifeval": { | |
| "prompt_level_strict_acc,none": 0.6709796672828097, | |
| "prompt_level_strict_acc_stderr,none": 0.02021944211853842, | |
| "inst_level_strict_acc,none": 0.7541966426858513, | |
| "inst_level_strict_acc_stderr,none": "N/A", | |
| "prompt_level_loose_acc,none": 0.7079482439926063, | |
| "prompt_level_loose_acc_stderr,none": 0.019567429846009482, | |
| "inst_level_loose_acc,none": 0.790167865707434, | |
| "inst_level_loose_acc_stderr,none": "N/A", | |
| "alias": "ifeval" | |
| } | |
| }, | |
| "group_subtasks": { | |
| "ifeval": [] | |
| }, | |
| "configs": { | |
| "ifeval": { | |
| "task": "ifeval", | |
| "dataset_path": "wis-k/instruction-following-eval", | |
| "test_split": "train", | |
| "doc_to_text": "prompt", | |
| "doc_to_target": 0, | |
| "process_results": "def process_results(doc, results):\n eval_logger.warning(\n \"This task is meant for chat-finetuned models, and may not give meaningful results for models other than `openai` or `anthropic` if `doc_to_text` in its YAML is not wrapped in the appropriate chat template string. This warning will be removed when chat templating support is added natively to local models\"\n )\n\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n", | |
| "description": "", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "prompt_level_strict_acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "inst_level_strict_acc", | |
| "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "prompt_level_loose_acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "inst_level_loose_acc", | |
| "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "until": [], | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "max_gen_toks": 1280 | |
| }, | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 2.0 | |
| } | |
| } | |
| }, | |
| "versions": { | |
| "ifeval": 2.0 | |
| }, | |
| "n-shot": { | |
| "ifeval": 0 | |
| }, | |
| "higher_is_better": { | |
| "ifeval": { | |
| "prompt_level_strict_acc": true, | |
| "inst_level_strict_acc": true, | |
| "prompt_level_loose_acc": true, | |
| "inst_level_loose_acc": true | |
| } | |
| }, | |
| "n-samples": { | |
| "ifeval": { | |
| "original": 541, | |
| "effective": 541 | |
| } | |
| }, | |
| "config": { | |
| "model": "local-chat-completions", | |
| "model_args": "base_url=https://openrouter.ai/api/v1,model=mistralai/mixtral-8x22b-instruct", | |
| "batch_size": 1, | |
| "batch_sizes": [], | |
| "device": null, | |
| "use_cache": null, | |
| "limit": null, | |
| "bootstrap_iters": 100000, | |
| "gen_kwargs": null, | |
| "random_seed": 0, | |
| "numpy_seed": 1234, | |
| "torch_seed": 1234, | |
| "fewshot_seed": 1234 | |
| }, | |
| "git_hash": "e39df01c", | |
| "date": 1717686455.6903582, | |
| "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Arch Linux (x86_64)\nGCC version: (GCC) 14.1.1 20240522\nClang version: Could not collect\nCMake version: version 3.29.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Apr 23 2024, 09:16:07) [GCC 13.2.1 20240417] (64-bit runtime)\nPython platform: Linux-6.9.2-arch1-1-x86_64-with-glibc2.39\nIs CUDA available: False\nCUDA runtime version: No CUDA\nCUDA_MODULE_LOADING set to: N/A\nGPU models and configuration: No CUDA\nNvidia driver version: No CUDA\ncuDNN version: No CUDA\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: AuthenticAMD\nModel name: AMD Ryzen 7 3700X 8-Core Processor\nCPU family: 23\nModel: 113\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 0\nFrequency boost: enabled\nCPU(s) scaling MHz: 91%\nCPU max MHz: 4426.1709\nCPU min MHz: 2200.0000\nBogoMIPS: 7189.76\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization: AMD-V\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 4 MiB (8 instances)\nL3 cache: 32 MiB (2 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow: Mitigation; Safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] torch==2.3.1\n[conda] Could not collect", | |
| "transformers_version": "4.41.2", | |
| "upper_git_hash": null, | |
| "task_hashes": {}, | |
| "model_source": "local-chat-completions", | |
| "model_name": "mistralai/mixtral-8x22b-instruct", | |
| "model_name_sanitized": "mistralai__mixtral-8x22b-instruct", | |
| "system_instruction": null, | |
| "system_instruction_sha": null, | |
| "chat_template": null, | |
| "chat_template_sha": null, | |
| "start_time": 629710.788420653, | |
| "end_time": 634749.510794009, | |
| "total_evaluation_time_seconds": "5038.722373355995" | |
| } |