| { | |
| "model": { | |
| "model": "meta-llama/Meta-Llama-3-8B", | |
| "api": "hf", | |
| "architecture": "LlamaForCausalLM", | |
| "max_length": 8192, | |
| "type": "pretrained", | |
| "dtype": "bfloat16", | |
| "num_parameters": "8b" | |
| }, | |
| "results": [ | |
| { | |
| "name": "belebele_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.5144, | |
| "acc_norm": 0.5144 | |
| }, | |
| { | |
| "name": "exams_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.3028, | |
| "acc_norm": 0.3537 | |
| }, | |
| { | |
| "name": "check_worthiness", | |
| "task": "multiple_choice", | |
| "acc": 0.37614259597806216, | |
| "acc_norm": 0.38391224862888484 | |
| }, | |
| { | |
| "name": "ironytr", | |
| "task": "text_classification", | |
| "acc": 0.515, | |
| "acc_norm": 0.525 | |
| }, | |
| { | |
| "name": "mkqa_tr", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.13465522343888725, | |
| "f1": 0.19144550324599957 | |
| }, | |
| { | |
| "name": "mnli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.3206, | |
| "acc_norm": 0.3329 | |
| }, | |
| { | |
| "name": "news_cat", | |
| "task": "text_classification", | |
| "acc": 0.724, | |
| "acc_norm": 0.656 | |
| }, | |
| { | |
| "name": "offenseval_tr", | |
| "task": "text_classification", | |
| "acc": 0.2193877551020408, | |
| "acc_norm": 0.48214285714285715 | |
| }, | |
| { | |
| "name": "relevance_judgment", | |
| "task": "multiple_choice", | |
| "acc": 0.42550274223034734, | |
| "acc_norm": 0.5173674588665448 | |
| }, | |
| { | |
| "name": "snli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.325, | |
| "acc_norm": 0.3766 | |
| }, | |
| { | |
| "name": "sts_tr", | |
| "task": "text_classification", | |
| "acc": 0.16388687454677303, | |
| "acc_norm": 0.19216823785351705 | |
| }, | |
| { | |
| "name": "tquad", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.28475336322869954, | |
| "f1": 0.5013148868557868 | |
| }, | |
| { | |
| "name": "turkish_plu_goal_inference", | |
| "task": "multiple_choice", | |
| "acc": 0.38948626045400236, | |
| "acc_norm": 0.4169653524492234 | |
| }, | |
| { | |
| "name": "turkish_plu_next_event_prediction", | |
| "task": "multiple_choice", | |
| "acc": 0.4488549618320611, | |
| "acc_norm": 0.5328244274809161 | |
| }, | |
| { | |
| "name": "turkish_plu_step_inference", | |
| "task": "multiple_choice", | |
| "acc": 0.32189542483660133, | |
| "acc_norm": 0.47058823529411764 | |
| }, | |
| { | |
| "name": "turkish_plu_step_ordering", | |
| "task": "multiple_choice", | |
| "acc": 0.6278158667972575, | |
| "acc_norm": 0.6278158667972575 | |
| }, | |
| { | |
| "name": "xcopa_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.618, | |
| "acc_norm": 0.618 | |
| }, | |
| { | |
| "name": "xnli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.4839357429718876, | |
| "acc_norm": 0.4839357429718876 | |
| }, | |
| { | |
| "name": "xquad_tr", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.20840336134453782, | |
| "f1": 0.33796418555415153 | |
| }, | |
| { | |
| "name": "gecturk_generation", | |
| "task": "grammatical_error_correction", | |
| "exact_match": 0.006692666955558766 | |
| }, | |
| { | |
| "name": "mlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.38446881575055203, | |
| "rouge2": 0.2503978598237102, | |
| "rougeL": 0.319713589198042 | |
| }, | |
| { | |
| "name": "wiki_lingua_tr", | |
| "task": "summarization", | |
| "rouge1": 0.2069234464456151, | |
| "rouge2": 0.06576422586110373, | |
| "rougeL": 0.1516869929958613 | |
| }, | |
| { | |
| "name": "wmt-tr-en-prompt", | |
| "task": "machine_translation", | |
| "wer": 0.9262281724087097, | |
| "bleu": 0.113320746345327 | |
| }, | |
| { | |
| "name": "tr-wikihow-summ", | |
| "task": "summarization", | |
| "rouge1": 0.22198334074612264, | |
| "rouge2": 0.06595253357908101, | |
| "rougeL": 0.1562686696082268 | |
| }, | |
| { | |
| "name": "xlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.2615001361521869, | |
| "rouge2": 0.11093149007661907, | |
| "rougeL": 0.20321693263972507 | |
| } | |
| ] | |
| } |