| { | |
| "model": { | |
| "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", | |
| "api": "hf", | |
| "dtype": "bfloat16", | |
| "max_length": 131072, | |
| "architecture": "LlamaForCausalLM", | |
| "type": "instruction-tuned", | |
| "num_parameters": "8b" | |
| }, | |
| "results": [ | |
| { | |
| "name": "belebele_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.7077777777777777, | |
| "acc_norm": 0.7077777777777777 | |
| }, | |
| { | |
| "name": "exams_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.3231552162849873, | |
| "acc_norm": 0.35877862595419846 | |
| }, | |
| { | |
| "name": "check_worthiness", | |
| "task": "multiple_choice", | |
| "acc": 0.37614259597806216, | |
| "acc_norm": 0.37614259597806216 | |
| }, | |
| { | |
| "name": "ironytr", | |
| "task": "text_classification", | |
| "acc": 0.5133333333333333, | |
| "acc_norm": 0.5666666666666667 | |
| }, | |
| { | |
| "name": "mkqa_tr", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.09115122817401598, | |
| "f1": 0.15627870028803578 | |
| }, | |
| { | |
| "name": "mnli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.3209, | |
| "acc_norm": 0.3596 | |
| }, | |
| { | |
| "name": "news_cat", | |
| "task": "text_classification", | |
| "acc": 0.66, | |
| "acc_norm": 0.604 | |
| }, | |
| { | |
| "name": "offenseval_tr", | |
| "task": "text_classification", | |
| "acc": 0.23582766439909297, | |
| "acc_norm": 0.3687641723356009 | |
| }, | |
| { | |
| "name": "relevance_judgment", | |
| "task": "multiple_choice", | |
| "acc": 0.4648080438756856, | |
| "acc_norm": 0.5648994515539305 | |
| }, | |
| { | |
| "name": "snli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.3028, | |
| "acc_norm": 0.3528 | |
| }, | |
| { | |
| "name": "sts_tr", | |
| "task": "text_classification", | |
| "acc": 0.19579405366207397, | |
| "acc_norm": 0.1551849166062364 | |
| }, | |
| { | |
| "name": "tquad", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.23318385650224216, | |
| "f1": 0.5062272078338648 | |
| }, | |
| { | |
| "name": "turkish_plu_goal_inference", | |
| "task": "multiple_choice", | |
| "acc": 0.40860215053763443, | |
| "acc_norm": 0.45997610513739545 | |
| }, | |
| { | |
| "name": "turkish_plu_next_event_prediction", | |
| "task": "multiple_choice", | |
| "acc": 0.4442748091603053, | |
| "acc_norm": 0.5419847328244275 | |
| }, | |
| { | |
| "name": "turkish_plu_step_inference", | |
| "task": "multiple_choice", | |
| "acc": 0.33169934640522875, | |
| "acc_norm": 0.4624183006535948 | |
| }, | |
| { | |
| "name": "turkish_plu_step_ordering", | |
| "task": "multiple_choice", | |
| "acc": 0.633692458374143, | |
| "acc_norm": 0.633692458374143 | |
| }, | |
| { | |
| "name": "xcopa_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.608, | |
| "acc_norm": 0.608 | |
| }, | |
| { | |
| "name": "xnli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.4807228915662651, | |
| "acc_norm": 0.4807228915662651 | |
| }, | |
| { | |
| "name": "xquad_tr", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.21428571428571427, | |
| "f1": 0.4170277103753468 | |
| }, | |
| { | |
| "name": "gecturk_generation", | |
| "task": "grammatical_error_correction", | |
| "exact_match": 0.006548220906158217 | |
| }, | |
| { | |
| "name": "mlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.3970732593089523, | |
| "rouge2": 0.2580847974481608, | |
| "rougeL": 0.32541870004323864 | |
| }, | |
| { | |
| "name": "tr-wikihow-summ", | |
| "task": "summarization", | |
| "rouge1": 0.2444219472309469, | |
| "rouge2": 0.07918689923056912, | |
| "rougeL": 0.1723711997990579 | |
| }, | |
| { | |
| "name": "wiki_lingua_tr", | |
| "task": "summarization", | |
| "rouge1": 0.22962802442673436, | |
| "rouge2": 0.07863769381205138, | |
| "rougeL": 0.16924353815052512 | |
| }, | |
| { | |
| "name": "wmt-tr-en-prompt", | |
| "task": "machine_translation", | |
| "wer": 0.7815049287082738, | |
| "bleu": 0.1564145890661644 | |
| }, | |
| { | |
| "name": "xlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.2805962791068744, | |
| "rouge2": 0.12421139697660691, | |
| "rougeL": 0.21080710839195932 | |
| } | |
| ] | |
| } |