| { | |
| "model": { | |
| "model": "meta-llama/Llama-3.2-1B", | |
| "api": "hf", | |
| "dtype": "bfloat16", | |
| "max_length": 131072, | |
| "architecture": "LlamaForCausalLM", | |
| "type": "pretrained", | |
| "num_parameters": "1b" | |
| }, | |
| "results": [ | |
| { | |
| "name": "belebele_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.29555555555555557, | |
| "acc_norm": 0.29555555555555557 | |
| }, | |
| { | |
| "name": "exams_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.28498727735368956, | |
| "acc_norm": 0.3053435114503817 | |
| }, | |
| { | |
| "name": "check_worthiness", | |
| "task": "multiple_choice", | |
| "acc": 0.3880255941499086, | |
| "acc_norm": 0.623400365630713 | |
| }, | |
| { | |
| "name": "gecturk_generation", | |
| "task": "grammatical_error_correction", | |
| "exact_match": 0.00741489720256151 | |
| }, | |
| { | |
| "name": "ironytr", | |
| "task": "text_classification", | |
| "acc": 0.5283333333333333, | |
| "acc_norm": 0.5033333333333333 | |
| }, | |
| { | |
| "name": "mkqa_tr", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.007694584196507843, | |
| "f1": 0.03304091036050505 | |
| }, | |
| { | |
| "name": "mlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.23283491254211872, | |
| "rouge2": 0.13426790568610214, | |
| "rougeL": 0.18915548037371513 | |
| }, | |
| { | |
| "name": "mnli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.3232, | |
| "acc_norm": 0.334 | |
| }, | |
| { | |
| "name": "news_cat", | |
| "task": "text_classification", | |
| "acc": 0.58, | |
| "acc_norm": 0.532 | |
| }, | |
| { | |
| "name": "offenseval_tr", | |
| "task": "text_classification", | |
| "acc": 0.4671201814058957, | |
| "acc_norm": 0.7820294784580499 | |
| }, | |
| { | |
| "name": "relevance_judgment", | |
| "task": "multiple_choice", | |
| "acc": 0.56672760511883, | |
| "acc_norm": 0.5781535648994516 | |
| }, | |
| { | |
| "name": "snli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.3239, | |
| "acc_norm": 0.3105 | |
| }, | |
| { | |
| "name": "sts_tr", | |
| "task": "text_classification", | |
| "acc": 0.17113850616388687, | |
| "acc_norm": 0.22552574329224076 | |
| }, | |
| { | |
| "name": "tquad", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.06278026905829596, | |
| "f1": 0.21486130318406463 | |
| }, | |
| { | |
| "name": "turkish_plu_goal_inference", | |
| "task": "multiple_choice", | |
| "acc": 0.35842293906810035, | |
| "acc_norm": 0.4026284348864994 | |
| }, | |
| { | |
| "name": "turkish_plu_next_event_prediction", | |
| "task": "multiple_choice", | |
| "acc": 0.3709923664122137, | |
| "acc_norm": 0.467175572519084 | |
| }, | |
| { | |
| "name": "turkish_plu_step_inference", | |
| "task": "multiple_choice", | |
| "acc": 0.27941176470588236, | |
| "acc_norm": 0.41830065359477125 | |
| }, | |
| { | |
| "name": "turkish_plu_step_ordering", | |
| "task": "multiple_choice", | |
| "acc": 0.5759059745347699, | |
| "acc_norm": 0.5759059745347699 | |
| }, | |
| { | |
| "name": "wiki_lingua_tr", | |
| "task": "summarization", | |
| "rouge1": 0.10861529436199803, | |
| "rouge2": 0.034862923521078545, | |
| "rougeL": 0.08692160533533941 | |
| }, | |
| { | |
| "name": "wmt-tr-en-prompt", | |
| "task": "machine_translation", | |
| "wer": 3.910683208136067, | |
| "bleu": 0.012043288243775466 | |
| }, | |
| { | |
| "name": "xcopa_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.556, | |
| "acc_norm": 0.556 | |
| }, | |
| { | |
| "name": "xlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.16924699150407269, | |
| "rouge2": 0.07190935921365724, | |
| "rougeL": 0.13255123335488528 | |
| }, | |
| { | |
| "name": "xnli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.4389558232931727, | |
| "acc_norm": 0.4389558232931727 | |
| }, | |
| { | |
| "name": "xquad_tr", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.04873949579831932, | |
| "f1": 0.11156636293859905 | |
| }, | |
| { | |
| "name": "gecturk_generation", | |
| "task": "grammatical_error_correction", | |
| "exact_match": 0.0073185998362944775 | |
| }, | |
| { | |
| "name": "mlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.35440052022111407, | |
| "rouge2": 0.2215476501673455, | |
| "rougeL": 0.2911311598176804 | |
| }, | |
| { | |
| "name": "wiki_lingua_tr", | |
| "task": "summarization", | |
| "rouge1": 0.18510384577665046, | |
| "rouge2": 0.056181066004903614, | |
| "rougeL": 0.1392211003290612 | |
| }, | |
| { | |
| "name": "wmt-tr-en-prompt", | |
| "task": "machine_translation", | |
| "wer": 1.311990023748812, | |
| "bleu": 0.02624044942774961 | |
| }, | |
| { | |
| "name": "tr-wikihow-summ", | |
| "task": "summarization", | |
| "rouge1": 0.20233829108374618, | |
| "rouge2": 0.05924574844289239, | |
| "rougeL": 0.14920828977218778 | |
| }, | |
| { | |
| "name": "xlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.2429304790539497, | |
| "rouge2": 0.09668008744707143, | |
| "rougeL": 0.18327092913535944 | |
| } | |
| ] | |
| } |