Cetvel / results /zero-shot /mistral-7b.json
Ilker Kesen
initialize the first version
500fbd7
{
"model": {
"dtype": "bfloat16",
"max_length": "4096",
"model": "mistralai/Mistral-7B-v0.1",
"api": "hf",
"architecture": "MixtralForCausalLM",
"type": "pretrained",
"num_parameters": "7b"
},
"results": [
{
"name": "xquad_tr",
"task": "extractive_question_answering",
"exact_match": 0.16722689075630254,
"f1": 0.32150094374615246
},
{
"name": "xcopa_tr",
"task": "multiple_choice",
"acc": 0.566,
"acc_norm": 0.566
},
{
"name": "turkish_plu",
"task": "multiple_choice",
"acc": 0.45152,
"acc_norm": 0.5136
},
{
"name": "turkish_plu_goal_inference",
"task": "multiple_choice",
"acc": 0.42771804062126645,
"acc_norm": 0.46714456391875747
},
{
"name": "turkish_plu_next_event_prediction",
"task": "multiple_choice",
"acc": 0.39541984732824426,
"acc_norm": 0.5022900763358779
},
{
"name": "turkish_plu_step_inference",
"task": "multiple_choice",
"acc": 0.29248366013071897,
"acc_norm": 0.4411764705882353
},
{
"name": "turkish_plu_step_ordering",
"task": "multiple_choice",
"acc": 0.6023506366307542,
"acc_norm": 0.6023506366307542
},
{
"name": "check_worthiness",
"task": "multiple_choice",
"acc": 0.37614259597806216,
"acc_norm": 0.42458866544789764
},
{
"name": "relevance_judgment",
"task": "multiple_choice",
"acc": 0.4218464351005484,
"acc_norm": 0.49588665447897623
},
{
"name": "tquad",
"task": "extractive_question_answering",
"exact_match": 0.2096412556053812,
"f1": 0.4767364701184728
},
{
"name": "sts_tr",
"task": "text_classification",
"acc": 0.135605511240029,
"acc_norm": 0.20522117476432197
},
{
"name": "offenseval_tr",
"task": "text_classification",
"acc": 0.2046485260770975,
"acc_norm": 0.3735827664399093
},
{
"name": "mnli_tr",
"task": "natural_language_inference",
"acc": 0.3194,
"acc_norm": 0.3267
},
{
"name": "snli_tr",
"task": "natural_language_inference",
"acc": 0.3196,
"acc_norm": 0.3201
},
{
"name": "xnli_tr",
"task": "natural_language_inference",
"acc": 0.331936127744511,
"acc_norm": 0.34910179640718564
},
{
"name": "news_cat",
"task": "text_classification",
"acc": 0.652,
"acc_norm": 0.44
},
{
"name": "mkqa_tr",
"task": "extractive_question_answering",
"exact_match": 0.12030186445693992,
"f1": 0.16163416207615164
},
{
"name": "ironytr",
"task": "text_classification",
"acc": 0.5016666666666667,
"acc_norm": 0.52
},
{
"name": "exams_tr",
"task": "multiple_choice",
"acc": 0.24173027989821882,
"acc_norm": 0.30279898218829515
},
{
"name": "belebele_tr",
"task": "multiple_choice",
"acc": 0.37444444444444447,
"acc_norm": 0.37444444444444447
},
{
"name": "gecturk_generation",
"task": "grammatical_error_correction",
"exact_match": 0.20660599932591844
},
{
"name": "mlsum_tr",
"task": "summarization",
"rouge1": 0.09403885616158554,
"rouge2": 0.06300721907752257,
"rougeL": 0.08169726458665999
},
{
"name": "wiki_lingua_tr",
"task": "summarization",
"rouge1": 0.1905392717787084,
"rouge2": 0.05957088325130176,
"rougeL": 0.1472985242082243
},
{
"name": "wmt-tr-en-prompt",
"task": "machine_translation",
"wer": 1.0876062644712858,
"bleu": 0.04973628734419603
},
{
"name": "xlsum_tr",
"task": "summarization",
"rouge1": 0.02720399421152351,
"rouge2": 0.012032606076011431,
"rougeL": 0.02311080687545987
}
]
}