diff --git a/mteb/AmazonCounterfactualClassification.json b/mteb/AmazonCounterfactualClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..1ed3a7a4a4f53f8351866f1dfac5cbbc482e7792 --- /dev/null +++ b/mteb/AmazonCounterfactualClassification.json @@ -0,0 +1,179 @@ +{ + "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205", + "task_name": "AmazonCounterfactualClassification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.65997, + "f1": 0.535951, + "f1_weighted": 0.725304, + "ap": 0.163093, + "ap_weighted": 0.163093, + "scores_per_experiment": [ + { + "accuracy": 0.644678, + "f1": 0.531347, + "f1_weighted": 0.713782, + "ap": 0.166602, + "ap_weighted": 0.166602 + }, + { + "accuracy": 0.608696, + "f1": 0.518256, + "f1_weighted": 0.683489, + "ap": 0.176106, + "ap_weighted": 0.176106 + }, + { + "accuracy": 0.643928, + "f1": 0.529901, + "f1_weighted": 0.713177, + "ap": 0.164896, + "ap_weighted": 0.164896 + }, + { + "accuracy": 0.641679, + "f1": 0.51983, + "f1_weighted": 0.711307, + "ap": 0.151661, + "ap_weighted": 0.151661 + }, + { + "accuracy": 0.668666, + "f1": 0.544738, + "f1_weighted": 0.732766, + "ap": 0.169199, + "ap_weighted": 0.169199 + }, + { + "accuracy": 0.707646, + "f1": 0.550963, + "f1_weighted": 0.760934, + "ap": 0.15327, + "ap_weighted": 0.15327 + }, + { + "accuracy": 0.654423, + "f1": 0.525677, + "f1_weighted": 0.721296, + "ap": 0.151452, + "ap_weighted": 0.151452 + }, + { + "accuracy": 0.664918, + "f1": 0.539092, + "f1_weighted": 0.729726, + "ap": 0.163361, + "ap_weighted": 0.163361 + }, + { + "accuracy": 0.696402, + "f1": 0.563025, + "f1_weighted": 0.754132, + "ap": 0.176998, + "ap_weighted": 0.176998 + }, + { + "accuracy": 0.668666, + "f1": 0.536679, + "f1_weighted": 0.732435, + "ap": 0.15738, + "ap_weighted": 0.15738 + } + ], + "main_score": 0.65997, + "hf_subset": "en-ext", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.64194, + "f1": 0.585969, + "f1_weighted": 0.677482, + "ap": 0.280748, + "ap_weighted": 0.280748, + "scores_per_experiment": [ + { + "accuracy": 0.659701, + "f1": 0.606678, + "f1_weighted": 0.694619, + "ap": 0.299469, + "ap_weighted": 0.299469 + }, + { + "accuracy": 0.61194, + "f1": 0.564082, + "f1_weighted": 0.652038, + "ap": 0.268001, + "ap_weighted": 0.268001 + }, + { + "accuracy": 0.592537, + "f1": 0.553977, + "f1_weighted": 0.633838, + "ap": 0.269473, + "ap_weighted": 0.269473 + }, + { + "accuracy": 0.61194, + "f1": 0.565044, + "f1_weighted": 0.652016, + "ap": 0.269697, + "ap_weighted": 0.269697 + }, + { + "accuracy": 0.668657, + "f1": 0.603271, + "f1_weighted": 0.70135, + "ap": 0.285582, + "ap_weighted": 0.285582 + }, + { + "accuracy": 0.625373, + "f1": 0.56863, + "f1_weighted": 0.663902, + "ap": 0.26389, + "ap_weighted": 0.26389 + }, + { + "accuracy": 0.744776, + "f1": 0.648233, + "f1_weighted": 0.760454, + "ap": 0.307423, + "ap_weighted": 0.307423 + }, + { + "accuracy": 0.667164, + "f1": 0.613827, + "f1_weighted": 0.701223, + "ap": 0.30575, + "ap_weighted": 0.30575 + }, + { + "accuracy": 0.591045, + "f1": 0.547427, + "f1_weighted": 0.632985, + "ap": 0.259001, + "ap_weighted": 0.259001 + }, + { + "accuracy": 0.646269, + "f1": 0.588519, + "f1_weighted": 0.682391, + "ap": 0.279194, + "ap_weighted": 0.279194 + } + ], + "main_score": 0.64194, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 18.05203342437744, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/AmazonPolarityClassification.json b/mteb/AmazonPolarityClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..f0f3a65bee36dd97551ff68a3acb74fa9b3a8c77 --- /dev/null +++ b/mteb/AmazonPolarityClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e2d317d38cd51312af73b3d32a06d1a08b442046", + "task_name": "AmazonPolarityClassification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.671713, + "f1": 0.669443, + "f1_weighted": 0.669443, + "ap": 0.617296, + "ap_weighted": 0.617296, + "scores_per_experiment": [ + { + "accuracy": 0.730195, + "f1": 0.729644, + "f1_weighted": 0.729644, + "ap": 0.673348, + "ap_weighted": 0.673348 + }, + { + "accuracy": 0.648605, + "f1": 0.647126, + "f1_weighted": 0.647126, + "ap": 0.593854, + "ap_weighted": 0.593854 + }, + { + "accuracy": 0.720182, + "f1": 0.716345, + "f1_weighted": 0.716345, + "ap": 0.649422, + "ap_weighted": 0.649422 + }, + { + "accuracy": 0.717512, + "f1": 0.715561, + "f1_weighted": 0.715561, + "ap": 0.665462, + "ap_weighted": 0.665462 + }, + { + "accuracy": 0.660528, + "f1": 0.659856, + "f1_weighted": 0.659856, + "ap": 0.60393, + "ap_weighted": 0.60393 + }, + { + "accuracy": 0.66763, + "f1": 0.66655, + "f1_weighted": 0.66655, + "ap": 0.609044, + "ap_weighted": 0.609044 + }, + { + "accuracy": 0.67263, + "f1": 0.670559, + "f1_weighted": 0.670559, + "ap": 0.612038, + "ap_weighted": 0.612038 + }, + { + "accuracy": 0.677455, + "f1": 0.674938, + "f1_weighted": 0.674938, + "ap": 0.626943, + "ap_weighted": 0.626943 + }, + { + "accuracy": 0.582143, + "f1": 0.574911, + "f1_weighted": 0.574911, + "ap": 0.546423, + "ap_weighted": 0.546423 + }, + { + "accuracy": 0.64025, + "f1": 0.638938, + "f1_weighted": 0.638938, + "ap": 0.592492, + "ap_weighted": 0.592492 + } + ], + "main_score": 0.671713, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 439.54625058174133, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/AmazonReviewsClassification.json b/mteb/AmazonReviewsClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..28a4b86fa77fcfccf6346ae904fa8eab9a0e4e4c --- /dev/null +++ b/mteb/AmazonReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d", + "task_name": "AmazonReviewsClassification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.32318, + "f1": 0.319973, + "f1_weighted": 0.319973, + "scores_per_experiment": [ + { + "accuracy": 0.341, + "f1": 0.333855, + "f1_weighted": 0.333855 + }, + { + "accuracy": 0.3368, + "f1": 0.338041, + "f1_weighted": 0.338041 + }, + { + "accuracy": 0.3336, + "f1": 0.330524, + "f1_weighted": 0.330524 + }, + { + "accuracy": 0.3086, + "f1": 0.309647, + "f1_weighted": 0.309647 + }, + { + "accuracy": 0.3114, + "f1": 0.305097, + "f1_weighted": 0.305097 + }, + { + "accuracy": 0.3334, + "f1": 0.330257, + "f1_weighted": 0.330257 + }, + { + "accuracy": 0.299, + "f1": 0.292306, + "f1_weighted": 0.292306 + }, + { + "accuracy": 0.3384, + "f1": 0.336788, + "f1_weighted": 0.336788 + }, + { + "accuracy": 0.3242, + "f1": 0.31879, + "f1_weighted": 0.31879 + }, + { + "accuracy": 0.3054, + "f1": 0.304421, + "f1_weighted": 0.304421 + } + ], + "main_score": 0.32318, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.712511777877808, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/ArguAna.json b/mteb/ArguAna.json new file mode 100644 index 0000000000000000000000000000000000000000..22daa51005ec9b728f797b73f9eeae1e1566003b --- /dev/null +++ b/mteb/ArguAna.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a", + "task_name": "ArguAna", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.24609, + "ndcg_at_3": 0.36278, + "ndcg_at_5": 0.40903, + "ndcg_at_10": 0.46381, + "ndcg_at_20": 0.49571, + "ndcg_at_100": 0.51666, + "ndcg_at_1000": 0.52098, + "map_at_1": 0.24609, + "map_at_3": 0.33404, + "map_at_5": 0.35968, + "map_at_10": 0.38204, + "map_at_20": 0.39113, + "map_at_100": 0.39432, + "map_at_1000": 0.3945, + "recall_at_1": 0.24609, + "recall_at_3": 0.44595, + "recall_at_5": 0.55832, + "recall_at_10": 0.72902, + "recall_at_20": 0.85277, + "recall_at_100": 0.96159, + "recall_at_1000": 0.99431, + "precision_at_1": 0.24609, + "precision_at_3": 0.14865, + "precision_at_5": 0.11166, + "precision_at_10": 0.0729, + "precision_at_20": 0.04264, + "precision_at_100": 0.00962, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.251067, + "mrr_at_3": 0.335941, + "mrr_at_5": 0.361546, + "mrr_at_10": 0.383849, + "mrr_at_20": 0.393004, + "mrr_at_100": 0.396189, + "mrr_at_1000": 0.396374, + "nauc_ndcg_at_1_max": -0.056006, + "nauc_ndcg_at_1_std": -0.048049, + "nauc_ndcg_at_1_diff1": 0.14903, + "nauc_ndcg_at_3_max": -0.00408, + "nauc_ndcg_at_3_std": -0.019375, + "nauc_ndcg_at_3_diff1": 0.121454, + "nauc_ndcg_at_5_max": -0.004685, + "nauc_ndcg_at_5_std": -0.020393, + "nauc_ndcg_at_5_diff1": 0.107318, + "nauc_ndcg_at_10_max": 0.030675, + "nauc_ndcg_at_10_std": -0.003638, + "nauc_ndcg_at_10_diff1": 0.113071, + "nauc_ndcg_at_20_max": 0.025101, + "nauc_ndcg_at_20_std": -0.003348, + "nauc_ndcg_at_20_diff1": 0.115383, + "nauc_ndcg_at_100_max": 0.010754, + "nauc_ndcg_at_100_std": 0.001077, + "nauc_ndcg_at_100_diff1": 0.118117, + "nauc_ndcg_at_1000_max": 0.005791, + "nauc_ndcg_at_1000_std": -0.006433, + "nauc_ndcg_at_1000_diff1": 0.119586, + "nauc_map_at_1_max": -0.056006, + "nauc_map_at_1_std": -0.048049, + "nauc_map_at_1_diff1": 0.14903, + "nauc_map_at_3_max": -0.015511, + "nauc_map_at_3_std": -0.024433, + "nauc_map_at_3_diff1": 0.126974, + "nauc_map_at_5_max": -0.015512, + "nauc_map_at_5_std": -0.024537, + "nauc_map_at_5_diff1": 0.11982, + "nauc_map_at_10_max": -0.00244, + "nauc_map_at_10_std": -0.017698, + "nauc_map_at_10_diff1": 0.122534, + "nauc_map_at_20_max": -0.00477, + "nauc_map_at_20_std": -0.018189, + "nauc_map_at_20_diff1": 0.123218, + "nauc_map_at_100_max": -0.006815, + "nauc_map_at_100_std": -0.017488, + "nauc_map_at_100_diff1": 0.123542, + "nauc_map_at_1000_max": -0.006971, + "nauc_map_at_1000_std": -0.017692, + "nauc_map_at_1000_diff1": 0.123594, + "nauc_recall_at_1_max": -0.056006, + "nauc_recall_at_1_std": -0.048049, + "nauc_recall_at_1_diff1": 0.14903, + "nauc_recall_at_3_max": 0.026873, + "nauc_recall_at_3_std": -0.006182, + "nauc_recall_at_3_diff1": 0.106725, + "nauc_recall_at_5_max": 0.025734, + "nauc_recall_at_5_std": -0.009853, + "nauc_recall_at_5_diff1": 0.068572, + "nauc_recall_at_10_max": 0.178111, + "nauc_recall_at_10_std": 0.056516, + "nauc_recall_at_10_diff1": 0.07601, + "nauc_recall_at_20_max": 0.239859, + "nauc_recall_at_20_std": 0.104235, + "nauc_recall_at_20_diff1": 0.070172, + "nauc_recall_at_100_max": 0.307669, + "nauc_recall_at_100_std": 0.509552, + "nauc_recall_at_100_diff1": 0.051657, + "nauc_recall_at_1000_max": 0.254176, + "nauc_recall_at_1000_std": 0.730444, + "nauc_recall_at_1000_diff1": 0.166349, + "nauc_precision_at_1_max": -0.056006, + "nauc_precision_at_1_std": -0.048049, + "nauc_precision_at_1_diff1": 0.14903, + "nauc_precision_at_3_max": 0.026873, + "nauc_precision_at_3_std": -0.006182, + "nauc_precision_at_3_diff1": 0.106725, + "nauc_precision_at_5_max": 0.025734, + "nauc_precision_at_5_std": -0.009853, + "nauc_precision_at_5_diff1": 0.068572, + "nauc_precision_at_10_max": 0.178111, + "nauc_precision_at_10_std": 0.056516, + "nauc_precision_at_10_diff1": 0.07601, + "nauc_precision_at_20_max": 0.239859, + "nauc_precision_at_20_std": 0.104235, + "nauc_precision_at_20_diff1": 0.070172, + "nauc_precision_at_100_max": 0.307669, + "nauc_precision_at_100_std": 0.509552, + "nauc_precision_at_100_diff1": 0.051657, + "nauc_precision_at_1000_max": 0.254176, + "nauc_precision_at_1000_std": 0.730444, + "nauc_precision_at_1000_diff1": 0.166349, + "nauc_mrr_at_1_max": -0.051381, + "nauc_mrr_at_1_std": -0.041855, + "nauc_mrr_at_1_diff1": 0.132056, + "nauc_mrr_at_3_max": -0.017796, + "nauc_mrr_at_3_std": -0.023432, + "nauc_mrr_at_3_diff1": 0.114369, + "nauc_mrr_at_5_max": -0.017394, + "nauc_mrr_at_5_std": -0.023168, + "nauc_mrr_at_5_diff1": 0.107454, + "nauc_mrr_at_10_max": -0.005076, + "nauc_mrr_at_10_std": -0.016223, + "nauc_mrr_at_10_diff1": 0.109067, + "nauc_mrr_at_20_max": -0.00715, + "nauc_mrr_at_20_std": -0.016808, + "nauc_mrr_at_20_diff1": 0.109766, + "nauc_mrr_at_100_max": -0.009223, + "nauc_mrr_at_100_std": -0.016097, + "nauc_mrr_at_100_diff1": 0.109945, + "nauc_mrr_at_1000_max": -0.00938, + "nauc_mrr_at_1000_std": -0.0163, + "nauc_mrr_at_1000_diff1": 0.109988, + "main_score": 0.46381, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 18.032260179519653, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/ArxivClusteringP2P.json b/mteb/ArxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..01ed4f0ebb6ed0158d42b11a27521662f91780b0 --- /dev/null +++ b/mteb/ArxivClusteringP2P.json @@ -0,0 +1,53 @@ +{ + "dataset_revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d", + "task_name": "ArxivClusteringP2P", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "v_measure": 0.389781, + "v_measure_std": 0.140064, + "v_measures": [ + 0.397318, + 0.395931, + 0.398734, + 0.421181, + 0.402112, + 0.39659, + 0.40027, + 0.4156, + 0.418672, + 0.408705, + 0.442656, + 0.441182, + 0.438416, + 0.447467, + 0.440814, + 0.442237, + 0.445092, + 0.440632, + 0.445347, + 0.437578, + 0.391066, + 0.265296, + 0.30278, + 0.296997, + 0.291818, + 0.212322, + 0.256389, + 0.159543, + 0.200181, + 1.0, + 0.230291 + ], + "main_score": 0.389781, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1234.510992050171, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/ArxivClusteringS2S.json b/mteb/ArxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..239503c80a116a05ccc215e9f6a9acf8766115c1 --- /dev/null +++ b/mteb/ArxivClusteringS2S.json @@ -0,0 +1,53 @@ +{ + "dataset_revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53", + "task_name": "ArxivClusteringS2S", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "v_measure": 0.289645, + "v_measure_std": 0.142769, + "v_measures": [ + 0.284647, + 0.288473, + 0.27734, + 0.269279, + 0.263681, + 0.266674, + 0.272582, + 0.251594, + 0.287376, + 0.275816, + 0.326475, + 0.330363, + 0.320361, + 0.330296, + 0.328716, + 0.325683, + 0.321654, + 0.324298, + 0.326664, + 0.316507, + 0.304434, + 0.188541, + 0.204597, + 0.259977, + 0.237603, + 0.164765, + 0.199714, + 0.108873, + 0.155307, + 1.0, + 0.166707 + ], + "main_score": 0.289645, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 300.5607361793518, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/AskUbuntuDupQuestions.json b/mteb/AskUbuntuDupQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..0c4b75b757d4abfbf7bc366f23b3b5ba70f774a4 --- /dev/null +++ b/mteb/AskUbuntuDupQuestions.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2000358ca161889fa9c082cb41daa8dcfb161a54", + "task_name": "AskUbuntuDupQuestions", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "map": 0.579553, + "mrr": 0.708293, + "nAUC_map_max": 0.232041, + "nAUC_map_std": 0.210013, + "nAUC_map_diff1": 0.067763, + "nAUC_mrr_max": 0.331435, + "nAUC_mrr_std": 0.258722, + "nAUC_mrr_diff1": 0.102834, + "main_score": 0.579553, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.8852455615997314, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/BIOSSES.json b/mteb/BIOSSES.json new file mode 100644 index 0000000000000000000000000000000000000000..65507bf63a5f71cbedc18786149507ed7dea53b0 --- /dev/null +++ b/mteb/BIOSSES.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a", + "task_name": "BIOSSES", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "pearson": 0.850909, + "spearman": 0.807386, + "cosine_pearson": 0.850909, + "cosine_spearman": 0.807386, + "manhattan_pearson": 0.820327, + "manhattan_spearman": 0.78562, + "euclidean_pearson": 0.826159, + "euclidean_spearman": 0.793542, + "main_score": 0.807386, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.15388250350952148, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/Banking77Classification.json b/mteb/Banking77Classification.json new file mode 100644 index 0000000000000000000000000000000000000000..ffe1bac7d5457ccac26ee41da879480254cc1b85 --- /dev/null +++ b/mteb/Banking77Classification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "0fd18e25b25c072e09e0d92ab615fda904d66300", + "task_name": "Banking77Classification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.82026, + "f1": 0.819738, + "f1_weighted": 0.819738, + "scores_per_experiment": [ + { + "accuracy": 0.820779, + "f1": 0.820659, + "f1_weighted": 0.820659 + }, + { + "accuracy": 0.827273, + "f1": 0.827807, + "f1_weighted": 0.827807 + }, + { + "accuracy": 0.837013, + "f1": 0.836189, + "f1_weighted": 0.836189 + }, + { + "accuracy": 0.827273, + "f1": 0.827111, + "f1_weighted": 0.827111 + }, + { + "accuracy": 0.816558, + "f1": 0.816573, + "f1_weighted": 0.816573 + }, + { + "accuracy": 0.800649, + "f1": 0.79907, + "f1_weighted": 0.79907 + }, + { + "accuracy": 0.816558, + "f1": 0.815777, + "f1_weighted": 0.815777 + }, + { + "accuracy": 0.812338, + "f1": 0.811554, + "f1_weighted": 0.811554 + }, + { + "accuracy": 0.828247, + "f1": 0.828126, + "f1_weighted": 0.828126 + }, + { + "accuracy": 0.815909, + "f1": 0.814514, + "f1_weighted": 0.814514 + } + ], + "main_score": 0.82026, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11.350855588912964, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/BiorxivClusteringP2P.json b/mteb/BiorxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..b61f99419e7617898e532c88153c6b611e2cf630 --- /dev/null +++ b/mteb/BiorxivClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40", + "task_name": "BiorxivClusteringP2P", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "v_measure": 0.30787, + "v_measure_std": 0.009286, + "v_measures": [ + 0.294563, + 0.300806, + 0.300434, + 0.298455, + 0.313446, + 0.313647, + 0.310572, + 0.303024, + 0.3205, + 0.323252 + ], + "main_score": 0.30787, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 183.2675678730011, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/BiorxivClusteringS2S.json b/mteb/BiorxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..edb6bf759830e41e1242e1837e00a01a70cf558c --- /dev/null +++ b/mteb/BiorxivClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908", + "task_name": "BiorxivClusteringS2S", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "v_measure": 0.270998, + "v_measure_std": 0.007908, + "v_measures": [ + 0.269737, + 0.264231, + 0.263528, + 0.267448, + 0.268571, + 0.273179, + 0.261491, + 0.27174, + 0.282342, + 0.287709 + ], + "main_score": 0.270998, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 32.66898798942566, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackAndroidRetrieval.json b/mteb/CQADupstackAndroidRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..8fdcd4cb062fde35c982679b5f188e2062fac656 --- /dev/null +++ b/mteb/CQADupstackAndroidRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f46a197baaae43b4f621051089b82a364682dfeb", + "task_name": "CQADupstackAndroidRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.29757, + "ndcg_at_3": 0.34736, + "ndcg_at_5": 0.36529, + "ndcg_at_10": 0.39114, + "ndcg_at_20": 0.4105, + "ndcg_at_100": 0.44359, + "ndcg_at_1000": 0.47015, + "map_at_1": 0.24501, + "map_at_3": 0.30857, + "map_at_5": 0.32307, + "map_at_10": 0.33582, + "map_at_20": 0.34229, + "map_at_100": 0.34801, + "map_at_1000": 0.34936, + "recall_at_1": 0.24501, + "recall_at_3": 0.37393, + "recall_at_5": 0.42701, + "recall_at_10": 0.5047, + "recall_at_20": 0.57451, + "recall_at_100": 0.72912, + "recall_at_1000": 0.90643, + "precision_at_1": 0.29757, + "precision_at_3": 0.16738, + "precision_at_5": 0.11788, + "precision_at_10": 0.07282, + "precision_at_20": 0.04349, + "precision_at_100": 0.01209, + "precision_at_1000": 0.00172, + "mrr_at_1": 0.297568, + "mrr_at_3": 0.360515, + "mrr_at_5": 0.37432, + "mrr_at_10": 0.384942, + "mrr_at_20": 0.389932, + "mrr_at_100": 0.393632, + "mrr_at_1000": 0.394271, + "nauc_ndcg_at_1_max": 0.35163, + "nauc_ndcg_at_1_std": -0.04949, + "nauc_ndcg_at_1_diff1": 0.506784, + "nauc_ndcg_at_3_max": 0.360879, + "nauc_ndcg_at_3_std": -0.032672, + "nauc_ndcg_at_3_diff1": 0.472847, + "nauc_ndcg_at_5_max": 0.364638, + "nauc_ndcg_at_5_std": -0.016767, + "nauc_ndcg_at_5_diff1": 0.466553, + "nauc_ndcg_at_10_max": 0.373779, + "nauc_ndcg_at_10_std": -0.007015, + "nauc_ndcg_at_10_diff1": 0.468114, + "nauc_ndcg_at_20_max": 0.374599, + "nauc_ndcg_at_20_std": -0.000285, + "nauc_ndcg_at_20_diff1": 0.460127, + "nauc_ndcg_at_100_max": 0.376178, + "nauc_ndcg_at_100_std": 0.003917, + "nauc_ndcg_at_100_diff1": 0.459498, + "nauc_ndcg_at_1000_max": 0.378896, + "nauc_ndcg_at_1000_std": 0.011141, + "nauc_ndcg_at_1000_diff1": 0.45977, + "nauc_map_at_1_max": 0.329735, + "nauc_map_at_1_std": -0.06633, + "nauc_map_at_1_diff1": 0.52641, + "nauc_map_at_3_max": 0.352305, + "nauc_map_at_3_std": -0.050847, + "nauc_map_at_3_diff1": 0.496157, + "nauc_map_at_5_max": 0.356133, + "nauc_map_at_5_std": -0.040399, + "nauc_map_at_5_diff1": 0.488895, + "nauc_map_at_10_max": 0.361679, + "nauc_map_at_10_std": -0.03574, + "nauc_map_at_10_diff1": 0.487396, + "nauc_map_at_20_max": 0.362918, + "nauc_map_at_20_std": -0.032796, + "nauc_map_at_20_diff1": 0.484663, + "nauc_map_at_100_max": 0.364498, + "nauc_map_at_100_std": -0.030688, + "nauc_map_at_100_diff1": 0.484524, + "nauc_map_at_1000_max": 0.364623, + "nauc_map_at_1000_std": -0.030062, + "nauc_map_at_1000_diff1": 0.484259, + "nauc_recall_at_1_max": 0.329735, + "nauc_recall_at_1_std": -0.06633, + "nauc_recall_at_1_diff1": 0.52641, + "nauc_recall_at_3_max": 0.34925, + "nauc_recall_at_3_std": -0.020609, + "nauc_recall_at_3_diff1": 0.441485, + "nauc_recall_at_5_max": 0.347606, + "nauc_recall_at_5_std": 0.016887, + "nauc_recall_at_5_diff1": 0.414227, + "nauc_recall_at_10_max": 0.3657, + "nauc_recall_at_10_std": 0.055427, + "nauc_recall_at_10_diff1": 0.409419, + "nauc_recall_at_20_max": 0.366151, + "nauc_recall_at_20_std": 0.091305, + "nauc_recall_at_20_diff1": 0.372562, + "nauc_recall_at_100_max": 0.375285, + "nauc_recall_at_100_std": 0.138761, + "nauc_recall_at_100_diff1": 0.355754, + "nauc_recall_at_1000_max": 0.485408, + "nauc_recall_at_1000_std": 0.470295, + "nauc_recall_at_1000_diff1": 0.311568, + "nauc_precision_at_1_max": 0.35163, + "nauc_precision_at_1_std": -0.04949, + "nauc_precision_at_1_diff1": 0.506784, + "nauc_precision_at_3_max": 0.332905, + "nauc_precision_at_3_std": -0.016148, + "nauc_precision_at_3_diff1": 0.331776, + "nauc_precision_at_5_max": 0.333257, + "nauc_precision_at_5_std": 0.04402, + "nauc_precision_at_5_diff1": 0.272371, + "nauc_precision_at_10_max": 0.307887, + "nauc_precision_at_10_std": 0.070138, + "nauc_precision_at_10_diff1": 0.20856, + "nauc_precision_at_20_max": 0.27621, + "nauc_precision_at_20_std": 0.099861, + "nauc_precision_at_20_diff1": 0.131662, + "nauc_precision_at_100_max": 0.176654, + "nauc_precision_at_100_std": 0.126523, + "nauc_precision_at_100_diff1": 0.007577, + "nauc_precision_at_1000_max": -0.000854, + "nauc_precision_at_1000_std": 0.095769, + "nauc_precision_at_1000_diff1": -0.16308, + "nauc_mrr_at_1_max": 0.35163, + "nauc_mrr_at_1_std": -0.04949, + "nauc_mrr_at_1_diff1": 0.506784, + "nauc_mrr_at_3_max": 0.363793, + "nauc_mrr_at_3_std": -0.03288, + "nauc_mrr_at_3_diff1": 0.469367, + "nauc_mrr_at_5_max": 0.364904, + "nauc_mrr_at_5_std": -0.021184, + "nauc_mrr_at_5_diff1": 0.463741, + "nauc_mrr_at_10_max": 0.368009, + "nauc_mrr_at_10_std": -0.016722, + "nauc_mrr_at_10_diff1": 0.464536, + "nauc_mrr_at_20_max": 0.368492, + "nauc_mrr_at_20_std": -0.015649, + "nauc_mrr_at_20_diff1": 0.462973, + "nauc_mrr_at_100_max": 0.367136, + "nauc_mrr_at_100_std": -0.016903, + "nauc_mrr_at_100_diff1": 0.463234, + "nauc_mrr_at_1000_max": 0.367098, + "nauc_mrr_at_1000_std": -0.016532, + "nauc_mrr_at_1000_diff1": 0.463207, + "main_score": 0.39114, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 27.28526473045349, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackEnglishRetrieval.json b/mteb/CQADupstackEnglishRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..1eca0d7a0bec63dbefc8aec97afb8461cdf281ae --- /dev/null +++ b/mteb/CQADupstackEnglishRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0", + "task_name": "CQADupstackEnglishRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.32229, + "ndcg_at_3": 0.35352, + "ndcg_at_5": 0.36907, + "ndcg_at_10": 0.38867, + "ndcg_at_20": 0.40624, + "ndcg_at_100": 0.43169, + "ndcg_at_1000": 0.45411, + "map_at_1": 0.25286, + "map_at_3": 0.31391, + "map_at_5": 0.3276, + "map_at_10": 0.33835, + "map_at_20": 0.34453, + "map_at_100": 0.34926, + "map_at_1000": 0.35039, + "recall_at_1": 0.25286, + "recall_at_3": 0.36961, + "recall_at_5": 0.4154, + "recall_at_10": 0.47543, + "recall_at_20": 0.53979, + "recall_at_100": 0.65744, + "recall_at_1000": 0.80437, + "precision_at_1": 0.32229, + "precision_at_3": 0.17113, + "precision_at_5": 0.11962, + "precision_at_10": 0.0721, + "precision_at_20": 0.04287, + "precision_at_100": 0.01177, + "precision_at_1000": 0.00165, + "mrr_at_1": 0.322293, + "mrr_at_3": 0.377707, + "mrr_at_5": 0.388917, + "mrr_at_10": 0.398009, + "mrr_at_20": 0.402384, + "mrr_at_100": 0.40498, + "mrr_at_1000": 0.40545, + "nauc_ndcg_at_1_max": 0.386829, + "nauc_ndcg_at_1_std": 0.074424, + "nauc_ndcg_at_1_diff1": 0.528493, + "nauc_ndcg_at_3_max": 0.370654, + "nauc_ndcg_at_3_std": 0.054984, + "nauc_ndcg_at_3_diff1": 0.466131, + "nauc_ndcg_at_5_max": 0.36923, + "nauc_ndcg_at_5_std": 0.061408, + "nauc_ndcg_at_5_diff1": 0.457153, + "nauc_ndcg_at_10_max": 0.365462, + "nauc_ndcg_at_10_std": 0.062711, + "nauc_ndcg_at_10_diff1": 0.455885, + "nauc_ndcg_at_20_max": 0.364087, + "nauc_ndcg_at_20_std": 0.064921, + "nauc_ndcg_at_20_diff1": 0.453433, + "nauc_ndcg_at_100_max": 0.366238, + "nauc_ndcg_at_100_std": 0.083159, + "nauc_ndcg_at_100_diff1": 0.447424, + "nauc_ndcg_at_1000_max": 0.37042, + "nauc_ndcg_at_1000_std": 0.089336, + "nauc_ndcg_at_1000_diff1": 0.448673, + "nauc_map_at_1_max": 0.332074, + "nauc_map_at_1_std": 0.021052, + "nauc_map_at_1_diff1": 0.546382, + "nauc_map_at_3_max": 0.352336, + "nauc_map_at_3_std": 0.025225, + "nauc_map_at_3_diff1": 0.49159, + "nauc_map_at_5_max": 0.355997, + "nauc_map_at_5_std": 0.037594, + "nauc_map_at_5_diff1": 0.484115, + "nauc_map_at_10_max": 0.358431, + "nauc_map_at_10_std": 0.042814, + "nauc_map_at_10_diff1": 0.482818, + "nauc_map_at_20_max": 0.359632, + "nauc_map_at_20_std": 0.046509, + "nauc_map_at_20_diff1": 0.481984, + "nauc_map_at_100_max": 0.361803, + "nauc_map_at_100_std": 0.051838, + "nauc_map_at_100_diff1": 0.480556, + "nauc_map_at_1000_max": 0.362442, + "nauc_map_at_1000_std": 0.052821, + "nauc_map_at_1000_diff1": 0.480396, + "nauc_recall_at_1_max": 0.332074, + "nauc_recall_at_1_std": 0.021052, + "nauc_recall_at_1_diff1": 0.546382, + "nauc_recall_at_3_max": 0.33786, + "nauc_recall_at_3_std": 0.022159, + "nauc_recall_at_3_diff1": 0.421871, + "nauc_recall_at_5_max": 0.336662, + "nauc_recall_at_5_std": 0.046278, + "nauc_recall_at_5_diff1": 0.393118, + "nauc_recall_at_10_max": 0.322503, + "nauc_recall_at_10_std": 0.05947, + "nauc_recall_at_10_diff1": 0.37952, + "nauc_recall_at_20_max": 0.312259, + "nauc_recall_at_20_std": 0.068895, + "nauc_recall_at_20_diff1": 0.355049, + "nauc_recall_at_100_max": 0.303406, + "nauc_recall_at_100_std": 0.156142, + "nauc_recall_at_100_diff1": 0.295623, + "nauc_recall_at_1000_max": 0.305454, + "nauc_recall_at_1000_std": 0.214645, + "nauc_recall_at_1000_diff1": 0.271848, + "nauc_precision_at_1_max": 0.386829, + "nauc_precision_at_1_std": 0.074424, + "nauc_precision_at_1_diff1": 0.528493, + "nauc_precision_at_3_max": 0.370629, + "nauc_precision_at_3_std": 0.114377, + "nauc_precision_at_3_diff1": 0.322216, + "nauc_precision_at_5_max": 0.356068, + "nauc_precision_at_5_std": 0.161786, + "nauc_precision_at_5_diff1": 0.256865, + "nauc_precision_at_10_max": 0.336078, + "nauc_precision_at_10_std": 0.188689, + "nauc_precision_at_10_diff1": 0.195934, + "nauc_precision_at_20_max": 0.294667, + "nauc_precision_at_20_std": 0.210742, + "nauc_precision_at_20_diff1": 0.13482, + "nauc_precision_at_100_max": 0.229317, + "nauc_precision_at_100_std": 0.273075, + "nauc_precision_at_100_diff1": -0.000666, + "nauc_precision_at_1000_max": 0.139545, + "nauc_precision_at_1000_std": 0.264182, + "nauc_precision_at_1000_diff1": -0.103752, + "nauc_mrr_at_1_max": 0.386829, + "nauc_mrr_at_1_std": 0.074424, + "nauc_mrr_at_1_diff1": 0.528493, + "nauc_mrr_at_3_max": 0.387082, + "nauc_mrr_at_3_std": 0.075774, + "nauc_mrr_at_3_diff1": 0.478237, + "nauc_mrr_at_5_max": 0.385714, + "nauc_mrr_at_5_std": 0.078757, + "nauc_mrr_at_5_diff1": 0.46983, + "nauc_mrr_at_10_max": 0.385054, + "nauc_mrr_at_10_std": 0.079545, + "nauc_mrr_at_10_diff1": 0.469465, + "nauc_mrr_at_20_max": 0.383847, + "nauc_mrr_at_20_std": 0.079177, + "nauc_mrr_at_20_diff1": 0.468517, + "nauc_mrr_at_100_max": 0.38363, + "nauc_mrr_at_100_std": 0.081133, + "nauc_mrr_at_100_diff1": 0.467934, + "nauc_mrr_at_1000_max": 0.383753, + "nauc_mrr_at_1000_std": 0.08116, + "nauc_mrr_at_1000_diff1": 0.468037, + "main_score": 0.38867, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 41.29191493988037, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackGamingRetrieval.json b/mteb/CQADupstackGamingRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..dbd255293cbcefcb9d72290ec3ceda0d5519b53b --- /dev/null +++ b/mteb/CQADupstackGamingRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4885aa143210c98657558c04aaf3dc47cfb54340", + "task_name": "CQADupstackGamingRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.37555, + "ndcg_at_3": 0.43752, + "ndcg_at_5": 0.4595, + "ndcg_at_10": 0.48593, + "ndcg_at_20": 0.50274, + "ndcg_at_100": 0.52922, + "ndcg_at_1000": 0.5442, + "map_at_1": 0.32732, + "map_at_3": 0.40392, + "map_at_5": 0.41928, + "map_at_10": 0.43189, + "map_at_20": 0.4374, + "map_at_100": 0.44174, + "map_at_1000": 0.44242, + "recall_at_1": 0.32732, + "recall_at_3": 0.48, + "recall_at_5": 0.53475, + "recall_at_10": 0.61265, + "recall_at_20": 0.67586, + "recall_at_100": 0.80604, + "recall_at_1000": 0.91408, + "precision_at_1": 0.37555, + "precision_at_3": 0.19645, + "precision_at_5": 0.13455, + "precision_at_10": 0.079, + "precision_at_20": 0.04436, + "precision_at_100": 0.0109, + "precision_at_1000": 0.00127, + "mrr_at_1": 0.375549, + "mrr_at_3": 0.443574, + "mrr_at_5": 0.455455, + "mrr_at_10": 0.465721, + "mrr_at_20": 0.469989, + "mrr_at_100": 0.473172, + "mrr_at_1000": 0.473553, + "nauc_ndcg_at_1_max": 0.34123, + "nauc_ndcg_at_1_std": -0.017841, + "nauc_ndcg_at_1_diff1": 0.518322, + "nauc_ndcg_at_3_max": 0.35477, + "nauc_ndcg_at_3_std": -0.012643, + "nauc_ndcg_at_3_diff1": 0.475991, + "nauc_ndcg_at_5_max": 0.362862, + "nauc_ndcg_at_5_std": -3.8e-05, + "nauc_ndcg_at_5_diff1": 0.465251, + "nauc_ndcg_at_10_max": 0.367788, + "nauc_ndcg_at_10_std": 0.008117, + "nauc_ndcg_at_10_diff1": 0.467285, + "nauc_ndcg_at_20_max": 0.375701, + "nauc_ndcg_at_20_std": 0.016506, + "nauc_ndcg_at_20_diff1": 0.46824, + "nauc_ndcg_at_100_max": 0.378108, + "nauc_ndcg_at_100_std": 0.029329, + "nauc_ndcg_at_100_diff1": 0.466243, + "nauc_ndcg_at_1000_max": 0.378672, + "nauc_ndcg_at_1000_std": 0.029359, + "nauc_ndcg_at_1000_diff1": 0.467787, + "nauc_map_at_1_max": 0.309187, + "nauc_map_at_1_std": -0.026286, + "nauc_map_at_1_diff1": 0.51255, + "nauc_map_at_3_max": 0.339493, + "nauc_map_at_3_std": -0.024583, + "nauc_map_at_3_diff1": 0.485462, + "nauc_map_at_5_max": 0.34567, + "nauc_map_at_5_std": -0.015681, + "nauc_map_at_5_diff1": 0.478419, + "nauc_map_at_10_max": 0.350466, + "nauc_map_at_10_std": -0.010182, + "nauc_map_at_10_diff1": 0.479976, + "nauc_map_at_20_max": 0.354014, + "nauc_map_at_20_std": -0.006443, + "nauc_map_at_20_diff1": 0.480216, + "nauc_map_at_100_max": 0.355061, + "nauc_map_at_100_std": -0.003215, + "nauc_map_at_100_diff1": 0.479392, + "nauc_map_at_1000_max": 0.35521, + "nauc_map_at_1000_std": -0.003054, + "nauc_map_at_1000_diff1": 0.479491, + "nauc_recall_at_1_max": 0.309187, + "nauc_recall_at_1_std": -0.026286, + "nauc_recall_at_1_diff1": 0.51255, + "nauc_recall_at_3_max": 0.34792, + "nauc_recall_at_3_std": -0.017638, + "nauc_recall_at_3_diff1": 0.440913, + "nauc_recall_at_5_max": 0.364564, + "nauc_recall_at_5_std": 0.014875, + "nauc_recall_at_5_diff1": 0.409842, + "nauc_recall_at_10_max": 0.372517, + "nauc_recall_at_10_std": 0.038871, + "nauc_recall_at_10_diff1": 0.405621, + "nauc_recall_at_20_max": 0.408612, + "nauc_recall_at_20_std": 0.080092, + "nauc_recall_at_20_diff1": 0.400661, + "nauc_recall_at_100_max": 0.431074, + "nauc_recall_at_100_std": 0.195316, + "nauc_recall_at_100_diff1": 0.363097, + "nauc_recall_at_1000_max": 0.493019, + "nauc_recall_at_1000_std": 0.343645, + "nauc_recall_at_1000_diff1": 0.316154, + "nauc_precision_at_1_max": 0.34123, + "nauc_precision_at_1_std": -0.017841, + "nauc_precision_at_1_diff1": 0.518322, + "nauc_precision_at_3_max": 0.358498, + "nauc_precision_at_3_std": 0.027106, + "nauc_precision_at_3_diff1": 0.347859, + "nauc_precision_at_5_max": 0.34631, + "nauc_precision_at_5_std": 0.066776, + "nauc_precision_at_5_diff1": 0.274746, + "nauc_precision_at_10_max": 0.338077, + "nauc_precision_at_10_std": 0.112782, + "nauc_precision_at_10_diff1": 0.220539, + "nauc_precision_at_20_max": 0.340661, + "nauc_precision_at_20_std": 0.161584, + "nauc_precision_at_20_diff1": 0.172042, + "nauc_precision_at_100_max": 0.287824, + "nauc_precision_at_100_std": 0.236474, + "nauc_precision_at_100_diff1": 0.044878, + "nauc_precision_at_1000_max": 0.236745, + "nauc_precision_at_1000_std": 0.224593, + "nauc_precision_at_1000_diff1": -0.026638, + "nauc_mrr_at_1_max": 0.34123, + "nauc_mrr_at_1_std": -0.017841, + "nauc_mrr_at_1_diff1": 0.518322, + "nauc_mrr_at_3_max": 0.36619, + "nauc_mrr_at_3_std": -0.003463, + "nauc_mrr_at_3_diff1": 0.487033, + "nauc_mrr_at_5_max": 0.369399, + "nauc_mrr_at_5_std": 0.001776, + "nauc_mrr_at_5_diff1": 0.481842, + "nauc_mrr_at_10_max": 0.370071, + "nauc_mrr_at_10_std": 0.003426, + "nauc_mrr_at_10_diff1": 0.483223, + "nauc_mrr_at_20_max": 0.371624, + "nauc_mrr_at_20_std": 0.005601, + "nauc_mrr_at_20_diff1": 0.483756, + "nauc_mrr_at_100_max": 0.371384, + "nauc_mrr_at_100_std": 0.006386, + "nauc_mrr_at_100_diff1": 0.483777, + "nauc_mrr_at_1000_max": 0.371237, + "nauc_mrr_at_1000_std": 0.00627, + "nauc_mrr_at_1000_diff1": 0.483789, + "main_score": 0.48593, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 45.477328062057495, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackGisRetrieval.json b/mteb/CQADupstackGisRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..5f1ce55b5d826343a6244d9f204ef35ace41f520 --- /dev/null +++ b/mteb/CQADupstackGisRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "5003b3064772da1887988e05400cf3806fe491f2", + "task_name": "CQADupstackGisRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.19322, + "ndcg_at_3": 0.2464, + "ndcg_at_5": 0.26398, + "ndcg_at_10": 0.28629, + "ndcg_at_20": 0.30424, + "ndcg_at_100": 0.33635, + "ndcg_at_1000": 0.36296, + "map_at_1": 0.1796, + "map_at_3": 0.22722, + "map_at_5": 0.23737, + "map_at_10": 0.24671, + "map_at_20": 0.25173, + "map_at_100": 0.25633, + "map_at_1000": 0.25725, + "recall_at_1": 0.1796, + "recall_at_3": 0.28451, + "recall_at_5": 0.32652, + "recall_at_10": 0.39283, + "recall_at_20": 0.46066, + "recall_at_100": 0.62619, + "recall_at_1000": 0.83354, + "precision_at_1": 0.19322, + "precision_at_3": 0.10395, + "precision_at_5": 0.07254, + "precision_at_10": 0.04475, + "precision_at_20": 0.02672, + "precision_at_100": 0.0074, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.19322, + "mrr_at_3": 0.243315, + "mrr_at_5": 0.25371, + "mrr_at_10": 0.263167, + "mrr_at_20": 0.268103, + "mrr_at_100": 0.27234, + "mrr_at_1000": 0.273059, + "nauc_ndcg_at_1_max": 0.3509, + "nauc_ndcg_at_1_std": -0.028343, + "nauc_ndcg_at_1_diff1": 0.4556, + "nauc_ndcg_at_3_max": 0.332288, + "nauc_ndcg_at_3_std": -0.001513, + "nauc_ndcg_at_3_diff1": 0.383786, + "nauc_ndcg_at_5_max": 0.323156, + "nauc_ndcg_at_5_std": -0.002329, + "nauc_ndcg_at_5_diff1": 0.375401, + "nauc_ndcg_at_10_max": 0.319542, + "nauc_ndcg_at_10_std": 0.001314, + "nauc_ndcg_at_10_diff1": 0.371061, + "nauc_ndcg_at_20_max": 0.322781, + "nauc_ndcg_at_20_std": 0.007944, + "nauc_ndcg_at_20_diff1": 0.359749, + "nauc_ndcg_at_100_max": 0.321408, + "nauc_ndcg_at_100_std": 0.021193, + "nauc_ndcg_at_100_diff1": 0.353214, + "nauc_ndcg_at_1000_max": 0.333213, + "nauc_ndcg_at_1000_std": 0.028037, + "nauc_ndcg_at_1000_diff1": 0.362077, + "nauc_map_at_1_max": 0.334646, + "nauc_map_at_1_std": -0.046069, + "nauc_map_at_1_diff1": 0.475264, + "nauc_map_at_3_max": 0.330952, + "nauc_map_at_3_std": -0.014788, + "nauc_map_at_3_diff1": 0.405655, + "nauc_map_at_5_max": 0.326424, + "nauc_map_at_5_std": -0.015209, + "nauc_map_at_5_diff1": 0.400335, + "nauc_map_at_10_max": 0.325464, + "nauc_map_at_10_std": -0.014892, + "nauc_map_at_10_diff1": 0.398555, + "nauc_map_at_20_max": 0.326796, + "nauc_map_at_20_std": -0.013008, + "nauc_map_at_20_diff1": 0.395344, + "nauc_map_at_100_max": 0.326354, + "nauc_map_at_100_std": -0.01094, + "nauc_map_at_100_diff1": 0.394208, + "nauc_map_at_1000_max": 0.326999, + "nauc_map_at_1000_std": -0.01055, + "nauc_map_at_1000_diff1": 0.394564, + "nauc_recall_at_1_max": 0.334646, + "nauc_recall_at_1_std": -0.046069, + "nauc_recall_at_1_diff1": 0.475264, + "nauc_recall_at_3_max": 0.312987, + "nauc_recall_at_3_std": 0.018809, + "nauc_recall_at_3_diff1": 0.329535, + "nauc_recall_at_5_max": 0.291171, + "nauc_recall_at_5_std": 0.015371, + "nauc_recall_at_5_diff1": 0.310569, + "nauc_recall_at_10_max": 0.278967, + "nauc_recall_at_10_std": 0.027129, + "nauc_recall_at_10_diff1": 0.297162, + "nauc_recall_at_20_max": 0.285044, + "nauc_recall_at_20_std": 0.048128, + "nauc_recall_at_20_diff1": 0.255835, + "nauc_recall_at_100_max": 0.265933, + "nauc_recall_at_100_std": 0.119517, + "nauc_recall_at_100_diff1": 0.202748, + "nauc_recall_at_1000_max": 0.373161, + "nauc_recall_at_1000_std": 0.285925, + "nauc_recall_at_1000_diff1": 0.204139, + "nauc_precision_at_1_max": 0.3509, + "nauc_precision_at_1_std": -0.028343, + "nauc_precision_at_1_diff1": 0.4556, + "nauc_precision_at_3_max": 0.349653, + "nauc_precision_at_3_std": 0.037263, + "nauc_precision_at_3_diff1": 0.316443, + "nauc_precision_at_5_max": 0.337659, + "nauc_precision_at_5_std": 0.038751, + "nauc_precision_at_5_diff1": 0.294004, + "nauc_precision_at_10_max": 0.327502, + "nauc_precision_at_10_std": 0.043505, + "nauc_precision_at_10_diff1": 0.285592, + "nauc_precision_at_20_max": 0.331996, + "nauc_precision_at_20_std": 0.076739, + "nauc_precision_at_20_diff1": 0.232499, + "nauc_precision_at_100_max": 0.299848, + "nauc_precision_at_100_std": 0.144632, + "nauc_precision_at_100_diff1": 0.154486, + "nauc_precision_at_1000_max": 0.28517, + "nauc_precision_at_1000_std": 0.204971, + "nauc_precision_at_1000_diff1": 0.090991, + "nauc_mrr_at_1_max": 0.3509, + "nauc_mrr_at_1_std": -0.028343, + "nauc_mrr_at_1_diff1": 0.4556, + "nauc_mrr_at_3_max": 0.343039, + "nauc_mrr_at_3_std": 0.003194, + "nauc_mrr_at_3_diff1": 0.395025, + "nauc_mrr_at_5_max": 0.338967, + "nauc_mrr_at_5_std": 0.002291, + "nauc_mrr_at_5_diff1": 0.389964, + "nauc_mrr_at_10_max": 0.337191, + "nauc_mrr_at_10_std": 0.004711, + "nauc_mrr_at_10_diff1": 0.388168, + "nauc_mrr_at_20_max": 0.338323, + "nauc_mrr_at_20_std": 0.006217, + "nauc_mrr_at_20_diff1": 0.385364, + "nauc_mrr_at_100_max": 0.337901, + "nauc_mrr_at_100_std": 0.007768, + "nauc_mrr_at_100_diff1": 0.384638, + "nauc_mrr_at_1000_max": 0.338161, + "nauc_mrr_at_1000_std": 0.00777, + "nauc_mrr_at_1000_diff1": 0.384956, + "main_score": 0.28629, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 75.08722925186157, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackMathematicaRetrieval.json b/mteb/CQADupstackMathematicaRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..1fe0a3691947e08b4146f4c8c93eedfa40b06b0f --- /dev/null +++ b/mteb/CQADupstackMathematicaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "90fceea13679c63fe563ded68f3b6f06e50061de", + "task_name": "CQADupstackMathematicaRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.1592, + "ndcg_at_3": 0.18727, + "ndcg_at_5": 0.20573, + "ndcg_at_10": 0.23092, + "ndcg_at_20": 0.24856, + "ndcg_at_100": 0.28661, + "ndcg_at_1000": 0.31839, + "map_at_1": 0.126, + "map_at_3": 0.1645, + "map_at_5": 0.17543, + "map_at_10": 0.18641, + "map_at_20": 0.19162, + "map_at_100": 0.19739, + "map_at_1000": 0.19864, + "recall_at_1": 0.126, + "recall_at_3": 0.20946, + "recall_at_5": 0.2564, + "recall_at_10": 0.32953, + "recall_at_20": 0.392, + "recall_at_100": 0.57855, + "recall_at_1000": 0.80748, + "precision_at_1": 0.1592, + "precision_at_3": 0.08955, + "precision_at_5": 0.06592, + "precision_at_10": 0.04366, + "precision_at_20": 0.02668, + "precision_at_100": 0.00832, + "precision_at_1000": 0.00124, + "mrr_at_1": 0.159204, + "mrr_at_3": 0.204187, + "mrr_at_5": 0.216563, + "mrr_at_10": 0.228361, + "mrr_at_20": 0.233146, + "mrr_at_100": 0.237804, + "mrr_at_1000": 0.238608, + "nauc_ndcg_at_1_max": 0.173331, + "nauc_ndcg_at_1_std": -0.070922, + "nauc_ndcg_at_1_diff1": 0.313574, + "nauc_ndcg_at_3_max": 0.149226, + "nauc_ndcg_at_3_std": -0.044527, + "nauc_ndcg_at_3_diff1": 0.231067, + "nauc_ndcg_at_5_max": 0.147868, + "nauc_ndcg_at_5_std": -0.033304, + "nauc_ndcg_at_5_diff1": 0.21667, + "nauc_ndcg_at_10_max": 0.136413, + "nauc_ndcg_at_10_std": -0.030496, + "nauc_ndcg_at_10_diff1": 0.19898, + "nauc_ndcg_at_20_max": 0.140685, + "nauc_ndcg_at_20_std": -0.020516, + "nauc_ndcg_at_20_diff1": 0.199934, + "nauc_ndcg_at_100_max": 0.159372, + "nauc_ndcg_at_100_std": 0.004912, + "nauc_ndcg_at_100_diff1": 0.205706, + "nauc_ndcg_at_1000_max": 0.158797, + "nauc_ndcg_at_1000_std": 0.000364, + "nauc_ndcg_at_1000_diff1": 0.203824, + "nauc_map_at_1_max": 0.176756, + "nauc_map_at_1_std": -0.061246, + "nauc_map_at_1_diff1": 0.294157, + "nauc_map_at_3_max": 0.151356, + "nauc_map_at_3_std": -0.048515, + "nauc_map_at_3_diff1": 0.241311, + "nauc_map_at_5_max": 0.151841, + "nauc_map_at_5_std": -0.042908, + "nauc_map_at_5_diff1": 0.23564, + "nauc_map_at_10_max": 0.146342, + "nauc_map_at_10_std": -0.040837, + "nauc_map_at_10_diff1": 0.226564, + "nauc_map_at_20_max": 0.148085, + "nauc_map_at_20_std": -0.036882, + "nauc_map_at_20_diff1": 0.226222, + "nauc_map_at_100_max": 0.151301, + "nauc_map_at_100_std": -0.032596, + "nauc_map_at_100_diff1": 0.226917, + "nauc_map_at_1000_max": 0.151171, + "nauc_map_at_1000_std": -0.032835, + "nauc_map_at_1000_diff1": 0.226596, + "nauc_recall_at_1_max": 0.176756, + "nauc_recall_at_1_std": -0.061246, + "nauc_recall_at_1_diff1": 0.294157, + "nauc_recall_at_3_max": 0.129382, + "nauc_recall_at_3_std": -0.026964, + "nauc_recall_at_3_diff1": 0.182064, + "nauc_recall_at_5_max": 0.126607, + "nauc_recall_at_5_std": -0.010177, + "nauc_recall_at_5_diff1": 0.159092, + "nauc_recall_at_10_max": 0.100506, + "nauc_recall_at_10_std": -0.007447, + "nauc_recall_at_10_diff1": 0.123114, + "nauc_recall_at_20_max": 0.109598, + "nauc_recall_at_20_std": 0.022768, + "nauc_recall_at_20_diff1": 0.126061, + "nauc_recall_at_100_max": 0.192576, + "nauc_recall_at_100_std": 0.141059, + "nauc_recall_at_100_diff1": 0.148286, + "nauc_recall_at_1000_max": 0.2255, + "nauc_recall_at_1000_std": 0.2101, + "nauc_recall_at_1000_diff1": 0.097776, + "nauc_precision_at_1_max": 0.173331, + "nauc_precision_at_1_std": -0.070922, + "nauc_precision_at_1_diff1": 0.313574, + "nauc_precision_at_3_max": 0.128165, + "nauc_precision_at_3_std": -0.037662, + "nauc_precision_at_3_diff1": 0.186901, + "nauc_precision_at_5_max": 0.135809, + "nauc_precision_at_5_std": -0.01395, + "nauc_precision_at_5_diff1": 0.16978, + "nauc_precision_at_10_max": 0.111158, + "nauc_precision_at_10_std": -0.011867, + "nauc_precision_at_10_diff1": 0.126989, + "nauc_precision_at_20_max": 0.111932, + "nauc_precision_at_20_std": 0.005621, + "nauc_precision_at_20_diff1": 0.115231, + "nauc_precision_at_100_max": 0.106532, + "nauc_precision_at_100_std": 0.058503, + "nauc_precision_at_100_diff1": 0.072954, + "nauc_precision_at_1000_max": 0.065429, + "nauc_precision_at_1000_std": -0.003839, + "nauc_precision_at_1000_diff1": 0.017772, + "nauc_mrr_at_1_max": 0.173331, + "nauc_mrr_at_1_std": -0.070922, + "nauc_mrr_at_1_diff1": 0.313574, + "nauc_mrr_at_3_max": 0.156386, + "nauc_mrr_at_3_std": -0.049645, + "nauc_mrr_at_3_diff1": 0.249522, + "nauc_mrr_at_5_max": 0.154441, + "nauc_mrr_at_5_std": -0.043804, + "nauc_mrr_at_5_diff1": 0.240661, + "nauc_mrr_at_10_max": 0.14987, + "nauc_mrr_at_10_std": -0.044188, + "nauc_mrr_at_10_diff1": 0.23307, + "nauc_mrr_at_20_max": 0.151533, + "nauc_mrr_at_20_std": -0.042259, + "nauc_mrr_at_20_diff1": 0.234266, + "nauc_mrr_at_100_max": 0.153064, + "nauc_mrr_at_100_std": -0.039756, + "nauc_mrr_at_100_diff1": 0.234561, + "nauc_mrr_at_1000_max": 0.152904, + "nauc_mrr_at_1000_std": -0.039874, + "nauc_mrr_at_1000_diff1": 0.234452, + "main_score": 0.23092, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 45.40352916717529, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackPhysicsRetrieval.json b/mteb/CQADupstackPhysicsRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..e5a5df6f6783c94fc72152a40fb39b44f9e67e13 --- /dev/null +++ b/mteb/CQADupstackPhysicsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4", + "task_name": "CQADupstackPhysicsRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.27238, + "ndcg_at_3": 0.32157, + "ndcg_at_5": 0.34044, + "ndcg_at_10": 0.37013, + "ndcg_at_20": 0.39337, + "ndcg_at_100": 0.42811, + "ndcg_at_1000": 0.45276, + "map_at_1": 0.22537, + "map_at_3": 0.2879, + "map_at_5": 0.3009, + "map_at_10": 0.31509, + "map_at_20": 0.32265, + "map_at_100": 0.32835, + "map_at_1000": 0.3296, + "recall_at_1": 0.22537, + "recall_at_3": 0.35122, + "recall_at_5": 0.39946, + "recall_at_10": 0.48803, + "recall_at_20": 0.56927, + "recall_at_100": 0.73288, + "recall_at_1000": 0.89725, + "precision_at_1": 0.27238, + "precision_at_3": 0.15271, + "precision_at_5": 0.1078, + "precision_at_10": 0.06756, + "precision_at_20": 0.04139, + "precision_at_100": 0.01153, + "precision_at_1000": 0.00154, + "mrr_at_1": 0.272377, + "mrr_at_3": 0.336702, + "mrr_at_5": 0.350369, + "mrr_at_10": 0.36334, + "mrr_at_20": 0.368893, + "mrr_at_100": 0.372666, + "mrr_at_1000": 0.373281, + "nauc_ndcg_at_1_max": 0.434963, + "nauc_ndcg_at_1_std": -0.016834, + "nauc_ndcg_at_1_diff1": 0.587191, + "nauc_ndcg_at_3_max": 0.376659, + "nauc_ndcg_at_3_std": -0.021128, + "nauc_ndcg_at_3_diff1": 0.498754, + "nauc_ndcg_at_5_max": 0.371104, + "nauc_ndcg_at_5_std": -0.006056, + "nauc_ndcg_at_5_diff1": 0.484702, + "nauc_ndcg_at_10_max": 0.371137, + "nauc_ndcg_at_10_std": 0.000755, + "nauc_ndcg_at_10_diff1": 0.472322, + "nauc_ndcg_at_20_max": 0.374454, + "nauc_ndcg_at_20_std": 0.009248, + "nauc_ndcg_at_20_diff1": 0.471307, + "nauc_ndcg_at_100_max": 0.385982, + "nauc_ndcg_at_100_std": 0.032502, + "nauc_ndcg_at_100_diff1": 0.473894, + "nauc_ndcg_at_1000_max": 0.39129, + "nauc_ndcg_at_1000_std": 0.036103, + "nauc_ndcg_at_1000_diff1": 0.475898, + "nauc_map_at_1_max": 0.387388, + "nauc_map_at_1_std": -0.063459, + "nauc_map_at_1_diff1": 0.595917, + "nauc_map_at_3_max": 0.368626, + "nauc_map_at_3_std": -0.041075, + "nauc_map_at_3_diff1": 0.520431, + "nauc_map_at_5_max": 0.369652, + "nauc_map_at_5_std": -0.028134, + "nauc_map_at_5_diff1": 0.512029, + "nauc_map_at_10_max": 0.373143, + "nauc_map_at_10_std": -0.021212, + "nauc_map_at_10_diff1": 0.505536, + "nauc_map_at_20_max": 0.375873, + "nauc_map_at_20_std": -0.017235, + "nauc_map_at_20_diff1": 0.50533, + "nauc_map_at_100_max": 0.378729, + "nauc_map_at_100_std": -0.013049, + "nauc_map_at_100_diff1": 0.505814, + "nauc_map_at_1000_max": 0.379222, + "nauc_map_at_1000_std": -0.012218, + "nauc_map_at_1000_diff1": 0.505796, + "nauc_recall_at_1_max": 0.387388, + "nauc_recall_at_1_std": -0.063459, + "nauc_recall_at_1_diff1": 0.595917, + "nauc_recall_at_3_max": 0.317313, + "nauc_recall_at_3_std": -0.038849, + "nauc_recall_at_3_diff1": 0.442818, + "nauc_recall_at_5_max": 0.31284, + "nauc_recall_at_5_std": 0.002032, + "nauc_recall_at_5_diff1": 0.407502, + "nauc_recall_at_10_max": 0.310181, + "nauc_recall_at_10_std": 0.029815, + "nauc_recall_at_10_diff1": 0.357645, + "nauc_recall_at_20_max": 0.305273, + "nauc_recall_at_20_std": 0.052793, + "nauc_recall_at_20_diff1": 0.339285, + "nauc_recall_at_100_max": 0.336613, + "nauc_recall_at_100_std": 0.198697, + "nauc_recall_at_100_diff1": 0.314998, + "nauc_recall_at_1000_max": 0.399223, + "nauc_recall_at_1000_std": 0.384869, + "nauc_recall_at_1000_diff1": 0.223792, + "nauc_precision_at_1_max": 0.434963, + "nauc_precision_at_1_std": -0.016834, + "nauc_precision_at_1_diff1": 0.587191, + "nauc_precision_at_3_max": 0.379844, + "nauc_precision_at_3_std": 0.058961, + "nauc_precision_at_3_diff1": 0.369786, + "nauc_precision_at_5_max": 0.367037, + "nauc_precision_at_5_std": 0.113331, + "nauc_precision_at_5_diff1": 0.304295, + "nauc_precision_at_10_max": 0.353315, + "nauc_precision_at_10_std": 0.159411, + "nauc_precision_at_10_diff1": 0.216986, + "nauc_precision_at_20_max": 0.321937, + "nauc_precision_at_20_std": 0.210608, + "nauc_precision_at_20_diff1": 0.15191, + "nauc_precision_at_100_max": 0.261556, + "nauc_precision_at_100_std": 0.287677, + "nauc_precision_at_100_diff1": 0.038748, + "nauc_precision_at_1000_max": 0.144137, + "nauc_precision_at_1000_std": 0.283117, + "nauc_precision_at_1000_diff1": -0.06849, + "nauc_mrr_at_1_max": 0.434963, + "nauc_mrr_at_1_std": -0.016834, + "nauc_mrr_at_1_diff1": 0.587191, + "nauc_mrr_at_3_max": 0.401226, + "nauc_mrr_at_3_std": -0.007172, + "nauc_mrr_at_3_diff1": 0.516348, + "nauc_mrr_at_5_max": 0.402557, + "nauc_mrr_at_5_std": 0.003495, + "nauc_mrr_at_5_diff1": 0.506254, + "nauc_mrr_at_10_max": 0.403164, + "nauc_mrr_at_10_std": 0.006847, + "nauc_mrr_at_10_diff1": 0.501766, + "nauc_mrr_at_20_max": 0.40341, + "nauc_mrr_at_20_std": 0.008421, + "nauc_mrr_at_20_diff1": 0.502319, + "nauc_mrr_at_100_max": 0.403681, + "nauc_mrr_at_100_std": 0.009655, + "nauc_mrr_at_100_diff1": 0.503195, + "nauc_mrr_at_1000_max": 0.403715, + "nauc_mrr_at_1000_std": 0.00956, + "nauc_mrr_at_1000_diff1": 0.503187, + "main_score": 0.37013, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 59.619898080825806, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackProgrammersRetrieval.json b/mteb/CQADupstackProgrammersRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..a9c2cae584f43419440cb7a23f6553c3b6d47b4a --- /dev/null +++ b/mteb/CQADupstackProgrammersRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6184bc1440d2dbc7612be22b50686b8826d22b32", + "task_name": "CQADupstackProgrammersRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.25228, + "ndcg_at_3": 0.29072, + "ndcg_at_5": 0.31289, + "ndcg_at_10": 0.33489, + "ndcg_at_20": 0.35566, + "ndcg_at_100": 0.39297, + "ndcg_at_1000": 0.41996, + "map_at_1": 0.20288, + "map_at_3": 0.25911, + "map_at_5": 0.27423, + "map_at_10": 0.2851, + "map_at_20": 0.2914, + "map_at_100": 0.29736, + "map_at_1000": 0.29862, + "recall_at_1": 0.20288, + "recall_at_3": 0.31413, + "recall_at_5": 0.37333, + "recall_at_10": 0.43861, + "recall_at_20": 0.51217, + "recall_at_100": 0.6923, + "recall_at_1000": 0.87747, + "precision_at_1": 0.25228, + "precision_at_3": 0.13889, + "precision_at_5": 0.10137, + "precision_at_10": 0.06084, + "precision_at_20": 0.03687, + "precision_at_100": 0.01065, + "precision_at_1000": 0.00146, + "mrr_at_1": 0.252283, + "mrr_at_3": 0.307078, + "mrr_at_5": 0.323002, + "mrr_at_10": 0.332576, + "mrr_at_20": 0.337973, + "mrr_at_100": 0.342415, + "mrr_at_1000": 0.343064, + "nauc_ndcg_at_1_max": 0.401716, + "nauc_ndcg_at_1_std": 0.065067, + "nauc_ndcg_at_1_diff1": 0.392721, + "nauc_ndcg_at_3_max": 0.383229, + "nauc_ndcg_at_3_std": 0.065445, + "nauc_ndcg_at_3_diff1": 0.331411, + "nauc_ndcg_at_5_max": 0.388223, + "nauc_ndcg_at_5_std": 0.079156, + "nauc_ndcg_at_5_diff1": 0.321325, + "nauc_ndcg_at_10_max": 0.382528, + "nauc_ndcg_at_10_std": 0.076964, + "nauc_ndcg_at_10_diff1": 0.319019, + "nauc_ndcg_at_20_max": 0.38324, + "nauc_ndcg_at_20_std": 0.088949, + "nauc_ndcg_at_20_diff1": 0.315701, + "nauc_ndcg_at_100_max": 0.394976, + "nauc_ndcg_at_100_std": 0.112611, + "nauc_ndcg_at_100_diff1": 0.318071, + "nauc_ndcg_at_1000_max": 0.400048, + "nauc_ndcg_at_1000_std": 0.116156, + "nauc_ndcg_at_1000_diff1": 0.320789, + "nauc_map_at_1_max": 0.357305, + "nauc_map_at_1_std": 0.015761, + "nauc_map_at_1_diff1": 0.384366, + "nauc_map_at_3_max": 0.373442, + "nauc_map_at_3_std": 0.047477, + "nauc_map_at_3_diff1": 0.342786, + "nauc_map_at_5_max": 0.37963, + "nauc_map_at_5_std": 0.058431, + "nauc_map_at_5_diff1": 0.336109, + "nauc_map_at_10_max": 0.379757, + "nauc_map_at_10_std": 0.059797, + "nauc_map_at_10_diff1": 0.336136, + "nauc_map_at_20_max": 0.381347, + "nauc_map_at_20_std": 0.064297, + "nauc_map_at_20_diff1": 0.335546, + "nauc_map_at_100_max": 0.383476, + "nauc_map_at_100_std": 0.069208, + "nauc_map_at_100_diff1": 0.335143, + "nauc_map_at_1000_max": 0.383756, + "nauc_map_at_1000_std": 0.069628, + "nauc_map_at_1000_diff1": 0.335194, + "nauc_recall_at_1_max": 0.357305, + "nauc_recall_at_1_std": 0.015761, + "nauc_recall_at_1_diff1": 0.384366, + "nauc_recall_at_3_max": 0.358924, + "nauc_recall_at_3_std": 0.056564, + "nauc_recall_at_3_diff1": 0.303315, + "nauc_recall_at_5_max": 0.367283, + "nauc_recall_at_5_std": 0.09274, + "nauc_recall_at_5_diff1": 0.273828, + "nauc_recall_at_10_max": 0.35053, + "nauc_recall_at_10_std": 0.089863, + "nauc_recall_at_10_diff1": 0.262794, + "nauc_recall_at_20_max": 0.339605, + "nauc_recall_at_20_std": 0.126648, + "nauc_recall_at_20_diff1": 0.242244, + "nauc_recall_at_100_max": 0.380486, + "nauc_recall_at_100_std": 0.249646, + "nauc_recall_at_100_diff1": 0.240527, + "nauc_recall_at_1000_max": 0.487529, + "nauc_recall_at_1000_std": 0.466888, + "nauc_recall_at_1000_diff1": 0.234965, + "nauc_precision_at_1_max": 0.401716, + "nauc_precision_at_1_std": 0.065067, + "nauc_precision_at_1_diff1": 0.392721, + "nauc_precision_at_3_max": 0.403259, + "nauc_precision_at_3_std": 0.142162, + "nauc_precision_at_3_diff1": 0.269316, + "nauc_precision_at_5_max": 0.404385, + "nauc_precision_at_5_std": 0.178013, + "nauc_precision_at_5_diff1": 0.224482, + "nauc_precision_at_10_max": 0.367242, + "nauc_precision_at_10_std": 0.17353, + "nauc_precision_at_10_diff1": 0.183739, + "nauc_precision_at_20_max": 0.327196, + "nauc_precision_at_20_std": 0.202914, + "nauc_precision_at_20_diff1": 0.143319, + "nauc_precision_at_100_max": 0.234403, + "nauc_precision_at_100_std": 0.246467, + "nauc_precision_at_100_diff1": 0.036827, + "nauc_precision_at_1000_max": 0.077956, + "nauc_precision_at_1000_std": 0.148892, + "nauc_precision_at_1000_diff1": -0.060821, + "nauc_mrr_at_1_max": 0.401716, + "nauc_mrr_at_1_std": 0.065067, + "nauc_mrr_at_1_diff1": 0.392721, + "nauc_mrr_at_3_max": 0.401914, + "nauc_mrr_at_3_std": 0.080097, + "nauc_mrr_at_3_diff1": 0.351381, + "nauc_mrr_at_5_max": 0.401255, + "nauc_mrr_at_5_std": 0.088183, + "nauc_mrr_at_5_diff1": 0.342425, + "nauc_mrr_at_10_max": 0.395882, + "nauc_mrr_at_10_std": 0.085347, + "nauc_mrr_at_10_diff1": 0.338442, + "nauc_mrr_at_20_max": 0.394937, + "nauc_mrr_at_20_std": 0.086579, + "nauc_mrr_at_20_diff1": 0.337513, + "nauc_mrr_at_100_max": 0.397282, + "nauc_mrr_at_100_std": 0.088992, + "nauc_mrr_at_100_diff1": 0.339029, + "nauc_mrr_at_1000_max": 0.397251, + "nauc_mrr_at_1000_std": 0.088846, + "nauc_mrr_at_1000_diff1": 0.339196, + "main_score": 0.33489, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 57.37495994567871, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackStatsRetrieval.json b/mteb/CQADupstackStatsRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..07b255ea0bcedccc6c44c4688d302f32670c6630 --- /dev/null +++ b/mteb/CQADupstackStatsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a", + "task_name": "CQADupstackStatsRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.21166, + "ndcg_at_3": 0.24697, + "ndcg_at_5": 0.26206, + "ndcg_at_10": 0.28144, + "ndcg_at_20": 0.29877, + "ndcg_at_100": 0.32798, + "ndcg_at_1000": 0.35119, + "map_at_1": 0.18672, + "map_at_3": 0.2269, + "map_at_5": 0.23653, + "map_at_10": 0.24469, + "map_at_20": 0.2497, + "map_at_100": 0.25401, + "map_at_1000": 0.25481, + "recall_at_1": 0.18672, + "recall_at_3": 0.27173, + "recall_at_5": 0.30953, + "recall_at_10": 0.3688, + "recall_at_20": 0.43444, + "recall_at_100": 0.58217, + "recall_at_1000": 0.75725, + "precision_at_1": 0.21166, + "precision_at_3": 0.10685, + "precision_at_5": 0.07485, + "precision_at_10": 0.04479, + "precision_at_20": 0.02669, + "precision_at_100": 0.00729, + "precision_at_1000": 0.001, + "mrr_at_1": 0.211656, + "mrr_at_3": 0.255112, + "mrr_at_5": 0.263471, + "mrr_at_10": 0.272333, + "mrr_at_20": 0.276509, + "mrr_at_100": 0.280587, + "mrr_at_1000": 0.28122, + "nauc_ndcg_at_1_max": 0.265288, + "nauc_ndcg_at_1_std": 0.001032, + "nauc_ndcg_at_1_diff1": 0.50655, + "nauc_ndcg_at_3_max": 0.259172, + "nauc_ndcg_at_3_std": 0.0391, + "nauc_ndcg_at_3_diff1": 0.464349, + "nauc_ndcg_at_5_max": 0.253671, + "nauc_ndcg_at_5_std": 0.057907, + "nauc_ndcg_at_5_diff1": 0.453087, + "nauc_ndcg_at_10_max": 0.252675, + "nauc_ndcg_at_10_std": 0.068572, + "nauc_ndcg_at_10_diff1": 0.431656, + "nauc_ndcg_at_20_max": 0.252661, + "nauc_ndcg_at_20_std": 0.085497, + "nauc_ndcg_at_20_diff1": 0.417731, + "nauc_ndcg_at_100_max": 0.257578, + "nauc_ndcg_at_100_std": 0.09627, + "nauc_ndcg_at_100_diff1": 0.409369, + "nauc_ndcg_at_1000_max": 0.26693, + "nauc_ndcg_at_1000_std": 0.100071, + "nauc_ndcg_at_1000_diff1": 0.41458, + "nauc_map_at_1_max": 0.280534, + "nauc_map_at_1_std": -0.03001, + "nauc_map_at_1_diff1": 0.538926, + "nauc_map_at_3_max": 0.264728, + "nauc_map_at_3_std": 0.014403, + "nauc_map_at_3_diff1": 0.48676, + "nauc_map_at_5_max": 0.262414, + "nauc_map_at_5_std": 0.030925, + "nauc_map_at_5_diff1": 0.479445, + "nauc_map_at_10_max": 0.262277, + "nauc_map_at_10_std": 0.037764, + "nauc_map_at_10_diff1": 0.470099, + "nauc_map_at_20_max": 0.262636, + "nauc_map_at_20_std": 0.04315, + "nauc_map_at_20_diff1": 0.465854, + "nauc_map_at_100_max": 0.263191, + "nauc_map_at_100_std": 0.045135, + "nauc_map_at_100_diff1": 0.463709, + "nauc_map_at_1000_max": 0.263864, + "nauc_map_at_1000_std": 0.045534, + "nauc_map_at_1000_diff1": 0.463922, + "nauc_recall_at_1_max": 0.280534, + "nauc_recall_at_1_std": -0.03001, + "nauc_recall_at_1_diff1": 0.538926, + "nauc_recall_at_3_max": 0.236962, + "nauc_recall_at_3_std": 0.054231, + "nauc_recall_at_3_diff1": 0.425405, + "nauc_recall_at_5_max": 0.224468, + "nauc_recall_at_5_std": 0.098421, + "nauc_recall_at_5_diff1": 0.39783, + "nauc_recall_at_10_max": 0.218317, + "nauc_recall_at_10_std": 0.128712, + "nauc_recall_at_10_diff1": 0.337508, + "nauc_recall_at_20_max": 0.213863, + "nauc_recall_at_20_std": 0.183436, + "nauc_recall_at_20_diff1": 0.285907, + "nauc_recall_at_100_max": 0.227284, + "nauc_recall_at_100_std": 0.243125, + "nauc_recall_at_100_diff1": 0.233571, + "nauc_recall_at_1000_max": 0.277326, + "nauc_recall_at_1000_std": 0.328397, + "nauc_recall_at_1000_diff1": 0.20892, + "nauc_precision_at_1_max": 0.265288, + "nauc_precision_at_1_std": 0.001032, + "nauc_precision_at_1_diff1": 0.50655, + "nauc_precision_at_3_max": 0.253493, + "nauc_precision_at_3_std": 0.115181, + "nauc_precision_at_3_diff1": 0.392497, + "nauc_precision_at_5_max": 0.234647, + "nauc_precision_at_5_std": 0.187151, + "nauc_precision_at_5_diff1": 0.338881, + "nauc_precision_at_10_max": 0.237545, + "nauc_precision_at_10_std": 0.213893, + "nauc_precision_at_10_diff1": 0.285351, + "nauc_precision_at_20_max": 0.230892, + "nauc_precision_at_20_std": 0.262866, + "nauc_precision_at_20_diff1": 0.210742, + "nauc_precision_at_100_max": 0.232832, + "nauc_precision_at_100_std": 0.294046, + "nauc_precision_at_100_diff1": 0.140878, + "nauc_precision_at_1000_max": 0.239556, + "nauc_precision_at_1000_std": 0.263343, + "nauc_precision_at_1000_diff1": 0.042393, + "nauc_mrr_at_1_max": 0.265288, + "nauc_mrr_at_1_std": 0.001032, + "nauc_mrr_at_1_diff1": 0.50655, + "nauc_mrr_at_3_max": 0.257994, + "nauc_mrr_at_3_std": 0.04097, + "nauc_mrr_at_3_diff1": 0.456167, + "nauc_mrr_at_5_max": 0.255945, + "nauc_mrr_at_5_std": 0.049945, + "nauc_mrr_at_5_diff1": 0.450037, + "nauc_mrr_at_10_max": 0.256333, + "nauc_mrr_at_10_std": 0.054663, + "nauc_mrr_at_10_diff1": 0.440708, + "nauc_mrr_at_20_max": 0.255786, + "nauc_mrr_at_20_std": 0.058257, + "nauc_mrr_at_20_diff1": 0.437883, + "nauc_mrr_at_100_max": 0.256685, + "nauc_mrr_at_100_std": 0.05889, + "nauc_mrr_at_100_diff1": 0.436408, + "nauc_mrr_at_1000_max": 0.256891, + "nauc_mrr_at_1000_std": 0.059147, + "nauc_mrr_at_1000_diff1": 0.436692, + "main_score": 0.28144, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 86.29045724868774, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackTexRetrieval.json b/mteb/CQADupstackTexRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..59bf496e2229318a687373e5818f9d9c1e58b31d --- /dev/null +++ b/mteb/CQADupstackTexRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "46989137a86843e03a6195de44b09deda022eec7", + "task_name": "CQADupstackTexRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.14178, + "ndcg_at_3": 0.17057, + "ndcg_at_5": 0.1856, + "ndcg_at_10": 0.20471, + "ndcg_at_20": 0.22044, + "ndcg_at_100": 0.2503, + "ndcg_at_1000": 0.28231, + "map_at_1": 0.11553, + "map_at_3": 0.15025, + "map_at_5": 0.15972, + "map_at_10": 0.16841, + "map_at_20": 0.17291, + "map_at_100": 0.17711, + "map_at_1000": 0.17832, + "recall_at_1": 0.11553, + "recall_at_3": 0.18958, + "recall_at_5": 0.22824, + "recall_at_10": 0.28569, + "recall_at_20": 0.34504, + "recall_at_100": 0.49491, + "recall_at_1000": 0.72826, + "precision_at_1": 0.14178, + "precision_at_3": 0.08098, + "precision_at_5": 0.05953, + "precision_at_10": 0.03823, + "precision_at_20": 0.02326, + "precision_at_100": 0.0071, + "precision_at_1000": 0.00114, + "mrr_at_1": 0.141776, + "mrr_at_3": 0.179456, + "mrr_at_5": 0.189694, + "mrr_at_10": 0.198676, + "mrr_at_20": 0.203388, + "mrr_at_100": 0.207227, + "mrr_at_1000": 0.208095, + "nauc_ndcg_at_1_max": 0.273731, + "nauc_ndcg_at_1_std": -0.000848, + "nauc_ndcg_at_1_diff1": 0.402551, + "nauc_ndcg_at_3_max": 0.254625, + "nauc_ndcg_at_3_std": 0.005476, + "nauc_ndcg_at_3_diff1": 0.343983, + "nauc_ndcg_at_5_max": 0.251083, + "nauc_ndcg_at_5_std": 0.011607, + "nauc_ndcg_at_5_diff1": 0.328852, + "nauc_ndcg_at_10_max": 0.24845, + "nauc_ndcg_at_10_std": 0.024927, + "nauc_ndcg_at_10_diff1": 0.316519, + "nauc_ndcg_at_20_max": 0.251553, + "nauc_ndcg_at_20_std": 0.038963, + "nauc_ndcg_at_20_diff1": 0.306872, + "nauc_ndcg_at_100_max": 0.25391, + "nauc_ndcg_at_100_std": 0.049506, + "nauc_ndcg_at_100_diff1": 0.300219, + "nauc_ndcg_at_1000_max": 0.25447, + "nauc_ndcg_at_1000_std": 0.057198, + "nauc_ndcg_at_1000_diff1": 0.297107, + "nauc_map_at_1_max": 0.24317, + "nauc_map_at_1_std": -0.013325, + "nauc_map_at_1_diff1": 0.430924, + "nauc_map_at_3_max": 0.245786, + "nauc_map_at_3_std": -0.001987, + "nauc_map_at_3_diff1": 0.367577, + "nauc_map_at_5_max": 0.245709, + "nauc_map_at_5_std": 0.001682, + "nauc_map_at_5_diff1": 0.356493, + "nauc_map_at_10_max": 0.24637, + "nauc_map_at_10_std": 0.009096, + "nauc_map_at_10_diff1": 0.348743, + "nauc_map_at_20_max": 0.247864, + "nauc_map_at_20_std": 0.013651, + "nauc_map_at_20_diff1": 0.345685, + "nauc_map_at_100_max": 0.248458, + "nauc_map_at_100_std": 0.015959, + "nauc_map_at_100_diff1": 0.344828, + "nauc_map_at_1000_max": 0.248587, + "nauc_map_at_1000_std": 0.016378, + "nauc_map_at_1000_diff1": 0.344495, + "nauc_recall_at_1_max": 0.24317, + "nauc_recall_at_1_std": -0.013325, + "nauc_recall_at_1_diff1": 0.430924, + "nauc_recall_at_3_max": 0.239063, + "nauc_recall_at_3_std": 0.009005, + "nauc_recall_at_3_diff1": 0.313126, + "nauc_recall_at_5_max": 0.228969, + "nauc_recall_at_5_std": 0.018906, + "nauc_recall_at_5_diff1": 0.274421, + "nauc_recall_at_10_max": 0.224291, + "nauc_recall_at_10_std": 0.047312, + "nauc_recall_at_10_diff1": 0.244958, + "nauc_recall_at_20_max": 0.229385, + "nauc_recall_at_20_std": 0.086611, + "nauc_recall_at_20_diff1": 0.212233, + "nauc_recall_at_100_max": 0.237652, + "nauc_recall_at_100_std": 0.122032, + "nauc_recall_at_100_diff1": 0.180425, + "nauc_recall_at_1000_max": 0.232788, + "nauc_recall_at_1000_std": 0.208114, + "nauc_recall_at_1000_diff1": 0.120909, + "nauc_precision_at_1_max": 0.273731, + "nauc_precision_at_1_std": -0.000848, + "nauc_precision_at_1_diff1": 0.402551, + "nauc_precision_at_3_max": 0.275815, + "nauc_precision_at_3_std": 0.023241, + "nauc_precision_at_3_diff1": 0.281068, + "nauc_precision_at_5_max": 0.271773, + "nauc_precision_at_5_std": 0.037777, + "nauc_precision_at_5_diff1": 0.24271, + "nauc_precision_at_10_max": 0.264364, + "nauc_precision_at_10_std": 0.073815, + "nauc_precision_at_10_diff1": 0.210453, + "nauc_precision_at_20_max": 0.270537, + "nauc_precision_at_20_std": 0.109882, + "nauc_precision_at_20_diff1": 0.180007, + "nauc_precision_at_100_max": 0.247712, + "nauc_precision_at_100_std": 0.141652, + "nauc_precision_at_100_diff1": 0.110556, + "nauc_precision_at_1000_max": 0.195539, + "nauc_precision_at_1000_std": 0.152915, + "nauc_precision_at_1000_diff1": 0.008828, + "nauc_mrr_at_1_max": 0.273731, + "nauc_mrr_at_1_std": -0.000848, + "nauc_mrr_at_1_diff1": 0.402551, + "nauc_mrr_at_3_max": 0.264306, + "nauc_mrr_at_3_std": 0.01339, + "nauc_mrr_at_3_diff1": 0.344013, + "nauc_mrr_at_5_max": 0.262192, + "nauc_mrr_at_5_std": 0.017274, + "nauc_mrr_at_5_diff1": 0.333294, + "nauc_mrr_at_10_max": 0.261096, + "nauc_mrr_at_10_std": 0.022398, + "nauc_mrr_at_10_diff1": 0.32811, + "nauc_mrr_at_20_max": 0.262244, + "nauc_mrr_at_20_std": 0.026409, + "nauc_mrr_at_20_diff1": 0.32533, + "nauc_mrr_at_100_max": 0.262496, + "nauc_mrr_at_100_std": 0.027576, + "nauc_mrr_at_100_diff1": 0.324713, + "nauc_mrr_at_1000_max": 0.262422, + "nauc_mrr_at_1000_std": 0.027625, + "nauc_mrr_at_1000_diff1": 0.324694, + "main_score": 0.20471, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 198.2237286567688, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackUnixRetrieval.json b/mteb/CQADupstackUnixRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..b0f1a7343dccc53f3c5aee94185307d891a39fe1 --- /dev/null +++ b/mteb/CQADupstackUnixRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53", + "task_name": "CQADupstackUnixRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.24347, + "ndcg_at_3": 0.2739, + "ndcg_at_5": 0.29021, + "ndcg_at_10": 0.3119, + "ndcg_at_20": 0.33183, + "ndcg_at_100": 0.36465, + "ndcg_at_1000": 0.39555, + "map_at_1": 0.20973, + "map_at_3": 0.25159, + "map_at_5": 0.2616, + "map_at_10": 0.27125, + "map_at_20": 0.27681, + "map_at_100": 0.28175, + "map_at_1000": 0.28303, + "recall_at_1": 0.20973, + "recall_at_3": 0.29951, + "recall_at_5": 0.34009, + "recall_at_10": 0.40363, + "recall_at_20": 0.47728, + "recall_at_100": 0.63718, + "recall_at_1000": 0.8574, + "precision_at_1": 0.24347, + "precision_at_3": 0.11971, + "precision_at_5": 0.08321, + "precision_at_10": 0.05028, + "precision_at_20": 0.03032, + "precision_at_100": 0.00867, + "precision_at_1000": 0.00126, + "mrr_at_1": 0.24347, + "mrr_at_3": 0.285914, + "mrr_at_5": 0.297015, + "mrr_at_10": 0.306006, + "mrr_at_20": 0.311716, + "mrr_at_100": 0.315759, + "mrr_at_1000": 0.316592, + "nauc_ndcg_at_1_max": 0.366074, + "nauc_ndcg_at_1_std": -0.002938, + "nauc_ndcg_at_1_diff1": 0.511682, + "nauc_ndcg_at_3_max": 0.355856, + "nauc_ndcg_at_3_std": 0.033005, + "nauc_ndcg_at_3_diff1": 0.455243, + "nauc_ndcg_at_5_max": 0.342306, + "nauc_ndcg_at_5_std": 0.024515, + "nauc_ndcg_at_5_diff1": 0.434606, + "nauc_ndcg_at_10_max": 0.340939, + "nauc_ndcg_at_10_std": 0.036514, + "nauc_ndcg_at_10_diff1": 0.421622, + "nauc_ndcg_at_20_max": 0.339306, + "nauc_ndcg_at_20_std": 0.041006, + "nauc_ndcg_at_20_diff1": 0.418551, + "nauc_ndcg_at_100_max": 0.339351, + "nauc_ndcg_at_100_std": 0.058532, + "nauc_ndcg_at_100_diff1": 0.413425, + "nauc_ndcg_at_1000_max": 0.345893, + "nauc_ndcg_at_1000_std": 0.063455, + "nauc_ndcg_at_1000_diff1": 0.41893, + "nauc_map_at_1_max": 0.364348, + "nauc_map_at_1_std": -0.01631, + "nauc_map_at_1_diff1": 0.529856, + "nauc_map_at_3_max": 0.356602, + "nauc_map_at_3_std": 0.017935, + "nauc_map_at_3_diff1": 0.473642, + "nauc_map_at_5_max": 0.349516, + "nauc_map_at_5_std": 0.015001, + "nauc_map_at_5_diff1": 0.461318, + "nauc_map_at_10_max": 0.350392, + "nauc_map_at_10_std": 0.022047, + "nauc_map_at_10_diff1": 0.455432, + "nauc_map_at_20_max": 0.3504, + "nauc_map_at_20_std": 0.023646, + "nauc_map_at_20_diff1": 0.454156, + "nauc_map_at_100_max": 0.349922, + "nauc_map_at_100_std": 0.026192, + "nauc_map_at_100_diff1": 0.4536, + "nauc_map_at_1000_max": 0.350113, + "nauc_map_at_1000_std": 0.026625, + "nauc_map_at_1000_diff1": 0.453732, + "nauc_recall_at_1_max": 0.364348, + "nauc_recall_at_1_std": -0.01631, + "nauc_recall_at_1_diff1": 0.529856, + "nauc_recall_at_3_max": 0.343817, + "nauc_recall_at_3_std": 0.054702, + "nauc_recall_at_3_diff1": 0.414897, + "nauc_recall_at_5_max": 0.312585, + "nauc_recall_at_5_std": 0.040548, + "nauc_recall_at_5_diff1": 0.367072, + "nauc_recall_at_10_max": 0.302046, + "nauc_recall_at_10_std": 0.067438, + "nauc_recall_at_10_diff1": 0.328025, + "nauc_recall_at_20_max": 0.289, + "nauc_recall_at_20_std": 0.079, + "nauc_recall_at_20_diff1": 0.309696, + "nauc_recall_at_100_max": 0.279702, + "nauc_recall_at_100_std": 0.178914, + "nauc_recall_at_100_diff1": 0.259407, + "nauc_recall_at_1000_max": 0.358183, + "nauc_recall_at_1000_std": 0.36287, + "nauc_recall_at_1000_diff1": 0.232753, + "nauc_precision_at_1_max": 0.366074, + "nauc_precision_at_1_std": -0.002938, + "nauc_precision_at_1_diff1": 0.511682, + "nauc_precision_at_3_max": 0.342215, + "nauc_precision_at_3_std": 0.079457, + "nauc_precision_at_3_diff1": 0.379603, + "nauc_precision_at_5_max": 0.308996, + "nauc_precision_at_5_std": 0.065465, + "nauc_precision_at_5_diff1": 0.327939, + "nauc_precision_at_10_max": 0.293998, + "nauc_precision_at_10_std": 0.10973, + "nauc_precision_at_10_diff1": 0.268081, + "nauc_precision_at_20_max": 0.265752, + "nauc_precision_at_20_std": 0.123159, + "nauc_precision_at_20_diff1": 0.234389, + "nauc_precision_at_100_max": 0.178161, + "nauc_precision_at_100_std": 0.176852, + "nauc_precision_at_100_diff1": 0.089218, + "nauc_precision_at_1000_max": 0.055458, + "nauc_precision_at_1000_std": 0.142567, + "nauc_precision_at_1000_diff1": -0.047612, + "nauc_mrr_at_1_max": 0.366074, + "nauc_mrr_at_1_std": -0.002938, + "nauc_mrr_at_1_diff1": 0.511682, + "nauc_mrr_at_3_max": 0.354753, + "nauc_mrr_at_3_std": 0.024454, + "nauc_mrr_at_3_diff1": 0.456803, + "nauc_mrr_at_5_max": 0.349086, + "nauc_mrr_at_5_std": 0.021245, + "nauc_mrr_at_5_diff1": 0.444838, + "nauc_mrr_at_10_max": 0.346014, + "nauc_mrr_at_10_std": 0.024307, + "nauc_mrr_at_10_diff1": 0.440129, + "nauc_mrr_at_20_max": 0.345043, + "nauc_mrr_at_20_std": 0.026122, + "nauc_mrr_at_20_diff1": 0.439379, + "nauc_mrr_at_100_max": 0.345434, + "nauc_mrr_at_100_std": 0.028055, + "nauc_mrr_at_100_diff1": 0.439384, + "nauc_mrr_at_1000_max": 0.34558, + "nauc_mrr_at_1000_std": 0.028083, + "nauc_mrr_at_1000_diff1": 0.439607, + "main_score": 0.3119, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 103.79455494880676, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackWebmastersRetrieval.json b/mteb/CQADupstackWebmastersRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..4cf29347b313cc0a23ccaf8cd16d186dc448b999 --- /dev/null +++ b/mteb/CQADupstackWebmastersRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "160c094312a0e1facb97e55eeddb698c0abe3571", + "task_name": "CQADupstackWebmastersRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.22925, + "ndcg_at_3": 0.2741, + "ndcg_at_5": 0.2929, + "ndcg_at_10": 0.31232, + "ndcg_at_20": 0.33154, + "ndcg_at_100": 0.36606, + "ndcg_at_1000": 0.39806, + "map_at_1": 0.19137, + "map_at_3": 0.23966, + "map_at_5": 0.25369, + "map_at_10": 0.26413, + "map_at_20": 0.27109, + "map_at_100": 0.27726, + "map_at_1000": 0.27925, + "recall_at_1": 0.19137, + "recall_at_3": 0.29464, + "recall_at_5": 0.34544, + "recall_at_10": 0.40694, + "recall_at_20": 0.4791, + "recall_at_100": 0.65313, + "recall_at_1000": 0.86383, + "precision_at_1": 0.22925, + "precision_at_3": 0.13109, + "precision_at_5": 0.09684, + "precision_at_10": 0.06047, + "precision_at_20": 0.03903, + "precision_at_100": 0.01235, + "precision_at_1000": 0.00207, + "mrr_at_1": 0.229249, + "mrr_at_3": 0.278656, + "mrr_at_5": 0.291601, + "mrr_at_10": 0.300793, + "mrr_at_20": 0.30649, + "mrr_at_100": 0.310727, + "mrr_at_1000": 0.311487, + "nauc_ndcg_at_1_max": 0.278397, + "nauc_ndcg_at_1_std": 0.074006, + "nauc_ndcg_at_1_diff1": 0.513375, + "nauc_ndcg_at_3_max": 0.277863, + "nauc_ndcg_at_3_std": 0.105389, + "nauc_ndcg_at_3_diff1": 0.48273, + "nauc_ndcg_at_5_max": 0.262458, + "nauc_ndcg_at_5_std": 0.109897, + "nauc_ndcg_at_5_diff1": 0.465795, + "nauc_ndcg_at_10_max": 0.259559, + "nauc_ndcg_at_10_std": 0.112909, + "nauc_ndcg_at_10_diff1": 0.458508, + "nauc_ndcg_at_20_max": 0.259624, + "nauc_ndcg_at_20_std": 0.115503, + "nauc_ndcg_at_20_diff1": 0.448082, + "nauc_ndcg_at_100_max": 0.257458, + "nauc_ndcg_at_100_std": 0.133477, + "nauc_ndcg_at_100_diff1": 0.437838, + "nauc_ndcg_at_1000_max": 0.264933, + "nauc_ndcg_at_1000_std": 0.139622, + "nauc_ndcg_at_1000_diff1": 0.448966, + "nauc_map_at_1_max": 0.264657, + "nauc_map_at_1_std": 0.035551, + "nauc_map_at_1_diff1": 0.536002, + "nauc_map_at_3_max": 0.271682, + "nauc_map_at_3_std": 0.077661, + "nauc_map_at_3_diff1": 0.495551, + "nauc_map_at_5_max": 0.26631, + "nauc_map_at_5_std": 0.080968, + "nauc_map_at_5_diff1": 0.481763, + "nauc_map_at_10_max": 0.263891, + "nauc_map_at_10_std": 0.084954, + "nauc_map_at_10_diff1": 0.477934, + "nauc_map_at_20_max": 0.263787, + "nauc_map_at_20_std": 0.088554, + "nauc_map_at_20_diff1": 0.475802, + "nauc_map_at_100_max": 0.263258, + "nauc_map_at_100_std": 0.095955, + "nauc_map_at_100_diff1": 0.475416, + "nauc_map_at_1000_max": 0.263256, + "nauc_map_at_1000_std": 0.09769, + "nauc_map_at_1000_diff1": 0.475749, + "nauc_recall_at_1_max": 0.264657, + "nauc_recall_at_1_std": 0.035551, + "nauc_recall_at_1_diff1": 0.536002, + "nauc_recall_at_3_max": 0.263286, + "nauc_recall_at_3_std": 0.110583, + "nauc_recall_at_3_diff1": 0.454661, + "nauc_recall_at_5_max": 0.241204, + "nauc_recall_at_5_std": 0.125151, + "nauc_recall_at_5_diff1": 0.411045, + "nauc_recall_at_10_max": 0.240086, + "nauc_recall_at_10_std": 0.137833, + "nauc_recall_at_10_diff1": 0.388961, + "nauc_recall_at_20_max": 0.231511, + "nauc_recall_at_20_std": 0.151779, + "nauc_recall_at_20_diff1": 0.35641, + "nauc_recall_at_100_max": 0.205584, + "nauc_recall_at_100_std": 0.242135, + "nauc_recall_at_100_diff1": 0.267027, + "nauc_recall_at_1000_max": 0.259238, + "nauc_recall_at_1000_std": 0.406363, + "nauc_recall_at_1000_diff1": 0.28158, + "nauc_precision_at_1_max": 0.278397, + "nauc_precision_at_1_std": 0.074006, + "nauc_precision_at_1_diff1": 0.513375, + "nauc_precision_at_3_max": 0.254205, + "nauc_precision_at_3_std": 0.150633, + "nauc_precision_at_3_diff1": 0.384094, + "nauc_precision_at_5_max": 0.20648, + "nauc_precision_at_5_std": 0.158803, + "nauc_precision_at_5_diff1": 0.310389, + "nauc_precision_at_10_max": 0.175023, + "nauc_precision_at_10_std": 0.188265, + "nauc_precision_at_10_diff1": 0.268628, + "nauc_precision_at_20_max": 0.160018, + "nauc_precision_at_20_std": 0.229377, + "nauc_precision_at_20_diff1": 0.216145, + "nauc_precision_at_100_max": 0.039446, + "nauc_precision_at_100_std": 0.309237, + "nauc_precision_at_100_diff1": 0.081114, + "nauc_precision_at_1000_max": -0.078246, + "nauc_precision_at_1000_std": 0.226591, + "nauc_precision_at_1000_diff1": -0.029668, + "nauc_mrr_at_1_max": 0.278397, + "nauc_mrr_at_1_std": 0.074006, + "nauc_mrr_at_1_diff1": 0.513375, + "nauc_mrr_at_3_max": 0.271664, + "nauc_mrr_at_3_std": 0.105992, + "nauc_mrr_at_3_diff1": 0.475073, + "nauc_mrr_at_5_max": 0.264996, + "nauc_mrr_at_5_std": 0.109831, + "nauc_mrr_at_5_diff1": 0.463956, + "nauc_mrr_at_10_max": 0.265579, + "nauc_mrr_at_10_std": 0.111244, + "nauc_mrr_at_10_diff1": 0.464022, + "nauc_mrr_at_20_max": 0.26659, + "nauc_mrr_at_20_std": 0.111591, + "nauc_mrr_at_20_diff1": 0.462309, + "nauc_mrr_at_100_max": 0.265909, + "nauc_mrr_at_100_std": 0.113546, + "nauc_mrr_at_100_diff1": 0.461344, + "nauc_mrr_at_1000_max": 0.266114, + "nauc_mrr_at_1000_std": 0.113636, + "nauc_mrr_at_1000_diff1": 0.461867, + "main_score": 0.31232, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 23.93776535987854, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/CQADupstackWordpressRetrieval.json b/mteb/CQADupstackWordpressRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..88b8df8cbdec73f9c048776b357497213d2c8031 --- /dev/null +++ b/mteb/CQADupstackWordpressRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4", + "task_name": "CQADupstackWordpressRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.18299, + "ndcg_at_3": 0.22947, + "ndcg_at_5": 0.24289, + "ndcg_at_10": 0.26195, + "ndcg_at_20": 0.28138, + "ndcg_at_100": 0.31194, + "ndcg_at_1000": 0.34148, + "map_at_1": 0.16833, + "map_at_3": 0.211, + "map_at_5": 0.2197, + "map_at_10": 0.22788, + "map_at_20": 0.23321, + "map_at_100": 0.2373, + "map_at_1000": 0.23838, + "recall_at_1": 0.16833, + "recall_at_3": 0.26334, + "recall_at_5": 0.29575, + "recall_at_10": 0.3512, + "recall_at_20": 0.42532, + "recall_at_100": 0.58598, + "recall_at_1000": 0.80809, + "precision_at_1": 0.18299, + "precision_at_3": 0.0992, + "precision_at_5": 0.06765, + "precision_at_10": 0.04067, + "precision_at_20": 0.02486, + "precision_at_100": 0.00701, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.182994, + "mrr_at_3": 0.227665, + "mrr_at_5": 0.235059, + "mrr_at_10": 0.243175, + "mrr_at_20": 0.248732, + "mrr_at_100": 0.252759, + "mrr_at_1000": 0.253625, + "nauc_ndcg_at_1_max": 0.290492, + "nauc_ndcg_at_1_std": -0.037429, + "nauc_ndcg_at_1_diff1": 0.427622, + "nauc_ndcg_at_3_max": 0.243326, + "nauc_ndcg_at_3_std": -0.02747, + "nauc_ndcg_at_3_diff1": 0.340168, + "nauc_ndcg_at_5_max": 0.238634, + "nauc_ndcg_at_5_std": -0.02659, + "nauc_ndcg_at_5_diff1": 0.330531, + "nauc_ndcg_at_10_max": 0.247575, + "nauc_ndcg_at_10_std": -0.017911, + "nauc_ndcg_at_10_diff1": 0.320757, + "nauc_ndcg_at_20_max": 0.245705, + "nauc_ndcg_at_20_std": 0.004907, + "nauc_ndcg_at_20_diff1": 0.320415, + "nauc_ndcg_at_100_max": 0.240886, + "nauc_ndcg_at_100_std": 0.023006, + "nauc_ndcg_at_100_diff1": 0.31633, + "nauc_ndcg_at_1000_max": 0.242627, + "nauc_ndcg_at_1000_std": 0.029307, + "nauc_ndcg_at_1000_diff1": 0.318627, + "nauc_map_at_1_max": 0.265847, + "nauc_map_at_1_std": -0.039069, + "nauc_map_at_1_diff1": 0.430649, + "nauc_map_at_3_max": 0.244435, + "nauc_map_at_3_std": -0.027471, + "nauc_map_at_3_diff1": 0.358874, + "nauc_map_at_5_max": 0.244507, + "nauc_map_at_5_std": -0.02695, + "nauc_map_at_5_diff1": 0.352913, + "nauc_map_at_10_max": 0.248881, + "nauc_map_at_10_std": -0.023212, + "nauc_map_at_10_diff1": 0.348928, + "nauc_map_at_20_max": 0.248784, + "nauc_map_at_20_std": -0.016656, + "nauc_map_at_20_diff1": 0.348707, + "nauc_map_at_100_max": 0.248009, + "nauc_map_at_100_std": -0.014712, + "nauc_map_at_100_diff1": 0.348352, + "nauc_map_at_1000_max": 0.248034, + "nauc_map_at_1000_std": -0.014321, + "nauc_map_at_1000_diff1": 0.348601, + "nauc_recall_at_1_max": 0.265847, + "nauc_recall_at_1_std": -0.039069, + "nauc_recall_at_1_diff1": 0.430649, + "nauc_recall_at_3_max": 0.213529, + "nauc_recall_at_3_std": -0.025467, + "nauc_recall_at_3_diff1": 0.286084, + "nauc_recall_at_5_max": 0.205815, + "nauc_recall_at_5_std": -0.025665, + "nauc_recall_at_5_diff1": 0.265638, + "nauc_recall_at_10_max": 0.223811, + "nauc_recall_at_10_std": -0.002912, + "nauc_recall_at_10_diff1": 0.236385, + "nauc_recall_at_20_max": 0.211582, + "nauc_recall_at_20_std": 0.067677, + "nauc_recall_at_20_diff1": 0.234745, + "nauc_recall_at_100_max": 0.182712, + "nauc_recall_at_100_std": 0.17472, + "nauc_recall_at_100_diff1": 0.201213, + "nauc_recall_at_1000_max": 0.174778, + "nauc_recall_at_1000_std": 0.340317, + "nauc_recall_at_1000_diff1": 0.125604, + "nauc_precision_at_1_max": 0.290492, + "nauc_precision_at_1_std": -0.037429, + "nauc_precision_at_1_diff1": 0.427622, + "nauc_precision_at_3_max": 0.227398, + "nauc_precision_at_3_std": -0.028379, + "nauc_precision_at_3_diff1": 0.266137, + "nauc_precision_at_5_max": 0.230675, + "nauc_precision_at_5_std": -0.017098, + "nauc_precision_at_5_diff1": 0.241378, + "nauc_precision_at_10_max": 0.243564, + "nauc_precision_at_10_std": 0.005694, + "nauc_precision_at_10_diff1": 0.209227, + "nauc_precision_at_20_max": 0.239849, + "nauc_precision_at_20_std": 0.089154, + "nauc_precision_at_20_diff1": 0.210395, + "nauc_precision_at_100_max": 0.187676, + "nauc_precision_at_100_std": 0.159012, + "nauc_precision_at_100_diff1": 0.145642, + "nauc_precision_at_1000_max": 0.040961, + "nauc_precision_at_1000_std": 0.149076, + "nauc_precision_at_1000_diff1": 0.029816, + "nauc_mrr_at_1_max": 0.290492, + "nauc_mrr_at_1_std": -0.037429, + "nauc_mrr_at_1_diff1": 0.427622, + "nauc_mrr_at_3_max": 0.261245, + "nauc_mrr_at_3_std": -0.029722, + "nauc_mrr_at_3_diff1": 0.363549, + "nauc_mrr_at_5_max": 0.259592, + "nauc_mrr_at_5_std": -0.028622, + "nauc_mrr_at_5_diff1": 0.357855, + "nauc_mrr_at_10_max": 0.264184, + "nauc_mrr_at_10_std": -0.024333, + "nauc_mrr_at_10_diff1": 0.353717, + "nauc_mrr_at_20_max": 0.264148, + "nauc_mrr_at_20_std": -0.017082, + "nauc_mrr_at_20_diff1": 0.353438, + "nauc_mrr_at_100_max": 0.263672, + "nauc_mrr_at_100_std": -0.015025, + "nauc_mrr_at_100_diff1": 0.353221, + "nauc_mrr_at_1000_max": 0.263556, + "nauc_mrr_at_1000_std": -0.014899, + "nauc_mrr_at_1000_diff1": 0.353607, + "main_score": 0.26195, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 105.96243500709534, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/ClimateFEVER.json b/mteb/ClimateFEVER.json new file mode 100644 index 0000000000000000000000000000000000000000..b7cbe391fcebb4cca3f5cec350f3ab8f75947adc --- /dev/null +++ b/mteb/ClimateFEVER.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380", + "task_name": "ClimateFEVER", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.21042, + "ndcg_at_3": 0.18007, + "ndcg_at_5": 0.19365, + "ndcg_at_10": 0.22698, + "ndcg_at_20": 0.25245, + "ndcg_at_100": 0.29216, + "ndcg_at_1000": 0.32348, + "map_at_1": 0.09732, + "map_at_3": 0.13098, + "map_at_5": 0.14246, + "map_at_10": 0.15738, + "map_at_20": 0.16557, + "map_at_100": 0.17281, + "map_at_1000": 0.17442, + "recall_at_1": 0.09732, + "recall_at_3": 0.16374, + "recall_at_5": 0.20321, + "recall_at_10": 0.27863, + "recall_at_20": 0.35101, + "recall_at_100": 0.50446, + "recall_at_1000": 0.68127, + "precision_at_1": 0.21042, + "precision_at_3": 0.12942, + "precision_at_5": 0.09967, + "precision_at_10": 0.07114, + "precision_at_20": 0.04635, + "precision_at_100": 0.014, + "precision_at_1000": 0.00197, + "mrr_at_1": 0.210423, + "mrr_at_3": 0.276439, + "mrr_at_5": 0.293377, + "mrr_at_10": 0.309231, + "mrr_at_20": 0.315901, + "mrr_at_100": 0.319553, + "mrr_at_1000": 0.320057, + "nauc_ndcg_at_1_max": 0.253668, + "nauc_ndcg_at_1_std": 0.041982, + "nauc_ndcg_at_1_diff1": 0.263596, + "nauc_ndcg_at_3_max": 0.303375, + "nauc_ndcg_at_3_std": 0.071602, + "nauc_ndcg_at_3_diff1": 0.231569, + "nauc_ndcg_at_5_max": 0.31039, + "nauc_ndcg_at_5_std": 0.08631, + "nauc_ndcg_at_5_diff1": 0.234282, + "nauc_ndcg_at_10_max": 0.321631, + "nauc_ndcg_at_10_std": 0.128647, + "nauc_ndcg_at_10_diff1": 0.224955, + "nauc_ndcg_at_20_max": 0.329699, + "nauc_ndcg_at_20_std": 0.159557, + "nauc_ndcg_at_20_diff1": 0.228594, + "nauc_ndcg_at_100_max": 0.345065, + "nauc_ndcg_at_100_std": 0.20812, + "nauc_ndcg_at_100_diff1": 0.230539, + "nauc_ndcg_at_1000_max": 0.351358, + "nauc_ndcg_at_1000_std": 0.224793, + "nauc_ndcg_at_1000_diff1": 0.228996, + "nauc_map_at_1_max": 0.281958, + "nauc_map_at_1_std": -0.021727, + "nauc_map_at_1_diff1": 0.313643, + "nauc_map_at_3_max": 0.298457, + "nauc_map_at_3_std": 0.022626, + "nauc_map_at_3_diff1": 0.257536, + "nauc_map_at_5_max": 0.304463, + "nauc_map_at_5_std": 0.038367, + "nauc_map_at_5_diff1": 0.257217, + "nauc_map_at_10_max": 0.31309, + "nauc_map_at_10_std": 0.066086, + "nauc_map_at_10_diff1": 0.251932, + "nauc_map_at_20_max": 0.31663, + "nauc_map_at_20_std": 0.07943, + "nauc_map_at_20_diff1": 0.253261, + "nauc_map_at_100_max": 0.320402, + "nauc_map_at_100_std": 0.092723, + "nauc_map_at_100_diff1": 0.252213, + "nauc_map_at_1000_max": 0.320828, + "nauc_map_at_1000_std": 0.094498, + "nauc_map_at_1000_diff1": 0.252002, + "nauc_recall_at_1_max": 0.281958, + "nauc_recall_at_1_std": -0.021727, + "nauc_recall_at_1_diff1": 0.313643, + "nauc_recall_at_3_max": 0.311157, + "nauc_recall_at_3_std": 0.060219, + "nauc_recall_at_3_diff1": 0.210486, + "nauc_recall_at_5_max": 0.297119, + "nauc_recall_at_5_std": 0.096385, + "nauc_recall_at_5_diff1": 0.194008, + "nauc_recall_at_10_max": 0.29758, + "nauc_recall_at_10_std": 0.167827, + "nauc_recall_at_10_diff1": 0.168048, + "nauc_recall_at_20_max": 0.302094, + "nauc_recall_at_20_std": 0.227934, + "nauc_recall_at_20_diff1": 0.167479, + "nauc_recall_at_100_max": 0.325903, + "nauc_recall_at_100_std": 0.356132, + "nauc_recall_at_100_diff1": 0.163043, + "nauc_recall_at_1000_max": 0.365571, + "nauc_recall_at_1000_std": 0.465573, + "nauc_recall_at_1000_diff1": 0.146935, + "nauc_precision_at_1_max": 0.253668, + "nauc_precision_at_1_std": 0.041982, + "nauc_precision_at_1_diff1": 0.263596, + "nauc_precision_at_3_max": 0.302638, + "nauc_precision_at_3_std": 0.164883, + "nauc_precision_at_3_diff1": 0.165101, + "nauc_precision_at_5_max": 0.299424, + "nauc_precision_at_5_std": 0.204229, + "nauc_precision_at_5_diff1": 0.152494, + "nauc_precision_at_10_max": 0.287718, + "nauc_precision_at_10_std": 0.276706, + "nauc_precision_at_10_diff1": 0.105829, + "nauc_precision_at_20_max": 0.276563, + "nauc_precision_at_20_std": 0.329983, + "nauc_precision_at_20_diff1": 0.10746, + "nauc_precision_at_100_max": 0.258573, + "nauc_precision_at_100_std": 0.432574, + "nauc_precision_at_100_diff1": 0.079839, + "nauc_precision_at_1000_max": 0.199198, + "nauc_precision_at_1000_std": 0.440233, + "nauc_precision_at_1000_diff1": 0.024789, + "nauc_mrr_at_1_max": 0.253668, + "nauc_mrr_at_1_std": 0.041982, + "nauc_mrr_at_1_diff1": 0.263596, + "nauc_mrr_at_3_max": 0.298863, + "nauc_mrr_at_3_std": 0.106849, + "nauc_mrr_at_3_diff1": 0.230179, + "nauc_mrr_at_5_max": 0.29585, + "nauc_mrr_at_5_std": 0.117695, + "nauc_mrr_at_5_diff1": 0.225112, + "nauc_mrr_at_10_max": 0.297507, + "nauc_mrr_at_10_std": 0.128053, + "nauc_mrr_at_10_diff1": 0.222167, + "nauc_mrr_at_20_max": 0.298771, + "nauc_mrr_at_20_std": 0.13317, + "nauc_mrr_at_20_diff1": 0.223668, + "nauc_mrr_at_100_max": 0.299334, + "nauc_mrr_at_100_std": 0.133076, + "nauc_mrr_at_100_diff1": 0.224883, + "nauc_mrr_at_1000_max": 0.299249, + "nauc_mrr_at_1000_std": 0.132749, + "nauc_mrr_at_1000_diff1": 0.224871, + "main_score": 0.22698, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5528.608217954636, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/DBPedia.json b/mteb/DBPedia.json new file mode 100644 index 0000000000000000000000000000000000000000..08c9f0b6d910b304a805a02786003c86f8a05bdb --- /dev/null +++ b/mteb/DBPedia.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659", + "task_name": "DBPedia", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.41375, + "ndcg_at_3": 0.32494, + "ndcg_at_5": 0.29511, + "ndcg_at_10": 0.27559, + "ndcg_at_20": 0.26821, + "ndcg_at_100": 0.29407, + "ndcg_at_1000": 0.34784, + "map_at_1": 0.07455, + "map_at_3": 0.10611, + "map_at_5": 0.1174, + "map_at_10": 0.13131, + "map_at_20": 0.14319, + "map_at_100": 0.1607, + "map_at_1000": 0.16731, + "recall_at_1": 0.07455, + "recall_at_3": 0.11476, + "recall_at_5": 0.13383, + "recall_at_10": 0.16742, + "recall_at_20": 0.2134, + "recall_at_100": 0.32125, + "recall_at_1000": 0.49776, + "precision_at_1": 0.51, + "precision_at_3": 0.3475, + "precision_at_5": 0.2705, + "precision_at_10": 0.19575, + "precision_at_20": 0.13988, + "precision_at_100": 0.0535, + "precision_at_1000": 0.01072, + "mrr_at_1": 0.51, + "mrr_at_3": 0.57875, + "mrr_at_5": 0.585375, + "mrr_at_10": 0.591851, + "mrr_at_20": 0.596647, + "mrr_at_100": 0.598812, + "mrr_at_1000": 0.599078, + "nauc_ndcg_at_1_max": 0.423916, + "nauc_ndcg_at_1_std": 0.155682, + "nauc_ndcg_at_1_diff1": 0.431251, + "nauc_ndcg_at_3_max": 0.438604, + "nauc_ndcg_at_3_std": 0.193398, + "nauc_ndcg_at_3_diff1": 0.308724, + "nauc_ndcg_at_5_max": 0.414941, + "nauc_ndcg_at_5_std": 0.182644, + "nauc_ndcg_at_5_diff1": 0.287275, + "nauc_ndcg_at_10_max": 0.390035, + "nauc_ndcg_at_10_std": 0.176473, + "nauc_ndcg_at_10_diff1": 0.279835, + "nauc_ndcg_at_20_max": 0.357069, + "nauc_ndcg_at_20_std": 0.165987, + "nauc_ndcg_at_20_diff1": 0.282564, + "nauc_ndcg_at_100_max": 0.343485, + "nauc_ndcg_at_100_std": 0.205251, + "nauc_ndcg_at_100_diff1": 0.258367, + "nauc_ndcg_at_1000_max": 0.380072, + "nauc_ndcg_at_1000_std": 0.265801, + "nauc_ndcg_at_1000_diff1": 0.257738, + "nauc_map_at_1_max": 0.118856, + "nauc_map_at_1_std": -0.093654, + "nauc_map_at_1_diff1": 0.383665, + "nauc_map_at_3_max": 0.163439, + "nauc_map_at_3_std": -0.050284, + "nauc_map_at_3_diff1": 0.323527, + "nauc_map_at_5_max": 0.17271, + "nauc_map_at_5_std": -0.034506, + "nauc_map_at_5_diff1": 0.305797, + "nauc_map_at_10_max": 0.187527, + "nauc_map_at_10_std": 0.003105, + "nauc_map_at_10_diff1": 0.281637, + "nauc_map_at_20_max": 0.206161, + "nauc_map_at_20_std": 0.043977, + "nauc_map_at_20_diff1": 0.26506, + "nauc_map_at_100_max": 0.236987, + "nauc_map_at_100_std": 0.115381, + "nauc_map_at_100_diff1": 0.233655, + "nauc_map_at_1000_max": 0.24553, + "nauc_map_at_1000_std": 0.132969, + "nauc_map_at_1000_diff1": 0.230466, + "nauc_recall_at_1_max": 0.118856, + "nauc_recall_at_1_std": -0.093654, + "nauc_recall_at_1_diff1": 0.383665, + "nauc_recall_at_3_max": 0.144271, + "nauc_recall_at_3_std": -0.053232, + "nauc_recall_at_3_diff1": 0.297432, + "nauc_recall_at_5_max": 0.150162, + "nauc_recall_at_5_std": -0.041646, + "nauc_recall_at_5_diff1": 0.283241, + "nauc_recall_at_10_max": 0.155396, + "nauc_recall_at_10_std": -0.00623, + "nauc_recall_at_10_diff1": 0.246545, + "nauc_recall_at_20_max": 0.155934, + "nauc_recall_at_20_std": 0.03434, + "nauc_recall_at_20_diff1": 0.217566, + "nauc_recall_at_100_max": 0.183441, + "nauc_recall_at_100_std": 0.179899, + "nauc_recall_at_100_diff1": 0.154327, + "nauc_recall_at_1000_max": 0.253257, + "nauc_recall_at_1000_std": 0.313636, + "nauc_recall_at_1000_diff1": 0.147442, + "nauc_precision_at_1_max": 0.451052, + "nauc_precision_at_1_std": 0.189451, + "nauc_precision_at_1_diff1": 0.462767, + "nauc_precision_at_3_max": 0.441379, + "nauc_precision_at_3_std": 0.265232, + "nauc_precision_at_3_diff1": 0.183643, + "nauc_precision_at_5_max": 0.412908, + "nauc_precision_at_5_std": 0.282232, + "nauc_precision_at_5_diff1": 0.105613, + "nauc_precision_at_10_max": 0.389052, + "nauc_precision_at_10_std": 0.366196, + "nauc_precision_at_10_diff1": 0.030044, + "nauc_precision_at_20_max": 0.372606, + "nauc_precision_at_20_std": 0.450747, + "nauc_precision_at_20_diff1": -0.014095, + "nauc_precision_at_100_max": 0.315374, + "nauc_precision_at_100_std": 0.498572, + "nauc_precision_at_100_diff1": -0.082395, + "nauc_precision_at_1000_max": 0.199796, + "nauc_precision_at_1000_std": 0.386212, + "nauc_precision_at_1000_diff1": -0.119969, + "nauc_mrr_at_1_max": 0.451052, + "nauc_mrr_at_1_std": 0.189451, + "nauc_mrr_at_1_diff1": 0.462767, + "nauc_mrr_at_3_max": 0.483251, + "nauc_mrr_at_3_std": 0.253881, + "nauc_mrr_at_3_diff1": 0.434542, + "nauc_mrr_at_5_max": 0.483313, + "nauc_mrr_at_5_std": 0.251794, + "nauc_mrr_at_5_diff1": 0.435443, + "nauc_mrr_at_10_max": 0.480391, + "nauc_mrr_at_10_std": 0.249537, + "nauc_mrr_at_10_diff1": 0.431642, + "nauc_mrr_at_20_max": 0.479453, + "nauc_mrr_at_20_std": 0.251964, + "nauc_mrr_at_20_diff1": 0.431792, + "nauc_mrr_at_100_max": 0.4801, + "nauc_mrr_at_100_std": 0.252416, + "nauc_mrr_at_100_diff1": 0.433313, + "nauc_mrr_at_1000_max": 0.479913, + "nauc_mrr_at_1000_std": 0.252061, + "nauc_mrr_at_1000_diff1": 0.433367, + "main_score": 0.27559, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3010.4430978298187, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/EmotionClassification.json b/mteb/EmotionClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..b4b43d139cb2ea441602dc1ad2e90ce20a20b856 --- /dev/null +++ b/mteb/EmotionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37", + "task_name": "EmotionClassification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.437, + "f1": 0.39358, + "f1_weighted": 0.45983, + "scores_per_experiment": [ + { + "accuracy": 0.4555, + "f1": 0.399212, + "f1_weighted": 0.484673 + }, + { + "accuracy": 0.432, + "f1": 0.394225, + "f1_weighted": 0.447477 + }, + { + "accuracy": 0.415, + "f1": 0.379312, + "f1_weighted": 0.440126 + }, + { + "accuracy": 0.413, + "f1": 0.374642, + "f1_weighted": 0.440663 + }, + { + "accuracy": 0.456, + "f1": 0.41365, + "f1_weighted": 0.47793 + }, + { + "accuracy": 0.4375, + "f1": 0.390637, + "f1_weighted": 0.45846 + }, + { + "accuracy": 0.447, + "f1": 0.402599, + "f1_weighted": 0.470503 + }, + { + "accuracy": 0.4135, + "f1": 0.376153, + "f1_weighted": 0.428264 + }, + { + "accuracy": 0.461, + "f1": 0.414495, + "f1_weighted": 0.481941 + }, + { + "accuracy": 0.4395, + "f1": 0.390877, + "f1_weighted": 0.468265 + } + ], + "main_score": 0.437, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.992861747741699, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/FEVER.json b/mteb/FEVER.json new file mode 100644 index 0000000000000000000000000000000000000000..cd93df819b674325935aeab3cbbacaaae6fd7521 --- /dev/null +++ b/mteb/FEVER.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12", + "task_name": "FEVER", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.52625, + "ndcg_at_3": 0.61126, + "ndcg_at_5": 0.63731, + "ndcg_at_10": 0.65721, + "ndcg_at_20": 0.66718, + "ndcg_at_100": 0.67996, + "ndcg_at_1000": 0.68554, + "map_at_1": 0.48757, + "map_at_3": 0.57408, + "map_at_5": 0.58912, + "map_at_10": 0.59765, + "map_at_20": 0.60053, + "map_at_100": 0.60235, + "map_at_1000": 0.60258, + "recall_at_1": 0.48757, + "recall_at_3": 0.67712, + "recall_at_5": 0.74102, + "recall_at_10": 0.80181, + "recall_at_20": 0.83964, + "recall_at_100": 0.90629, + "recall_at_1000": 0.9479, + "precision_at_1": 0.52625, + "precision_at_3": 0.24482, + "precision_at_5": 0.16112, + "precision_at_10": 0.08738, + "precision_at_20": 0.0459, + "precision_at_100": 0.00997, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.526253, + "mrr_at_3": 0.615912, + "mrr_at_5": 0.631183, + "mrr_at_10": 0.639444, + "mrr_at_20": 0.642131, + "mrr_at_100": 0.643808, + "mrr_at_1000": 0.643955, + "nauc_ndcg_at_1_max": 0.245061, + "nauc_ndcg_at_1_std": -0.137009, + "nauc_ndcg_at_1_diff1": 0.55709, + "nauc_ndcg_at_3_max": 0.248319, + "nauc_ndcg_at_3_std": -0.104366, + "nauc_ndcg_at_3_diff1": 0.457855, + "nauc_ndcg_at_5_max": 0.255093, + "nauc_ndcg_at_5_std": -0.091996, + "nauc_ndcg_at_5_diff1": 0.456859, + "nauc_ndcg_at_10_max": 0.250742, + "nauc_ndcg_at_10_std": -0.080252, + "nauc_ndcg_at_10_diff1": 0.461142, + "nauc_ndcg_at_20_max": 0.247866, + "nauc_ndcg_at_20_std": -0.075957, + "nauc_ndcg_at_20_diff1": 0.459826, + "nauc_ndcg_at_100_max": 0.24549, + "nauc_ndcg_at_100_std": -0.07457, + "nauc_ndcg_at_100_diff1": 0.460618, + "nauc_ndcg_at_1000_max": 0.245226, + "nauc_ndcg_at_1000_std": -0.077771, + "nauc_ndcg_at_1000_diff1": 0.462122, + "nauc_map_at_1_max": 0.223226, + "nauc_map_at_1_std": -0.111902, + "nauc_map_at_1_diff1": 0.506577, + "nauc_map_at_3_max": 0.236994, + "nauc_map_at_3_std": -0.1019, + "nauc_map_at_3_diff1": 0.460434, + "nauc_map_at_5_max": 0.241242, + "nauc_map_at_5_std": -0.096404, + "nauc_map_at_5_diff1": 0.461683, + "nauc_map_at_10_max": 0.239856, + "nauc_map_at_10_std": -0.092091, + "nauc_map_at_10_diff1": 0.463714, + "nauc_map_at_20_max": 0.239147, + "nauc_map_at_20_std": -0.091015, + "nauc_map_at_20_diff1": 0.463512, + "nauc_map_at_100_max": 0.238856, + "nauc_map_at_100_std": -0.090851, + "nauc_map_at_100_diff1": 0.463642, + "nauc_map_at_1000_max": 0.238833, + "nauc_map_at_1000_std": -0.090946, + "nauc_map_at_1000_diff1": 0.463676, + "nauc_recall_at_1_max": 0.223226, + "nauc_recall_at_1_std": -0.111902, + "nauc_recall_at_1_diff1": 0.506577, + "nauc_recall_at_3_max": 0.240556, + "nauc_recall_at_3_std": -0.078741, + "nauc_recall_at_3_diff1": 0.372825, + "nauc_recall_at_5_max": 0.257287, + "nauc_recall_at_5_std": -0.039755, + "nauc_recall_at_5_diff1": 0.352184, + "nauc_recall_at_10_max": 0.236078, + "nauc_recall_at_10_std": 0.021126, + "nauc_recall_at_10_diff1": 0.340759, + "nauc_recall_at_20_max": 0.211544, + "nauc_recall_at_20_std": 0.068803, + "nauc_recall_at_20_diff1": 0.302302, + "nauc_recall_at_100_max": 0.146115, + "nauc_recall_at_100_std": 0.18329, + "nauc_recall_at_100_diff1": 0.20034, + "nauc_recall_at_1000_max": 0.071731, + "nauc_recall_at_1000_std": 0.257758, + "nauc_recall_at_1000_diff1": 0.069671, + "nauc_precision_at_1_max": 0.245061, + "nauc_precision_at_1_std": -0.137009, + "nauc_precision_at_1_diff1": 0.55709, + "nauc_precision_at_3_max": 0.280319, + "nauc_precision_at_3_std": -0.106341, + "nauc_precision_at_3_diff1": 0.421819, + "nauc_precision_at_5_max": 0.306886, + "nauc_precision_at_5_std": -0.063768, + "nauc_precision_at_5_diff1": 0.400642, + "nauc_precision_at_10_max": 0.281175, + "nauc_precision_at_10_std": 0.010407, + "nauc_precision_at_10_diff1": 0.373056, + "nauc_precision_at_20_max": 0.246245, + "nauc_precision_at_20_std": 0.065341, + "nauc_precision_at_20_diff1": 0.313258, + "nauc_precision_at_100_max": 0.161083, + "nauc_precision_at_100_std": 0.165796, + "nauc_precision_at_100_diff1": 0.14073, + "nauc_precision_at_1000_max": 0.055871, + "nauc_precision_at_1000_std": 0.147183, + "nauc_precision_at_1000_diff1": -0.017647, + "nauc_mrr_at_1_max": 0.245061, + "nauc_mrr_at_1_std": -0.137009, + "nauc_mrr_at_1_diff1": 0.55709, + "nauc_mrr_at_3_max": 0.26592, + "nauc_mrr_at_3_std": -0.128604, + "nauc_mrr_at_3_diff1": 0.515274, + "nauc_mrr_at_5_max": 0.270103, + "nauc_mrr_at_5_std": -0.122957, + "nauc_mrr_at_5_diff1": 0.516955, + "nauc_mrr_at_10_max": 0.268861, + "nauc_mrr_at_10_std": -0.119639, + "nauc_mrr_at_10_diff1": 0.520291, + "nauc_mrr_at_20_max": 0.268109, + "nauc_mrr_at_20_std": -0.11923, + "nauc_mrr_at_20_diff1": 0.520273, + "nauc_mrr_at_100_max": 0.267881, + "nauc_mrr_at_100_std": -0.119438, + "nauc_mrr_at_100_diff1": 0.520616, + "nauc_mrr_at_1000_max": 0.267828, + "nauc_mrr_at_1000_std": -0.119632, + "nauc_mrr_at_1000_diff1": 0.520671, + "main_score": 0.65721, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5951.847348451614, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/FiQA2018.json b/mteb/FiQA2018.json new file mode 100644 index 0000000000000000000000000000000000000000..3b26596f4c2c5174c6c952f8c6296a46e7df1038 --- /dev/null +++ b/mteb/FiQA2018.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06", + "task_name": "FiQA2018", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.29167, + "ndcg_at_3": 0.26738, + "ndcg_at_5": 0.27998, + "ndcg_at_10": 0.30568, + "ndcg_at_20": 0.32864, + "ndcg_at_100": 0.36517, + "ndcg_at_1000": 0.4022, + "map_at_1": 0.13794, + "map_at_3": 0.19969, + "map_at_5": 0.21926, + "map_at_10": 0.23496, + "map_at_20": 0.24335, + "map_at_100": 0.2499, + "map_at_1000": 0.25182, + "recall_at_1": 0.13794, + "recall_at_3": 0.24295, + "recall_at_5": 0.29735, + "recall_at_10": 0.38048, + "recall_at_20": 0.44955, + "recall_at_100": 0.60103, + "recall_at_1000": 0.82989, + "precision_at_1": 0.29167, + "precision_at_3": 0.18313, + "precision_at_5": 0.13827, + "precision_at_10": 0.08827, + "precision_at_20": 0.05386, + "precision_at_100": 0.01489, + "precision_at_1000": 0.00213, + "mrr_at_1": 0.291667, + "mrr_at_3": 0.344136, + "mrr_at_5": 0.357639, + "mrr_at_10": 0.370531, + "mrr_at_20": 0.375432, + "mrr_at_100": 0.379265, + "mrr_at_1000": 0.379918, + "nauc_ndcg_at_1_max": 0.382865, + "nauc_ndcg_at_1_std": -0.018401, + "nauc_ndcg_at_1_diff1": 0.497381, + "nauc_ndcg_at_3_max": 0.323251, + "nauc_ndcg_at_3_std": -0.015778, + "nauc_ndcg_at_3_diff1": 0.393907, + "nauc_ndcg_at_5_max": 0.321995, + "nauc_ndcg_at_5_std": 0.010656, + "nauc_ndcg_at_5_diff1": 0.390196, + "nauc_ndcg_at_10_max": 0.307513, + "nauc_ndcg_at_10_std": 0.025637, + "nauc_ndcg_at_10_diff1": 0.376314, + "nauc_ndcg_at_20_max": 0.308166, + "nauc_ndcg_at_20_std": 0.041018, + "nauc_ndcg_at_20_diff1": 0.366269, + "nauc_ndcg_at_100_max": 0.329718, + "nauc_ndcg_at_100_std": 0.06222, + "nauc_ndcg_at_100_diff1": 0.377411, + "nauc_ndcg_at_1000_max": 0.342405, + "nauc_ndcg_at_1000_std": 0.066923, + "nauc_ndcg_at_1000_diff1": 0.379008, + "nauc_map_at_1_max": 0.25955, + "nauc_map_at_1_std": -0.068494, + "nauc_map_at_1_diff1": 0.455645, + "nauc_map_at_3_max": 0.284944, + "nauc_map_at_3_std": -0.028556, + "nauc_map_at_3_diff1": 0.403951, + "nauc_map_at_5_max": 0.302179, + "nauc_map_at_5_std": -0.006055, + "nauc_map_at_5_diff1": 0.398122, + "nauc_map_at_10_max": 0.304114, + "nauc_map_at_10_std": 0.002683, + "nauc_map_at_10_diff1": 0.391394, + "nauc_map_at_20_max": 0.305538, + "nauc_map_at_20_std": 0.008069, + "nauc_map_at_20_diff1": 0.387448, + "nauc_map_at_100_max": 0.311454, + "nauc_map_at_100_std": 0.013278, + "nauc_map_at_100_diff1": 0.389779, + "nauc_map_at_1000_max": 0.312485, + "nauc_map_at_1000_std": 0.014026, + "nauc_map_at_1000_diff1": 0.389806, + "nauc_recall_at_1_max": 0.25955, + "nauc_recall_at_1_std": -0.068494, + "nauc_recall_at_1_diff1": 0.455645, + "nauc_recall_at_3_max": 0.240749, + "nauc_recall_at_3_std": -0.009688, + "nauc_recall_at_3_diff1": 0.338397, + "nauc_recall_at_5_max": 0.249765, + "nauc_recall_at_5_std": 0.041173, + "nauc_recall_at_5_diff1": 0.306595, + "nauc_recall_at_10_max": 0.194711, + "nauc_recall_at_10_std": 0.068708, + "nauc_recall_at_10_diff1": 0.235346, + "nauc_recall_at_20_max": 0.187834, + "nauc_recall_at_20_std": 0.115026, + "nauc_recall_at_20_diff1": 0.195265, + "nauc_recall_at_100_max": 0.248323, + "nauc_recall_at_100_std": 0.209214, + "nauc_recall_at_100_diff1": 0.222041, + "nauc_recall_at_1000_max": 0.30524, + "nauc_recall_at_1000_std": 0.365714, + "nauc_recall_at_1000_diff1": 0.148893, + "nauc_precision_at_1_max": 0.382865, + "nauc_precision_at_1_std": -0.018401, + "nauc_precision_at_1_diff1": 0.497381, + "nauc_precision_at_3_max": 0.367225, + "nauc_precision_at_3_std": 0.013713, + "nauc_precision_at_3_diff1": 0.337091, + "nauc_precision_at_5_max": 0.378168, + "nauc_precision_at_5_std": 0.063933, + "nauc_precision_at_5_diff1": 0.305218, + "nauc_precision_at_10_max": 0.365227, + "nauc_precision_at_10_std": 0.099963, + "nauc_precision_at_10_diff1": 0.252887, + "nauc_precision_at_20_max": 0.342121, + "nauc_precision_at_20_std": 0.122905, + "nauc_precision_at_20_diff1": 0.214998, + "nauc_precision_at_100_max": 0.342554, + "nauc_precision_at_100_std": 0.163248, + "nauc_precision_at_100_diff1": 0.1778, + "nauc_precision_at_1000_max": 0.28898, + "nauc_precision_at_1000_std": 0.146869, + "nauc_precision_at_1000_diff1": 0.081735, + "nauc_mrr_at_1_max": 0.382865, + "nauc_mrr_at_1_std": -0.018401, + "nauc_mrr_at_1_diff1": 0.497381, + "nauc_mrr_at_3_max": 0.361507, + "nauc_mrr_at_3_std": -0.010684, + "nauc_mrr_at_3_diff1": 0.448131, + "nauc_mrr_at_5_max": 0.364134, + "nauc_mrr_at_5_std": 0.0015, + "nauc_mrr_at_5_diff1": 0.448739, + "nauc_mrr_at_10_max": 0.358277, + "nauc_mrr_at_10_std": 0.003988, + "nauc_mrr_at_10_diff1": 0.439851, + "nauc_mrr_at_20_max": 0.358842, + "nauc_mrr_at_20_std": 0.008399, + "nauc_mrr_at_20_diff1": 0.438136, + "nauc_mrr_at_100_max": 0.359867, + "nauc_mrr_at_100_std": 0.009229, + "nauc_mrr_at_100_diff1": 0.439474, + "nauc_mrr_at_1000_max": 0.359973, + "nauc_mrr_at_1000_std": 0.008946, + "nauc_mrr_at_1000_diff1": 0.439639, + "main_score": 0.30568, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 80.82175087928772, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/HotpotQA.json b/mteb/HotpotQA.json new file mode 100644 index 0000000000000000000000000000000000000000..e75a8b1449217a2386e500531fad17d6d56be112 --- /dev/null +++ b/mteb/HotpotQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ab518f4d6fcca38d87c25209f94beba119d02014", + "task_name": "HotpotQA", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.60162, + "ndcg_at_3": 0.44502, + "ndcg_at_5": 0.46295, + "ndcg_at_10": 0.47809, + "ndcg_at_20": 0.48903, + "ndcg_at_100": 0.50535, + "ndcg_at_1000": 0.51923, + "map_at_1": 0.30081, + "map_at_3": 0.36953, + "map_at_5": 0.38167, + "map_at_10": 0.38938, + "map_at_20": 0.39314, + "map_at_100": 0.39603, + "map_at_1000": 0.39663, + "recall_at_1": 0.30081, + "recall_at_3": 0.40614, + "recall_at_5": 0.44159, + "recall_at_10": 0.47961, + "recall_at_20": 0.51499, + "recall_at_100": 0.58731, + "recall_at_1000": 0.68035, + "precision_at_1": 0.60162, + "precision_at_3": 0.27076, + "precision_at_5": 0.17664, + "precision_at_10": 0.09592, + "precision_at_20": 0.0515, + "precision_at_100": 0.01175, + "precision_at_1000": 0.00136, + "mrr_at_1": 0.601621, + "mrr_at_3": 0.652622, + "mrr_at_5": 0.660826, + "mrr_at_10": 0.665901, + "mrr_at_20": 0.668133, + "mrr_at_100": 0.669551, + "mrr_at_1000": 0.669784, + "nauc_ndcg_at_1_max": 0.5116, + "nauc_ndcg_at_1_std": 0.073935, + "nauc_ndcg_at_1_diff1": 0.715694, + "nauc_ndcg_at_3_max": 0.378233, + "nauc_ndcg_at_3_std": 0.067468, + "nauc_ndcg_at_3_diff1": 0.494853, + "nauc_ndcg_at_5_max": 0.359559, + "nauc_ndcg_at_5_std": 0.073257, + "nauc_ndcg_at_5_diff1": 0.465013, + "nauc_ndcg_at_10_max": 0.339948, + "nauc_ndcg_at_10_std": 0.076042, + "nauc_ndcg_at_10_diff1": 0.443659, + "nauc_ndcg_at_20_max": 0.328131, + "nauc_ndcg_at_20_std": 0.074644, + "nauc_ndcg_at_20_diff1": 0.432222, + "nauc_ndcg_at_100_max": 0.312544, + "nauc_ndcg_at_100_std": 0.078449, + "nauc_ndcg_at_100_diff1": 0.415341, + "nauc_ndcg_at_1000_max": 0.305183, + "nauc_ndcg_at_1000_std": 0.078385, + "nauc_ndcg_at_1000_diff1": 0.409027, + "nauc_map_at_1_max": 0.5116, + "nauc_map_at_1_std": 0.073935, + "nauc_map_at_1_diff1": 0.715694, + "nauc_map_at_3_max": 0.334791, + "nauc_map_at_3_std": 0.058541, + "nauc_map_at_3_diff1": 0.444158, + "nauc_map_at_5_max": 0.31872, + "nauc_map_at_5_std": 0.060961, + "nauc_map_at_5_diff1": 0.421822, + "nauc_map_at_10_max": 0.307675, + "nauc_map_at_10_std": 0.06241, + "nauc_map_at_10_diff1": 0.410375, + "nauc_map_at_20_max": 0.303193, + "nauc_map_at_20_std": 0.061659, + "nauc_map_at_20_diff1": 0.406205, + "nauc_map_at_100_max": 0.299934, + "nauc_map_at_100_std": 0.062056, + "nauc_map_at_100_diff1": 0.402882, + "nauc_map_at_1000_max": 0.299604, + "nauc_map_at_1000_std": 0.06212, + "nauc_map_at_1000_diff1": 0.402604, + "nauc_recall_at_1_max": 0.5116, + "nauc_recall_at_1_std": 0.073935, + "nauc_recall_at_1_diff1": 0.715694, + "nauc_recall_at_3_max": 0.295614, + "nauc_recall_at_3_std": 0.061532, + "nauc_recall_at_3_diff1": 0.365997, + "nauc_recall_at_5_max": 0.250882, + "nauc_recall_at_5_std": 0.072136, + "nauc_recall_at_5_diff1": 0.296869, + "nauc_recall_at_10_max": 0.187916, + "nauc_recall_at_10_std": 0.075065, + "nauc_recall_at_10_diff1": 0.227623, + "nauc_recall_at_20_max": 0.139061, + "nauc_recall_at_20_std": 0.067366, + "nauc_recall_at_20_diff1": 0.176912, + "nauc_recall_at_100_max": 0.053559, + "nauc_recall_at_100_std": 0.077563, + "nauc_recall_at_100_diff1": 0.080251, + "nauc_recall_at_1000_max": -0.03957, + "nauc_recall_at_1000_std": 0.069824, + "nauc_recall_at_1000_diff1": -0.020025, + "nauc_precision_at_1_max": 0.5116, + "nauc_precision_at_1_std": 0.073935, + "nauc_precision_at_1_diff1": 0.715694, + "nauc_precision_at_3_max": 0.295614, + "nauc_precision_at_3_std": 0.061532, + "nauc_precision_at_3_diff1": 0.365997, + "nauc_precision_at_5_max": 0.250882, + "nauc_precision_at_5_std": 0.072136, + "nauc_precision_at_5_diff1": 0.296869, + "nauc_precision_at_10_max": 0.187916, + "nauc_precision_at_10_std": 0.075065, + "nauc_precision_at_10_diff1": 0.227623, + "nauc_precision_at_20_max": 0.139061, + "nauc_precision_at_20_std": 0.067366, + "nauc_precision_at_20_diff1": 0.176912, + "nauc_precision_at_100_max": 0.053559, + "nauc_precision_at_100_std": 0.077563, + "nauc_precision_at_100_diff1": 0.080251, + "nauc_precision_at_1000_max": -0.03957, + "nauc_precision_at_1000_std": 0.069824, + "nauc_precision_at_1000_diff1": -0.020025, + "nauc_mrr_at_1_max": 0.5116, + "nauc_mrr_at_1_std": 0.073935, + "nauc_mrr_at_1_diff1": 0.715694, + "nauc_mrr_at_3_max": 0.516213, + "nauc_mrr_at_3_std": 0.084836, + "nauc_mrr_at_3_diff1": 0.68779, + "nauc_mrr_at_5_max": 0.518266, + "nauc_mrr_at_5_std": 0.090287, + "nauc_mrr_at_5_diff1": 0.684759, + "nauc_mrr_at_10_max": 0.516795, + "nauc_mrr_at_10_std": 0.091841, + "nauc_mrr_at_10_diff1": 0.682263, + "nauc_mrr_at_20_max": 0.516508, + "nauc_mrr_at_20_std": 0.092047, + "nauc_mrr_at_20_diff1": 0.682142, + "nauc_mrr_at_100_max": 0.516723, + "nauc_mrr_at_100_std": 0.093048, + "nauc_mrr_at_100_diff1": 0.682074, + "nauc_mrr_at_1000_max": 0.516691, + "nauc_mrr_at_1000_std": 0.093012, + "nauc_mrr_at_1000_diff1": 0.682124, + "main_score": 0.47809, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3732.872656583786, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/ImdbClassification.json b/mteb/ImdbClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..a9c5c2fccc6e2164327cf06295ebce881dd5a19a --- /dev/null +++ b/mteb/ImdbClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7", + "task_name": "ImdbClassification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.61454, + "f1": 0.611751, + "f1_weighted": 0.611751, + "ap": 0.572238, + "ap_weighted": 0.572238, + "scores_per_experiment": [ + { + "accuracy": 0.64184, + "f1": 0.641046, + "f1_weighted": 0.641046, + "ap": 0.593128, + "ap_weighted": 0.593128 + }, + { + "accuracy": 0.64316, + "f1": 0.643142, + "f1_weighted": 0.643142, + "ap": 0.591785, + "ap_weighted": 0.591785 + }, + { + "accuracy": 0.5602, + "f1": 0.556495, + "f1_weighted": 0.556495, + "ap": 0.534535, + "ap_weighted": 0.534535 + }, + { + "accuracy": 0.63604, + "f1": 0.635737, + "f1_weighted": 0.635737, + "ap": 0.58766, + "ap_weighted": 0.58766 + }, + { + "accuracy": 0.60992, + "f1": 0.604073, + "f1_weighted": 0.604073, + "ap": 0.56468, + "ap_weighted": 0.56468 + }, + { + "accuracy": 0.60764, + "f1": 0.606284, + "f1_weighted": 0.606284, + "ap": 0.566947, + "ap_weighted": 0.566947 + }, + { + "accuracy": 0.60716, + "f1": 0.592887, + "f1_weighted": 0.592887, + "ap": 0.571938, + "ap_weighted": 0.571938 + }, + { + "accuracy": 0.60104, + "f1": 0.599904, + "f1_weighted": 0.599904, + "ap": 0.561947, + "ap_weighted": 0.561947 + }, + { + "accuracy": 0.5806, + "f1": 0.580167, + "f1_weighted": 0.580167, + "ap": 0.546404, + "ap_weighted": 0.546404 + }, + { + "accuracy": 0.6578, + "f1": 0.657772, + "f1_weighted": 0.657772, + "ap": 0.603361, + "ap_weighted": 0.603361 + } + ], + "main_score": 0.61454, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 65.28035974502563, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/MSMARCO.json b/mteb/MSMARCO.json new file mode 100644 index 0000000000000000000000000000000000000000..c6c59cd3e09c181b08253d82093028192f26b957 --- /dev/null +++ b/mteb/MSMARCO.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c5a29a104738b98a9e76336939199e264163d4a0", + "task_name": "MSMARCO", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.76744, + "ndcg_at_3": 0.71949, + "ndcg_at_5": 0.68963, + "ndcg_at_10": 0.66652, + "ndcg_at_20": 0.63025, + "ndcg_at_100": 0.57169, + "ndcg_at_1000": 0.64349, + "map_at_1": 0.02314, + "map_at_3": 0.05259, + "map_at_5": 0.07976, + "map_at_10": 0.13288, + "map_at_20": 0.19885, + "map_at_100": 0.32109, + "map_at_1000": 0.38117, + "recall_at_1": 0.02314, + "recall_at_3": 0.05397, + "recall_at_5": 0.08814, + "recall_at_10": 0.15347, + "recall_at_20": 0.23955, + "recall_at_100": 0.44498, + "recall_at_1000": 0.69357, + "precision_at_1": 0.95349, + "precision_at_3": 0.86047, + "precision_at_5": 0.79535, + "precision_at_10": 0.72558, + "precision_at_20": 0.62907, + "precision_at_100": 0.31977, + "precision_at_1000": 0.06049, + "mrr_at_1": 0.953488, + "mrr_at_3": 0.965116, + "mrr_at_5": 0.969767, + "mrr_at_10": 0.969767, + "mrr_at_20": 0.969767, + "mrr_at_100": 0.969767, + "mrr_at_1000": 0.969767, + "nauc_ndcg_at_1_max": 0.335394, + "nauc_ndcg_at_1_std": 0.12295, + "nauc_ndcg_at_1_diff1": 0.160874, + "nauc_ndcg_at_3_max": 0.329494, + "nauc_ndcg_at_3_std": 0.212066, + "nauc_ndcg_at_3_diff1": 0.019303, + "nauc_ndcg_at_5_max": 0.268626, + "nauc_ndcg_at_5_std": 0.198698, + "nauc_ndcg_at_5_diff1": -0.130704, + "nauc_ndcg_at_10_max": 0.258121, + "nauc_ndcg_at_10_std": 0.314913, + "nauc_ndcg_at_10_diff1": -0.321204, + "nauc_ndcg_at_20_max": 0.308474, + "nauc_ndcg_at_20_std": 0.486443, + "nauc_ndcg_at_20_diff1": -0.412659, + "nauc_ndcg_at_100_max": 0.319625, + "nauc_ndcg_at_100_std": 0.561189, + "nauc_ndcg_at_100_diff1": -0.451756, + "nauc_ndcg_at_1000_max": 0.388789, + "nauc_ndcg_at_1000_std": 0.603502, + "nauc_ndcg_at_1000_diff1": -0.463901, + "nauc_map_at_1_max": -0.286523, + "nauc_map_at_1_std": -0.331061, + "nauc_map_at_1_diff1": -0.11994, + "nauc_map_at_3_max": -0.160122, + "nauc_map_at_3_std": -0.219302, + "nauc_map_at_3_diff1": -0.038522, + "nauc_map_at_5_max": -0.120035, + "nauc_map_at_5_std": -0.209837, + "nauc_map_at_5_diff1": -0.05801, + "nauc_map_at_10_max": -0.044091, + "nauc_map_at_10_std": -0.134428, + "nauc_map_at_10_diff1": -0.104398, + "nauc_map_at_20_max": 0.029722, + "nauc_map_at_20_std": -0.021077, + "nauc_map_at_20_diff1": -0.205479, + "nauc_map_at_100_max": 0.181352, + "nauc_map_at_100_std": 0.363231, + "nauc_map_at_100_diff1": -0.383395, + "nauc_map_at_1000_max": 0.28169, + "nauc_map_at_1000_std": 0.600619, + "nauc_map_at_1000_diff1": -0.438413, + "nauc_recall_at_1_max": -0.286523, + "nauc_recall_at_1_std": -0.331061, + "nauc_recall_at_1_diff1": -0.11994, + "nauc_recall_at_3_max": -0.162157, + "nauc_recall_at_3_std": -0.230294, + "nauc_recall_at_3_diff1": -0.042975, + "nauc_recall_at_5_max": -0.183205, + "nauc_recall_at_5_std": -0.268365, + "nauc_recall_at_5_diff1": -0.101943, + "nauc_recall_at_10_max": -0.119993, + "nauc_recall_at_10_std": -0.20104, + "nauc_recall_at_10_diff1": -0.124464, + "nauc_recall_at_20_max": -0.068623, + "nauc_recall_at_20_std": -0.114267, + "nauc_recall_at_20_diff1": -0.202857, + "nauc_recall_at_100_max": 0.055972, + "nauc_recall_at_100_std": 0.207599, + "nauc_recall_at_100_diff1": -0.354162, + "nauc_recall_at_1000_max": 0.260786, + "nauc_recall_at_1000_std": 0.526085, + "nauc_recall_at_1000_diff1": -0.411833, + "nauc_precision_at_1_max": 0.558702, + "nauc_precision_at_1_std": 0.67915, + "nauc_precision_at_1_diff1": 0.614293, + "nauc_precision_at_3_max": 0.373974, + "nauc_precision_at_3_std": 0.616163, + "nauc_precision_at_3_diff1": -0.125103, + "nauc_precision_at_5_max": 0.317393, + "nauc_precision_at_5_std": 0.453637, + "nauc_precision_at_5_diff1": -0.301655, + "nauc_precision_at_10_max": 0.346793, + "nauc_precision_at_10_std": 0.589706, + "nauc_precision_at_10_diff1": -0.421495, + "nauc_precision_at_20_max": 0.339533, + "nauc_precision_at_20_std": 0.666475, + "nauc_precision_at_20_diff1": -0.418728, + "nauc_precision_at_100_max": 0.292942, + "nauc_precision_at_100_std": 0.742772, + "nauc_precision_at_100_diff1": -0.334855, + "nauc_precision_at_1000_max": 0.2712, + "nauc_precision_at_1000_std": 0.677077, + "nauc_precision_at_1000_diff1": -0.222352, + "nauc_mrr_at_1_max": 0.558702, + "nauc_mrr_at_1_std": 0.67915, + "nauc_mrr_at_1_diff1": 0.614293, + "nauc_mrr_at_3_max": 0.705801, + "nauc_mrr_at_3_std": 0.7861, + "nauc_mrr_at_3_diff1": 0.699624, + "nauc_mrr_at_5_max": 0.66054, + "nauc_mrr_at_5_std": 0.753192, + "nauc_mrr_at_5_diff1": 0.673369, + "nauc_mrr_at_10_max": 0.66054, + "nauc_mrr_at_10_std": 0.753192, + "nauc_mrr_at_10_diff1": 0.673369, + "nauc_mrr_at_20_max": 0.66054, + "nauc_mrr_at_20_std": 0.753192, + "nauc_mrr_at_20_diff1": 0.673369, + "nauc_mrr_at_100_max": 0.66054, + "nauc_mrr_at_100_std": 0.753192, + "nauc_mrr_at_100_diff1": 0.673369, + "nauc_mrr_at_1000_max": 0.66054, + "nauc_mrr_at_1000_std": 0.753192, + "nauc_mrr_at_1000_diff1": 0.673369, + "main_score": 0.66652, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5779.706394433975, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/MTOPDomainClassification.json b/mteb/MTOPDomainClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..a2d5ed3d0ed7706d9f7757550b6b18565fadba55 --- /dev/null +++ b/mteb/MTOPDomainClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf", + "task_name": "MTOPDomainClassification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.889124, + "f1": 0.88758, + "f1_weighted": 0.889747, + "scores_per_experiment": [ + { + "accuracy": 0.854537, + "f1": 0.857721, + "f1_weighted": 0.855151 + }, + { + "accuracy": 0.900593, + "f1": 0.89766, + "f1_weighted": 0.901084 + }, + { + "accuracy": 0.901049, + "f1": 0.898648, + "f1_weighted": 0.900877 + }, + { + "accuracy": 0.897629, + "f1": 0.894282, + "f1_weighted": 0.898338 + }, + { + "accuracy": 0.891473, + "f1": 0.890253, + "f1_weighted": 0.892135 + }, + { + "accuracy": 0.882809, + "f1": 0.884257, + "f1_weighted": 0.885064 + }, + { + "accuracy": 0.887597, + "f1": 0.884408, + "f1_weighted": 0.887453 + }, + { + "accuracy": 0.891017, + "f1": 0.887588, + "f1_weighted": 0.890818 + }, + { + "accuracy": 0.891929, + "f1": 0.89255, + "f1_weighted": 0.892768 + }, + { + "accuracy": 0.892613, + "f1": 0.888437, + "f1_weighted": 0.893779 + } + ], + "main_score": 0.889124, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.538909912109375, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/MTOPIntentClassification.json b/mteb/MTOPIntentClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..dba6fdf6af688794239af47f19daf79db27cd0a6 --- /dev/null +++ b/mteb/MTOPIntentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba", + "task_name": "MTOPIntentClassification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.656475, + "f1": 0.490948, + "f1_weighted": 0.693994, + "scores_per_experiment": [ + { + "accuracy": 0.641587, + "f1": 0.456483, + "f1_weighted": 0.683725 + }, + { + "accuracy": 0.671911, + "f1": 0.48881, + "f1_weighted": 0.709701 + }, + { + "accuracy": 0.673279, + "f1": 0.501567, + "f1_weighted": 0.711032 + }, + { + "accuracy": 0.656407, + "f1": 0.514271, + "f1_weighted": 0.692525 + }, + { + "accuracy": 0.639763, + "f1": 0.480901, + "f1_weighted": 0.671092 + }, + { + "accuracy": 0.645691, + "f1": 0.501634, + "f1_weighted": 0.685544 + }, + { + "accuracy": 0.663703, + "f1": 0.479191, + "f1_weighted": 0.704379 + }, + { + "accuracy": 0.674647, + "f1": 0.516005, + "f1_weighted": 0.709525 + }, + { + "accuracy": 0.650935, + "f1": 0.480141, + "f1_weighted": 0.689081 + }, + { + "accuracy": 0.646831, + "f1": 0.490481, + "f1_weighted": 0.683338 + } + ], + "main_score": 0.656475, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.826844215393066, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/MassiveIntentClassification.json b/mteb/MassiveIntentClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..721571e6adc8ed76f47c4f3021351e1ed3a2de0f --- /dev/null +++ b/mteb/MassiveIntentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "task_name": "MassiveIntentClassification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.652354, + "f1": 0.635945, + "f1_weighted": 0.650395, + "scores_per_experiment": [ + { + "accuracy": 0.644923, + "f1": 0.639513, + "f1_weighted": 0.640635 + }, + { + "accuracy": 0.677539, + "f1": 0.652575, + "f1_weighted": 0.67554 + }, + { + "accuracy": 0.649294, + "f1": 0.630992, + "f1_weighted": 0.645129 + }, + { + "accuracy": 0.680565, + "f1": 0.659978, + "f1_weighted": 0.675892 + }, + { + "accuracy": 0.653329, + "f1": 0.637184, + "f1_weighted": 0.649634 + }, + { + "accuracy": 0.641896, + "f1": 0.624381, + "f1_weighted": 0.640526 + }, + { + "accuracy": 0.642233, + "f1": 0.633383, + "f1_weighted": 0.643923 + }, + { + "accuracy": 0.638534, + "f1": 0.618431, + "f1_weighted": 0.633252 + }, + { + "accuracy": 0.64694, + "f1": 0.63405, + "f1_weighted": 0.649656 + }, + { + "accuracy": 0.648285, + "f1": 0.628963, + "f1_weighted": 0.649767 + } + ], + "main_score": 0.652354, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11.81202483177185, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/MassiveScenarioClassification.json b/mteb/MassiveScenarioClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..babf7fc54d8db2f7b4be2f991d2dabb8f0bddf29 --- /dev/null +++ b/mteb/MassiveScenarioClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "task_name": "MassiveScenarioClassification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.726126, + "f1": 0.724117, + "f1_weighted": 0.726333, + "scores_per_experiment": [ + { + "accuracy": 0.726295, + "f1": 0.724368, + "f1_weighted": 0.72364 + }, + { + "accuracy": 0.734701, + "f1": 0.737144, + "f1_weighted": 0.733128 + }, + { + "accuracy": 0.711163, + "f1": 0.711532, + "f1_weighted": 0.709362 + }, + { + "accuracy": 0.705783, + "f1": 0.699515, + "f1_weighted": 0.709819 + }, + { + "accuracy": 0.737727, + "f1": 0.733102, + "f1_weighted": 0.739757 + }, + { + "accuracy": 0.740081, + "f1": 0.729691, + "f1_weighted": 0.74056 + }, + { + "accuracy": 0.72226, + "f1": 0.724384, + "f1_weighted": 0.726585 + }, + { + "accuracy": 0.722596, + "f1": 0.725063, + "f1_weighted": 0.721339 + }, + { + "accuracy": 0.736382, + "f1": 0.733771, + "f1_weighted": 0.736615 + }, + { + "accuracy": 0.724277, + "f1": 0.722599, + "f1_weighted": 0.722527 + } + ], + "main_score": 0.726126, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 4.415860176086426, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/MedrxivClusteringP2P.json b/mteb/MedrxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..4da03e3d05c1b17a76a89bb5d9255c6760bbd8f7 --- /dev/null +++ b/mteb/MedrxivClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "task_name": "MedrxivClusteringP2P", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "v_measure": 0.2895, + "v_measure_std": 0.013911, + "v_measures": [ + 0.271819, + 0.271103, + 0.278713, + 0.272723, + 0.288926, + 0.304441, + 0.300211, + 0.303506, + 0.306792, + 0.296767 + ], + "main_score": 0.2895, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 114.98579716682434, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/MedrxivClusteringS2S.json b/mteb/MedrxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..402d3cfeadeb178dfe54d58849b324f363ecfa06 --- /dev/null +++ b/mteb/MedrxivClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "task_name": "MedrxivClusteringS2S", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "v_measure": 0.27076, + "v_measure_std": 0.017323, + "v_measures": [ + 0.256262, + 0.252559, + 0.253638, + 0.253727, + 0.25903, + 0.301547, + 0.277187, + 0.287709, + 0.292024, + 0.273922 + ], + "main_score": 0.27076, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 17.49012589454651, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/MindSmallReranking.json b/mteb/MindSmallReranking.json new file mode 100644 index 0000000000000000000000000000000000000000..8e10f2e398d07f63fe0c96596c107b10107ce02c --- /dev/null +++ b/mteb/MindSmallReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "59042f120c80e8afa9cdbb224f67076cec0fc9a7", + "task_name": "MindSmallReranking", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "map": 0.306437, + "mrr": 0.316768, + "nAUC_map_max": -0.182556, + "nAUC_map_std": -0.016765, + "nAUC_map_diff1": 0.134889, + "nAUC_mrr_max": -0.128279, + "nAUC_mrr_std": 0.003295, + "nAUC_mrr_diff1": 0.128104, + "main_score": 0.306437, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 120.96634793281555, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/NFCorpus.json b/mteb/NFCorpus.json new file mode 100644 index 0000000000000000000000000000000000000000..d503c7504bd1e7e1dd1953b625928e5c2b680889 --- /dev/null +++ b/mteb/NFCorpus.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814", + "task_name": "NFCorpus", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.34675, + "ndcg_at_3": 0.31048, + "ndcg_at_5": 0.29451, + "ndcg_at_10": 0.2663, + "ndcg_at_20": 0.24402, + "ndcg_at_100": 0.24319, + "ndcg_at_1000": 0.33458, + "map_at_1": 0.04323, + "map_at_3": 0.06706, + "map_at_5": 0.08085, + "map_at_10": 0.09278, + "map_at_20": 0.1011, + "map_at_100": 0.11312, + "map_at_1000": 0.12461, + "recall_at_1": 0.04323, + "recall_at_3": 0.07573, + "recall_at_5": 0.10248, + "recall_at_10": 0.12779, + "recall_at_20": 0.15931, + "recall_at_100": 0.25619, + "recall_at_1000": 0.57667, + "precision_at_1": 0.36223, + "precision_at_3": 0.29102, + "precision_at_5": 0.25573, + "precision_at_10": 0.19505, + "precision_at_20": 0.13824, + "precision_at_100": 0.06037, + "precision_at_1000": 0.0188, + "mrr_at_1": 0.362229, + "mrr_at_3": 0.427245, + "mrr_at_5": 0.443189, + "mrr_at_10": 0.452435, + "mrr_at_20": 0.45666, + "mrr_at_100": 0.459342, + "mrr_at_1000": 0.459762, + "nauc_ndcg_at_1_max": 0.489247, + "nauc_ndcg_at_1_std": 0.207882, + "nauc_ndcg_at_1_diff1": 0.455278, + "nauc_ndcg_at_3_max": 0.484014, + "nauc_ndcg_at_3_std": 0.279633, + "nauc_ndcg_at_3_diff1": 0.373273, + "nauc_ndcg_at_5_max": 0.499214, + "nauc_ndcg_at_5_std": 0.285193, + "nauc_ndcg_at_5_diff1": 0.377699, + "nauc_ndcg_at_10_max": 0.471546, + "nauc_ndcg_at_10_std": 0.303156, + "nauc_ndcg_at_10_diff1": 0.326637, + "nauc_ndcg_at_20_max": 0.452904, + "nauc_ndcg_at_20_std": 0.286146, + "nauc_ndcg_at_20_diff1": 0.311691, + "nauc_ndcg_at_100_max": 0.471783, + "nauc_ndcg_at_100_std": 0.276204, + "nauc_ndcg_at_100_diff1": 0.330981, + "nauc_ndcg_at_1000_max": 0.534989, + "nauc_ndcg_at_1000_std": 0.320777, + "nauc_ndcg_at_1000_diff1": 0.346119, + "nauc_map_at_1_max": 0.328405, + "nauc_map_at_1_std": -0.055909, + "nauc_map_at_1_diff1": 0.464563, + "nauc_map_at_3_max": 0.317683, + "nauc_map_at_3_std": -0.002782, + "nauc_map_at_3_diff1": 0.38562, + "nauc_map_at_5_max": 0.374515, + "nauc_map_at_5_std": 0.023936, + "nauc_map_at_5_diff1": 0.41153, + "nauc_map_at_10_max": 0.397591, + "nauc_map_at_10_std": 0.069354, + "nauc_map_at_10_diff1": 0.375102, + "nauc_map_at_20_max": 0.412683, + "nauc_map_at_20_std": 0.109156, + "nauc_map_at_20_diff1": 0.360845, + "nauc_map_at_100_max": 0.424984, + "nauc_map_at_100_std": 0.159183, + "nauc_map_at_100_diff1": 0.342464, + "nauc_map_at_1000_max": 0.43113, + "nauc_map_at_1000_std": 0.18875, + "nauc_map_at_1000_diff1": 0.333251, + "nauc_recall_at_1_max": 0.328405, + "nauc_recall_at_1_std": -0.055909, + "nauc_recall_at_1_diff1": 0.464563, + "nauc_recall_at_3_max": 0.266527, + "nauc_recall_at_3_std": -0.012704, + "nauc_recall_at_3_diff1": 0.333511, + "nauc_recall_at_5_max": 0.317243, + "nauc_recall_at_5_std": -0.002978, + "nauc_recall_at_5_diff1": 0.38356, + "nauc_recall_at_10_max": 0.307398, + "nauc_recall_at_10_std": 0.039968, + "nauc_recall_at_10_diff1": 0.313532, + "nauc_recall_at_20_max": 0.304815, + "nauc_recall_at_20_std": 0.072316, + "nauc_recall_at_20_diff1": 0.271323, + "nauc_recall_at_100_max": 0.278974, + "nauc_recall_at_100_std": 0.152896, + "nauc_recall_at_100_diff1": 0.216284, + "nauc_recall_at_1000_max": 0.242439, + "nauc_recall_at_1000_std": 0.170621, + "nauc_recall_at_1000_diff1": 0.144589, + "nauc_precision_at_1_max": 0.493121, + "nauc_precision_at_1_std": 0.211866, + "nauc_precision_at_1_diff1": 0.439682, + "nauc_precision_at_3_max": 0.482029, + "nauc_precision_at_3_std": 0.326316, + "nauc_precision_at_3_diff1": 0.30629, + "nauc_precision_at_5_max": 0.501486, + "nauc_precision_at_5_std": 0.359998, + "nauc_precision_at_5_diff1": 0.271906, + "nauc_precision_at_10_max": 0.437322, + "nauc_precision_at_10_std": 0.424984, + "nauc_precision_at_10_diff1": 0.151228, + "nauc_precision_at_20_max": 0.363724, + "nauc_precision_at_20_std": 0.454707, + "nauc_precision_at_20_diff1": 0.083594, + "nauc_precision_at_100_max": 0.213989, + "nauc_precision_at_100_std": 0.452484, + "nauc_precision_at_100_diff1": -0.018369, + "nauc_precision_at_1000_max": 0.062987, + "nauc_precision_at_1000_std": 0.315722, + "nauc_precision_at_1000_diff1": -0.063268, + "nauc_mrr_at_1_max": 0.493121, + "nauc_mrr_at_1_std": 0.211866, + "nauc_mrr_at_1_diff1": 0.439682, + "nauc_mrr_at_3_max": 0.502532, + "nauc_mrr_at_3_std": 0.249172, + "nauc_mrr_at_3_diff1": 0.410622, + "nauc_mrr_at_5_max": 0.515072, + "nauc_mrr_at_5_std": 0.250577, + "nauc_mrr_at_5_diff1": 0.409294, + "nauc_mrr_at_10_max": 0.510025, + "nauc_mrr_at_10_std": 0.25547, + "nauc_mrr_at_10_diff1": 0.404908, + "nauc_mrr_at_20_max": 0.510314, + "nauc_mrr_at_20_std": 0.258227, + "nauc_mrr_at_20_diff1": 0.404249, + "nauc_mrr_at_100_max": 0.511729, + "nauc_mrr_at_100_std": 0.258183, + "nauc_mrr_at_100_diff1": 0.405071, + "nauc_mrr_at_1000_max": 0.511509, + "nauc_mrr_at_1000_std": 0.257945, + "nauc_mrr_at_1000_diff1": 0.405027, + "main_score": 0.2663, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.661004543304443, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/NQ.json b/mteb/NQ.json new file mode 100644 index 0000000000000000000000000000000000000000..129c5ebcfad2efa4dad322e6745864f8f9552936 --- /dev/null +++ b/mteb/NQ.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31", + "task_name": "NQ", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.28708, + "ndcg_at_3": 0.38322, + "ndcg_at_5": 0.42087, + "ndcg_at_10": 0.4532, + "ndcg_at_20": 0.4742, + "ndcg_at_100": 0.49995, + "ndcg_at_1000": 0.51263, + "map_at_1": 0.25702, + "map_at_3": 0.34911, + "map_at_5": 0.37137, + "map_at_10": 0.38587, + "map_at_20": 0.39226, + "map_at_100": 0.39638, + "map_at_1000": 0.39689, + "recall_at_1": 0.25702, + "recall_at_3": 0.45394, + "recall_at_5": 0.54089, + "recall_at_10": 0.63504, + "recall_at_20": 0.71323, + "recall_at_100": 0.84154, + "recall_at_1000": 0.93687, + "precision_at_1": 0.28708, + "precision_at_3": 0.17468, + "precision_at_5": 0.1263, + "precision_at_10": 0.07512, + "precision_at_20": 0.0425, + "precision_at_100": 0.01018, + "precision_at_1000": 0.00114, + "mrr_at_1": 0.28708, + "mrr_at_3": 0.377221, + "mrr_at_5": 0.396833, + "mrr_at_10": 0.409104, + "mrr_at_20": 0.41419, + "mrr_at_100": 0.417396, + "mrr_at_1000": 0.417789, + "nauc_ndcg_at_1_max": 0.250479, + "nauc_ndcg_at_1_std": 0.019234, + "nauc_ndcg_at_1_diff1": 0.335807, + "nauc_ndcg_at_3_max": 0.294583, + "nauc_ndcg_at_3_std": 0.050666, + "nauc_ndcg_at_3_diff1": 0.293529, + "nauc_ndcg_at_5_max": 0.307405, + "nauc_ndcg_at_5_std": 0.060516, + "nauc_ndcg_at_5_diff1": 0.287018, + "nauc_ndcg_at_10_max": 0.320593, + "nauc_ndcg_at_10_std": 0.084058, + "nauc_ndcg_at_10_diff1": 0.287482, + "nauc_ndcg_at_20_max": 0.328514, + "nauc_ndcg_at_20_std": 0.096083, + "nauc_ndcg_at_20_diff1": 0.286127, + "nauc_ndcg_at_100_max": 0.324329, + "nauc_ndcg_at_100_std": 0.105305, + "nauc_ndcg_at_100_diff1": 0.283013, + "nauc_ndcg_at_1000_max": 0.318549, + "nauc_ndcg_at_1000_std": 0.097215, + "nauc_ndcg_at_1000_diff1": 0.286643, + "nauc_map_at_1_max": 0.242088, + "nauc_map_at_1_std": 0.00276, + "nauc_map_at_1_diff1": 0.340467, + "nauc_map_at_3_max": 0.282923, + "nauc_map_at_3_std": 0.036139, + "nauc_map_at_3_diff1": 0.305555, + "nauc_map_at_5_max": 0.291093, + "nauc_map_at_5_std": 0.042332, + "nauc_map_at_5_diff1": 0.302144, + "nauc_map_at_10_max": 0.29688, + "nauc_map_at_10_std": 0.053188, + "nauc_map_at_10_diff1": 0.301168, + "nauc_map_at_20_max": 0.299107, + "nauc_map_at_20_std": 0.0566, + "nauc_map_at_20_diff1": 0.301049, + "nauc_map_at_100_max": 0.298118, + "nauc_map_at_100_std": 0.057722, + "nauc_map_at_100_diff1": 0.300482, + "nauc_map_at_1000_max": 0.297934, + "nauc_map_at_1000_std": 0.057475, + "nauc_map_at_1000_diff1": 0.300611, + "nauc_recall_at_1_max": 0.242088, + "nauc_recall_at_1_std": 0.00276, + "nauc_recall_at_1_diff1": 0.340467, + "nauc_recall_at_3_max": 0.314631, + "nauc_recall_at_3_std": 0.072352, + "nauc_recall_at_3_diff1": 0.260977, + "nauc_recall_at_5_max": 0.343255, + "nauc_recall_at_5_std": 0.094105, + "nauc_recall_at_5_diff1": 0.242641, + "nauc_recall_at_10_max": 0.389135, + "nauc_recall_at_10_std": 0.170459, + "nauc_recall_at_10_diff1": 0.240436, + "nauc_recall_at_20_max": 0.440327, + "nauc_recall_at_20_std": 0.243425, + "nauc_recall_at_20_diff1": 0.225185, + "nauc_recall_at_100_max": 0.488993, + "nauc_recall_at_100_std": 0.423469, + "nauc_recall_at_100_diff1": 0.170894, + "nauc_recall_at_1000_max": 0.535473, + "nauc_recall_at_1000_std": 0.585053, + "nauc_recall_at_1000_diff1": 0.161342, + "nauc_precision_at_1_max": 0.250479, + "nauc_precision_at_1_std": 0.019234, + "nauc_precision_at_1_diff1": 0.335807, + "nauc_precision_at_3_max": 0.312609, + "nauc_precision_at_3_std": 0.098992, + "nauc_precision_at_3_diff1": 0.232633, + "nauc_precision_at_5_max": 0.321341, + "nauc_precision_at_5_std": 0.121296, + "nauc_precision_at_5_diff1": 0.194865, + "nauc_precision_at_10_max": 0.316709, + "nauc_precision_at_10_std": 0.18397, + "nauc_precision_at_10_diff1": 0.157855, + "nauc_precision_at_20_max": 0.310475, + "nauc_precision_at_20_std": 0.225964, + "nauc_precision_at_20_diff1": 0.117234, + "nauc_precision_at_100_max": 0.217555, + "nauc_precision_at_100_std": 0.269504, + "nauc_precision_at_100_diff1": 0.029149, + "nauc_precision_at_1000_max": 0.094419, + "nauc_precision_at_1000_std": 0.203673, + "nauc_precision_at_1000_diff1": -0.029269, + "nauc_mrr_at_1_max": 0.250479, + "nauc_mrr_at_1_std": 0.019234, + "nauc_mrr_at_1_diff1": 0.335807, + "nauc_mrr_at_3_max": 0.285525, + "nauc_mrr_at_3_std": 0.053665, + "nauc_mrr_at_3_diff1": 0.296797, + "nauc_mrr_at_5_max": 0.290497, + "nauc_mrr_at_5_std": 0.058275, + "nauc_mrr_at_5_diff1": 0.293153, + "nauc_mrr_at_10_max": 0.29465, + "nauc_mrr_at_10_std": 0.065692, + "nauc_mrr_at_10_diff1": 0.295064, + "nauc_mrr_at_20_max": 0.295426, + "nauc_mrr_at_20_std": 0.066635, + "nauc_mrr_at_20_diff1": 0.295253, + "nauc_mrr_at_100_max": 0.294819, + "nauc_mrr_at_100_std": 0.067348, + "nauc_mrr_at_100_diff1": 0.295063, + "nauc_mrr_at_1000_max": 0.29466, + "nauc_mrr_at_1000_std": 0.067075, + "nauc_mrr_at_1000_diff1": 0.295171, + "main_score": 0.4532, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2530.8694610595703, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/QuoraRetrieval.json b/mteb/QuoraRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..5616819ca4cf7cb035fdd3266e71157250b2c502 --- /dev/null +++ b/mteb/QuoraRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e4e08e0b7dbe3c8700f0daef558ff32256715259", + "task_name": "QuoraRetrieval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.7981, + "ndcg_at_3": 0.83923, + "ndcg_at_5": 0.85516, + "ndcg_at_10": 0.86825, + "ndcg_at_20": 0.87562, + "ndcg_at_100": 0.88241, + "ndcg_at_1000": 0.88406, + "map_at_1": 0.69387, + "map_at_3": 0.80109, + "map_at_5": 0.81907, + "map_at_10": 0.82976, + "map_at_20": 0.83391, + "map_at_100": 0.83627, + "map_at_1000": 0.83648, + "recall_at_1": 0.69387, + "recall_at_3": 0.85762, + "recall_at_5": 0.90305, + "recall_at_10": 0.94156, + "recall_at_20": 0.9656, + "recall_at_100": 0.99099, + "recall_at_1000": 0.99905, + "precision_at_1": 0.7981, + "precision_at_3": 0.3661, + "precision_at_5": 0.24042, + "precision_at_10": 0.13102, + "precision_at_20": 0.06961, + "precision_at_100": 0.0151, + "precision_at_1000": 0.00156, + "mrr_at_1": 0.7979, + "mrr_at_3": 0.8504, + "mrr_at_5": 0.85789, + "mrr_at_10": 0.861217, + "mrr_at_20": 0.862272, + "mrr_at_100": 0.862594, + "mrr_at_1000": 0.862613, + "nauc_ndcg_at_1_max": 0.385323, + "nauc_ndcg_at_1_std": -0.352997, + "nauc_ndcg_at_1_diff1": 0.768904, + "nauc_ndcg_at_3_max": 0.358767, + "nauc_ndcg_at_3_std": -0.422362, + "nauc_ndcg_at_3_diff1": 0.750812, + "nauc_ndcg_at_5_max": 0.365901, + "nauc_ndcg_at_5_std": -0.437432, + "nauc_ndcg_at_5_diff1": 0.756063, + "nauc_ndcg_at_10_max": 0.371952, + "nauc_ndcg_at_10_std": -0.432721, + "nauc_ndcg_at_10_diff1": 0.758216, + "nauc_ndcg_at_20_max": 0.375014, + "nauc_ndcg_at_20_std": -0.419042, + "nauc_ndcg_at_20_diff1": 0.75704, + "nauc_ndcg_at_100_max": 0.380351, + "nauc_ndcg_at_100_std": -0.39988, + "nauc_ndcg_at_100_diff1": 0.756946, + "nauc_ndcg_at_1000_max": 0.380961, + "nauc_ndcg_at_1000_std": -0.395691, + "nauc_ndcg_at_1000_diff1": 0.756467, + "nauc_map_at_1_max": 0.276589, + "nauc_map_at_1_std": -0.386526, + "nauc_map_at_1_diff1": 0.7886, + "nauc_map_at_3_max": 0.332655, + "nauc_map_at_3_std": -0.448285, + "nauc_map_at_3_diff1": 0.762881, + "nauc_map_at_5_max": 0.348765, + "nauc_map_at_5_std": -0.448588, + "nauc_map_at_5_diff1": 0.761948, + "nauc_map_at_10_max": 0.357223, + "nauc_map_at_10_std": -0.437594, + "nauc_map_at_10_diff1": 0.760072, + "nauc_map_at_20_max": 0.360747, + "nauc_map_at_20_std": -0.427652, + "nauc_map_at_20_diff1": 0.75873, + "nauc_map_at_100_max": 0.363031, + "nauc_map_at_100_std": -0.420231, + "nauc_map_at_100_diff1": 0.758438, + "nauc_map_at_1000_max": 0.363384, + "nauc_map_at_1000_std": -0.41938, + "nauc_map_at_1000_diff1": 0.758393, + "nauc_recall_at_1_max": 0.276589, + "nauc_recall_at_1_std": -0.386526, + "nauc_recall_at_1_diff1": 0.7886, + "nauc_recall_at_3_max": 0.29953, + "nauc_recall_at_3_std": -0.507123, + "nauc_recall_at_3_diff1": 0.723936, + "nauc_recall_at_5_max": 0.318398, + "nauc_recall_at_5_std": -0.564493, + "nauc_recall_at_5_diff1": 0.717385, + "nauc_recall_at_10_max": 0.330489, + "nauc_recall_at_10_std": -0.596467, + "nauc_recall_at_10_diff1": 0.714574, + "nauc_recall_at_20_max": 0.34185, + "nauc_recall_at_20_std": -0.570183, + "nauc_recall_at_20_diff1": 0.707112, + "nauc_recall_at_100_max": 0.426654, + "nauc_recall_at_100_std": -0.408769, + "nauc_recall_at_100_diff1": 0.720717, + "nauc_recall_at_1000_max": 0.441165, + "nauc_recall_at_1000_std": 0.174422, + "nauc_recall_at_1000_diff1": 0.582066, + "nauc_precision_at_1_max": 0.385323, + "nauc_precision_at_1_std": -0.352997, + "nauc_precision_at_1_diff1": 0.768904, + "nauc_precision_at_3_max": 0.115283, + "nauc_precision_at_3_std": 0.034073, + "nauc_precision_at_3_diff1": -0.11805, + "nauc_precision_at_5_max": 0.067543, + "nauc_precision_at_5_std": 0.137509, + "nauc_precision_at_5_diff1": -0.254885, + "nauc_precision_at_10_max": 0.025493, + "nauc_precision_at_10_std": 0.236938, + "nauc_precision_at_10_diff1": -0.342161, + "nauc_precision_at_20_max": -0.001939, + "nauc_precision_at_20_std": 0.310031, + "nauc_precision_at_20_diff1": -0.388267, + "nauc_precision_at_100_max": -0.02205, + "nauc_precision_at_100_std": 0.38886, + "nauc_precision_at_100_diff1": -0.41661, + "nauc_precision_at_1000_max": -0.027452, + "nauc_precision_at_1000_std": 0.411492, + "nauc_precision_at_1000_diff1": -0.424162, + "nauc_mrr_at_1_max": 0.383192, + "nauc_mrr_at_1_std": -0.353824, + "nauc_mrr_at_1_diff1": 0.769376, + "nauc_mrr_at_3_max": 0.386992, + "nauc_mrr_at_3_std": -0.374313, + "nauc_mrr_at_3_diff1": 0.758136, + "nauc_mrr_at_5_max": 0.389249, + "nauc_mrr_at_5_std": -0.377732, + "nauc_mrr_at_5_diff1": 0.760258, + "nauc_mrr_at_10_max": 0.390148, + "nauc_mrr_at_10_std": -0.375749, + "nauc_mrr_at_10_diff1": 0.761165, + "nauc_mrr_at_20_max": 0.389778, + "nauc_mrr_at_20_std": -0.373585, + "nauc_mrr_at_20_diff1": 0.761091, + "nauc_mrr_at_100_max": 0.389507, + "nauc_mrr_at_100_std": -0.373448, + "nauc_mrr_at_100_diff1": 0.761166, + "nauc_mrr_at_1000_max": 0.389482, + "nauc_mrr_at_1000_std": -0.373441, + "nauc_mrr_at_1000_diff1": 0.76116, + "main_score": 0.86825, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 304.4700849056244, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/RedditClustering.json b/mteb/RedditClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..c68274e1dc54f8f9e6f90fd5c5ce154ffb840870 --- /dev/null +++ b/mteb/RedditClustering.json @@ -0,0 +1,47 @@ +{ + "dataset_revision": "24640382cdbf8abc73003fb0fa6d111a705499eb", + "task_name": "RedditClustering", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "v_measure": 0.411257, + "v_measure_std": 0.039637, + "v_measures": [ + 0.47826, + 0.463098, + 0.378164, + 0.408031, + 0.375972, + 0.408828, + 0.442864, + 0.366117, + 0.364104, + 0.38536, + 0.404472, + 0.412944, + 0.391103, + 0.42435, + 0.401238, + 0.36973, + 0.433922, + 0.482263, + 0.399745, + 0.386039, + 0.37345, + 0.392968, + 0.523073, + 0.426964, + 0.388366 + ], + "main_score": 0.411257, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 178.5294632911682, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/RedditClusteringP2P.json b/mteb/RedditClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..11c619d8ad5d1eed5f3c3ee9d98164686c15eb43 --- /dev/null +++ b/mteb/RedditClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "385e3cb46b4cfa89021f56c4380204149d0efe33", + "task_name": "RedditClusteringP2P", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "v_measure": 0.513872, + "v_measure_std": 0.116705, + "v_measures": [ + 0.563847, + 0.569759, + 0.580057, + 0.339426, + 0.569614, + 0.511353, + 0.24021, + 0.608951, + 0.56097, + 0.594533 + ], + "main_score": 0.513872, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 662.1253287792206, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/SCIDOCS.json b/mteb/SCIDOCS.json new file mode 100644 index 0000000000000000000000000000000000000000..1ceb7e42420b27ddc8acd0164972514cc1c66c8b --- /dev/null +++ b/mteb/SCIDOCS.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88", + "task_name": "SCIDOCS", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.173, + "ndcg_at_3": 0.13497, + "ndcg_at_5": 0.11591, + "ndcg_at_10": 0.13679, + "ndcg_at_20": 0.15566, + "ndcg_at_100": 0.19218, + "ndcg_at_1000": 0.23768, + "map_at_1": 0.0349, + "map_at_3": 0.05844, + "map_at_5": 0.069, + "map_at_10": 0.07798, + "map_at_20": 0.08404, + "map_at_100": 0.09027, + "map_at_1000": 0.09238, + "recall_at_1": 0.0349, + "recall_at_3": 0.0747, + "recall_at_5": 0.10045, + "recall_at_10": 0.1389, + "recall_at_20": 0.18338, + "recall_at_100": 0.3019, + "recall_at_1000": 0.52682, + "precision_at_1": 0.173, + "precision_at_3": 0.12333, + "precision_at_5": 0.0992, + "precision_at_10": 0.0685, + "precision_at_20": 0.0452, + "precision_at_100": 0.01486, + "precision_at_1000": 0.00259, + "mrr_at_1": 0.173, + "mrr_at_3": 0.231667, + "mrr_at_5": 0.244817, + "mrr_at_10": 0.256625, + "mrr_at_20": 0.262569, + "mrr_at_100": 0.267332, + "mrr_at_1000": 0.268125, + "nauc_ndcg_at_1_max": 0.242005, + "nauc_ndcg_at_1_std": 0.119834, + "nauc_ndcg_at_1_diff1": 0.24468, + "nauc_ndcg_at_3_max": 0.29079, + "nauc_ndcg_at_3_std": 0.16193, + "nauc_ndcg_at_3_diff1": 0.212417, + "nauc_ndcg_at_5_max": 0.302191, + "nauc_ndcg_at_5_std": 0.183763, + "nauc_ndcg_at_5_diff1": 0.194039, + "nauc_ndcg_at_10_max": 0.307147, + "nauc_ndcg_at_10_std": 0.215882, + "nauc_ndcg_at_10_diff1": 0.177705, + "nauc_ndcg_at_20_max": 0.311477, + "nauc_ndcg_at_20_std": 0.2469, + "nauc_ndcg_at_20_diff1": 0.169089, + "nauc_ndcg_at_100_max": 0.321562, + "nauc_ndcg_at_100_std": 0.30425, + "nauc_ndcg_at_100_diff1": 0.171303, + "nauc_ndcg_at_1000_max": 0.324754, + "nauc_ndcg_at_1000_std": 0.325746, + "nauc_ndcg_at_1000_diff1": 0.174281, + "nauc_map_at_1_max": 0.244036, + "nauc_map_at_1_std": 0.122252, + "nauc_map_at_1_diff1": 0.245346, + "nauc_map_at_3_max": 0.30043, + "nauc_map_at_3_std": 0.162703, + "nauc_map_at_3_diff1": 0.218286, + "nauc_map_at_5_max": 0.315919, + "nauc_map_at_5_std": 0.186765, + "nauc_map_at_5_diff1": 0.191174, + "nauc_map_at_10_max": 0.321404, + "nauc_map_at_10_std": 0.210341, + "nauc_map_at_10_diff1": 0.183068, + "nauc_map_at_20_max": 0.325777, + "nauc_map_at_20_std": 0.230543, + "nauc_map_at_20_diff1": 0.175999, + "nauc_map_at_100_max": 0.32757, + "nauc_map_at_100_std": 0.252143, + "nauc_map_at_100_diff1": 0.175648, + "nauc_map_at_1000_max": 0.32836, + "nauc_map_at_1000_std": 0.256243, + "nauc_map_at_1000_diff1": 0.176171, + "nauc_recall_at_1_max": 0.244036, + "nauc_recall_at_1_std": 0.122252, + "nauc_recall_at_1_diff1": 0.245346, + "nauc_recall_at_3_max": 0.304984, + "nauc_recall_at_3_std": 0.17664, + "nauc_recall_at_3_diff1": 0.193246, + "nauc_recall_at_5_max": 0.308366, + "nauc_recall_at_5_std": 0.203564, + "nauc_recall_at_5_diff1": 0.156459, + "nauc_recall_at_10_max": 0.303941, + "nauc_recall_at_10_std": 0.251872, + "nauc_recall_at_10_diff1": 0.122576, + "nauc_recall_at_20_max": 0.294931, + "nauc_recall_at_20_std": 0.298445, + "nauc_recall_at_20_diff1": 0.102911, + "nauc_recall_at_100_max": 0.295687, + "nauc_recall_at_100_std": 0.404675, + "nauc_recall_at_100_diff1": 0.108211, + "nauc_recall_at_1000_max": 0.261128, + "nauc_recall_at_1000_std": 0.42622, + "nauc_recall_at_1000_diff1": 0.095162, + "nauc_precision_at_1_max": 0.242005, + "nauc_precision_at_1_std": 0.119834, + "nauc_precision_at_1_diff1": 0.24468, + "nauc_precision_at_3_max": 0.305687, + "nauc_precision_at_3_std": 0.175571, + "nauc_precision_at_3_diff1": 0.193951, + "nauc_precision_at_5_max": 0.310911, + "nauc_precision_at_5_std": 0.203866, + "nauc_precision_at_5_diff1": 0.160506, + "nauc_precision_at_10_max": 0.310221, + "nauc_precision_at_10_std": 0.253927, + "nauc_precision_at_10_diff1": 0.128608, + "nauc_precision_at_20_max": 0.300773, + "nauc_precision_at_20_std": 0.299155, + "nauc_precision_at_20_diff1": 0.108257, + "nauc_precision_at_100_max": 0.299943, + "nauc_precision_at_100_std": 0.4021, + "nauc_precision_at_100_diff1": 0.112121, + "nauc_precision_at_1000_max": 0.263822, + "nauc_precision_at_1000_std": 0.417393, + "nauc_precision_at_1000_diff1": 0.099425, + "nauc_mrr_at_1_max": 0.242005, + "nauc_mrr_at_1_std": 0.119834, + "nauc_mrr_at_1_diff1": 0.24468, + "nauc_mrr_at_3_max": 0.265063, + "nauc_mrr_at_3_std": 0.146784, + "nauc_mrr_at_3_diff1": 0.224465, + "nauc_mrr_at_5_max": 0.270678, + "nauc_mrr_at_5_std": 0.151935, + "nauc_mrr_at_5_diff1": 0.226012, + "nauc_mrr_at_10_max": 0.269741, + "nauc_mrr_at_10_std": 0.159622, + "nauc_mrr_at_10_diff1": 0.217963, + "nauc_mrr_at_20_max": 0.270563, + "nauc_mrr_at_20_std": 0.162419, + "nauc_mrr_at_20_diff1": 0.217493, + "nauc_mrr_at_100_max": 0.270795, + "nauc_mrr_at_100_std": 0.164246, + "nauc_mrr_at_100_diff1": 0.218387, + "nauc_mrr_at_1000_max": 0.270832, + "nauc_mrr_at_1000_std": 0.164069, + "nauc_mrr_at_1000_diff1": 0.218429, + "main_score": 0.13679, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 46.585543632507324, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/SICK-R.json b/mteb/SICK-R.json new file mode 100644 index 0000000000000000000000000000000000000000..6b14ef521d1d758d49b92b644cfe26e822691b46 --- /dev/null +++ b/mteb/SICK-R.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "20a6d6f312dd54037fe07a32d58e5e168867909d", + "task_name": "SICK-R", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "pearson": 0.782689, + "spearman": 0.709342, + "cosine_pearson": 0.782689, + "cosine_spearman": 0.709342, + "manhattan_pearson": 0.747536, + "manhattan_spearman": 0.708905, + "euclidean_pearson": 0.747187, + "euclidean_spearman": 0.70888, + "main_score": 0.709342, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 8.33829402923584, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/STS12.json b/mteb/STS12.json new file mode 100644 index 0000000000000000000000000000000000000000..cea56abd39aaed17352987088969da791b9c73cc --- /dev/null +++ b/mteb/STS12.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", + "task_name": "STS12", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "pearson": 0.733449, + "spearman": 0.672036, + "cosine_pearson": 0.733449, + "cosine_spearman": 0.672036, + "manhattan_pearson": 0.685626, + "manhattan_spearman": 0.668326, + "euclidean_pearson": 0.684606, + "euclidean_spearman": 0.668238, + "main_score": 0.672036, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.7828357219696045, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/STS13.json b/mteb/STS13.json new file mode 100644 index 0000000000000000000000000000000000000000..e42c26bb58eb0de9bf0694cb31b1371460fde040 --- /dev/null +++ b/mteb/STS13.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca", + "task_name": "STS13", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "pearson": 0.773137, + "spearman": 0.774032, + "cosine_pearson": 0.773137, + "cosine_spearman": 0.774032, + "manhattan_pearson": 0.762551, + "manhattan_spearman": 0.766366, + "euclidean_pearson": 0.761261, + "euclidean_spearman": 0.765129, + "main_score": 0.774032, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.3985404968261719, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/STS14.json b/mteb/STS14.json new file mode 100644 index 0000000000000000000000000000000000000000..356c236d3b1b8229f87f9e0c26d9939be100ab89 --- /dev/null +++ b/mteb/STS14.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", + "task_name": "STS14", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "pearson": 0.771255, + "spearman": 0.723476, + "cosine_pearson": 0.771255, + "cosine_spearman": 0.723476, + "manhattan_pearson": 0.741639, + "manhattan_spearman": 0.721268, + "euclidean_pearson": 0.74118, + "euclidean_spearman": 0.721061, + "main_score": 0.723476, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.2774949073791504, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/STS15.json b/mteb/STS15.json new file mode 100644 index 0000000000000000000000000000000000000000..17dc37d112b4608be03528a3d757be358e78dc4c --- /dev/null +++ b/mteb/STS15.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3", + "task_name": "STS15", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "pearson": 0.81992, + "spearman": 0.824958, + "cosine_pearson": 0.81992, + "cosine_spearman": 0.824958, + "manhattan_pearson": 0.813437, + "manhattan_spearman": 0.819727, + "euclidean_pearson": 0.813543, + "euclidean_spearman": 0.819557, + "main_score": 0.824958, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.9901087284088135, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/STS16.json b/mteb/STS16.json new file mode 100644 index 0000000000000000000000000000000000000000..63e490907e97f95ceb7c88d6b7630b3e47842f5c --- /dev/null +++ b/mteb/STS16.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "4d8694f8f0e0100860b497b999b3dbed754a0513", + "task_name": "STS16", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "pearson": 0.78755, + "spearman": 0.79408, + "cosine_pearson": 0.78755, + "cosine_spearman": 0.79408, + "manhattan_pearson": 0.800336, + "manhattan_spearman": 0.805912, + "euclidean_pearson": 0.800315, + "euclidean_spearman": 0.805413, + "main_score": 0.79408, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.2775132656097412, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/STS17.json b/mteb/STS17.json new file mode 100644 index 0000000000000000000000000000000000000000..82ff0cb0af8949320896fd4a1ea3cdaf1cd58f93 --- /dev/null +++ b/mteb/STS17.json @@ -0,0 +1,138 @@ +{ + "dataset_revision": "faeb762787bd10488a50c8b5be4a3b82e411949c", + "task_name": "STS17", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "pearson": 0.412807, + "spearman": 0.399908, + "cosine_pearson": 0.412807, + "cosine_spearman": 0.399908, + "manhattan_pearson": 0.41515, + "manhattan_spearman": 0.396907, + "euclidean_pearson": 0.415204, + "euclidean_spearman": 0.396877, + "main_score": 0.399908, + "hf_subset": "fr-en", + "languages": [ + "fra-Latn", + "eng-Latn" + ] + }, + { + "pearson": 0.843554, + "spearman": 0.847607, + "cosine_pearson": 0.843554, + "cosine_spearman": 0.847607, + "manhattan_pearson": 0.841083, + "manhattan_spearman": 0.843598, + "euclidean_pearson": 0.841249, + "euclidean_spearman": 0.843307, + "main_score": 0.847607, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ] + }, + { + "pearson": 0.360455, + "spearman": 0.369017, + "cosine_pearson": 0.360455, + "cosine_spearman": 0.369017, + "manhattan_pearson": 0.331281, + "manhattan_spearman": 0.339834, + "euclidean_pearson": 0.334346, + "euclidean_spearman": 0.346078, + "main_score": 0.369017, + "hf_subset": "es-en", + "languages": [ + "spa-Latn", + "eng-Latn" + ] + }, + { + "pearson": 0.383329, + "spearman": 0.368844, + "cosine_pearson": 0.38333, + "cosine_spearman": 0.368844, + "manhattan_pearson": 0.347257, + "manhattan_spearman": 0.340722, + "euclidean_pearson": 0.350828, + "euclidean_spearman": 0.341548, + "main_score": 0.368844, + "hf_subset": "en-de", + "languages": [ + "eng-Latn", + "deu-Latn" + ] + }, + { + "pearson": 0.085032, + "spearman": 0.056236, + "cosine_pearson": 0.085032, + "cosine_spearman": 0.056236, + "manhattan_pearson": 0.117706, + "manhattan_spearman": 0.085126, + "euclidean_pearson": 0.116449, + "euclidean_spearman": 0.077363, + "main_score": 0.056236, + "hf_subset": "en-tr", + "languages": [ + "eng-Latn", + "tur-Latn" + ] + }, + { + "pearson": 0.307729, + "spearman": 0.278259, + "cosine_pearson": 0.307729, + "cosine_spearman": 0.278259, + "manhattan_pearson": 0.341556, + "manhattan_spearman": 0.328526, + "euclidean_pearson": 0.338333, + "euclidean_spearman": 0.328262, + "main_score": 0.278259, + "hf_subset": "it-en", + "languages": [ + "ita-Latn", + "eng-Latn" + ] + }, + { + "pearson": 0.374576, + "spearman": 0.33533, + "cosine_pearson": 0.374575, + "cosine_spearman": 0.335406, + "manhattan_pearson": 0.387747, + "manhattan_spearman": 0.368791, + "euclidean_pearson": 0.394756, + "euclidean_spearman": 0.383307, + "main_score": 0.335406, + "hf_subset": "nl-en", + "languages": [ + "nld-Latn", + "eng-Latn" + ] + }, + { + "pearson": 0.128259, + "spearman": 0.132374, + "cosine_pearson": 0.128259, + "cosine_spearman": 0.132374, + "manhattan_pearson": 0.160609, + "manhattan_spearman": 0.148943, + "euclidean_pearson": 0.141566, + "euclidean_spearman": 0.126913, + "main_score": 0.132374, + "hf_subset": "en-ar", + "languages": [ + "eng-Latn", + "ara-Arab" + ] + } + ] + }, + "evaluation_time": 2.167579174041748, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/STS22.json b/mteb/STS22.json new file mode 100644 index 0000000000000000000000000000000000000000..2a2a04f9de05f7ce75015338fcd3dafba6b7e659 --- /dev/null +++ b/mteb/STS22.json @@ -0,0 +1,90 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "task_name": "STS22", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "pearson": 0.661988, + "spearman": 0.666842, + "cosine_pearson": 0.661988, + "cosine_spearman": 0.666842, + "manhattan_pearson": 0.667457, + "manhattan_spearman": 0.66002, + "euclidean_pearson": 0.669588, + "euclidean_spearman": 0.661658, + "main_score": 0.666842, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "pearson": 0.55526, + "spearman": 0.587906, + "cosine_pearson": 0.55526, + "cosine_spearman": 0.587906, + "manhattan_pearson": 0.579185, + "manhattan_spearman": 0.604479, + "euclidean_pearson": 0.580086, + "euclidean_spearman": 0.605008, + "main_score": 0.587906, + "hf_subset": "es-en", + "languages": [ + "spa-Latn", + "eng-Latn" + ] + }, + { + "pearson": 0.363507, + "spearman": 0.399716, + "cosine_pearson": 0.363506, + "cosine_spearman": 0.399716, + "manhattan_pearson": 0.253105, + "manhattan_spearman": 0.281907, + "euclidean_pearson": 0.278303, + "euclidean_spearman": 0.298408, + "main_score": 0.399716, + "hf_subset": "zh-en", + "languages": [ + "cmn-Hans", + "eng-Latn" + ] + }, + { + "pearson": 0.477598, + "spearman": 0.490005, + "cosine_pearson": 0.477598, + "cosine_spearman": 0.490005, + "manhattan_pearson": 0.462315, + "manhattan_spearman": 0.499363, + "euclidean_pearson": 0.468186, + "euclidean_spearman": 0.507778, + "main_score": 0.490005, + "hf_subset": "de-en", + "languages": [ + "deu-Latn", + "eng-Latn" + ] + }, + { + "pearson": 0.492874, + "spearman": 0.435968, + "cosine_pearson": 0.492874, + "cosine_spearman": 0.435968, + "manhattan_pearson": 0.560733, + "manhattan_spearman": 0.510045, + "euclidean_pearson": 0.563564, + "euclidean_spearman": 0.499678, + "main_score": 0.435968, + "hf_subset": "pl-en", + "languages": [ + "pol-Latn", + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11.842099905014038, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/STSBenchmark.json b/mteb/STSBenchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..89b7cce0a9d9ad0a1b1191945348e862ed544022 --- /dev/null +++ b/mteb/STSBenchmark.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831", + "task_name": "STSBenchmark", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "pearson": 0.786387, + "spearman": 0.774847, + "cosine_pearson": 0.786387, + "cosine_spearman": 0.774847, + "manhattan_pearson": 0.775199, + "manhattan_spearman": 0.771479, + "euclidean_pearson": 0.775843, + "euclidean_spearman": 0.771979, + "main_score": 0.774847, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.3312280178070068, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/SciDocsRR.json b/mteb/SciDocsRR.json new file mode 100644 index 0000000000000000000000000000000000000000..6910cf649ffd2fb87ae333c98083faaba739fe25 --- /dev/null +++ b/mteb/SciDocsRR.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab", + "task_name": "SciDocsRR", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "map": 0.724633, + "mrr": 0.907647, + "nAUC_map_max": 0.51911, + "nAUC_map_std": 0.610636, + "nAUC_map_diff1": 0.108873, + "nAUC_mrr_max": 0.740887, + "nAUC_mrr_std": 0.663654, + "nAUC_mrr_diff1": 0.481591, + "main_score": 0.724633, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 38.36741638183594, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/SciFact.json b/mteb/SciFact.json new file mode 100644 index 0000000000000000000000000000000000000000..10f5145c972b57415fcc765ae2108fa54d846393 --- /dev/null +++ b/mteb/SciFact.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "0228b52cf27578f30900b9e5271d331663a030d7", + "task_name": "SciFact", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.49, + "ndcg_at_3": 0.55969, + "ndcg_at_5": 0.58111, + "ndcg_at_10": 0.60427, + "ndcg_at_20": 0.61527, + "ndcg_at_100": 0.63535, + "ndcg_at_1000": 0.64773, + "map_at_1": 0.46344, + "map_at_3": 0.53188, + "map_at_5": 0.54567, + "map_at_10": 0.55704, + "map_at_20": 0.56055, + "map_at_100": 0.5635, + "map_at_1000": 0.56397, + "recall_at_1": 0.46344, + "recall_at_3": 0.60911, + "recall_at_5": 0.66294, + "recall_at_10": 0.72872, + "recall_at_20": 0.77033, + "recall_at_100": 0.87656, + "recall_at_1000": 0.97333, + "precision_at_1": 0.49, + "precision_at_3": 0.22111, + "precision_at_5": 0.146, + "precision_at_10": 0.082, + "precision_at_20": 0.04383, + "precision_at_100": 0.00997, + "precision_at_1000": 0.0011, + "mrr_at_1": 0.49, + "mrr_at_3": 0.555556, + "mrr_at_5": 0.568056, + "mrr_at_10": 0.575951, + "mrr_at_20": 0.578088, + "mrr_at_100": 0.580653, + "mrr_at_1000": 0.581053, + "nauc_ndcg_at_1_max": 0.494156, + "nauc_ndcg_at_1_std": 0.179823, + "nauc_ndcg_at_1_diff1": 0.701234, + "nauc_ndcg_at_3_max": 0.543891, + "nauc_ndcg_at_3_std": 0.143104, + "nauc_ndcg_at_3_diff1": 0.673382, + "nauc_ndcg_at_5_max": 0.529631, + "nauc_ndcg_at_5_std": 0.117775, + "nauc_ndcg_at_5_diff1": 0.662906, + "nauc_ndcg_at_10_max": 0.549531, + "nauc_ndcg_at_10_std": 0.133055, + "nauc_ndcg_at_10_diff1": 0.664653, + "nauc_ndcg_at_20_max": 0.547342, + "nauc_ndcg_at_20_std": 0.145716, + "nauc_ndcg_at_20_diff1": 0.663893, + "nauc_ndcg_at_100_max": 0.542887, + "nauc_ndcg_at_100_std": 0.163373, + "nauc_ndcg_at_100_diff1": 0.662149, + "nauc_ndcg_at_1000_max": 0.544544, + "nauc_ndcg_at_1000_std": 0.159653, + "nauc_ndcg_at_1000_diff1": 0.667584, + "nauc_map_at_1_max": 0.484528, + "nauc_map_at_1_std": 0.112364, + "nauc_map_at_1_diff1": 0.713798, + "nauc_map_at_3_max": 0.52629, + "nauc_map_at_3_std": 0.127539, + "nauc_map_at_3_diff1": 0.684559, + "nauc_map_at_5_max": 0.521172, + "nauc_map_at_5_std": 0.11887, + "nauc_map_at_5_diff1": 0.678594, + "nauc_map_at_10_max": 0.5326, + "nauc_map_at_10_std": 0.128623, + "nauc_map_at_10_diff1": 0.679285, + "nauc_map_at_20_max": 0.531401, + "nauc_map_at_20_std": 0.133123, + "nauc_map_at_20_diff1": 0.678706, + "nauc_map_at_100_max": 0.530781, + "nauc_map_at_100_std": 0.135701, + "nauc_map_at_100_diff1": 0.678354, + "nauc_map_at_1000_max": 0.530964, + "nauc_map_at_1000_std": 0.135718, + "nauc_map_at_1000_diff1": 0.678608, + "nauc_recall_at_1_max": 0.484528, + "nauc_recall_at_1_std": 0.112364, + "nauc_recall_at_1_diff1": 0.713798, + "nauc_recall_at_3_max": 0.552991, + "nauc_recall_at_3_std": 0.104741, + "nauc_recall_at_3_diff1": 0.639389, + "nauc_recall_at_5_max": 0.523593, + "nauc_recall_at_5_std": 0.052456, + "nauc_recall_at_5_diff1": 0.599453, + "nauc_recall_at_10_max": 0.580827, + "nauc_recall_at_10_std": 0.080142, + "nauc_recall_at_10_diff1": 0.596446, + "nauc_recall_at_20_max": 0.589062, + "nauc_recall_at_20_std": 0.139035, + "nauc_recall_at_20_diff1": 0.59781, + "nauc_recall_at_100_max": 0.58319, + "nauc_recall_at_100_std": 0.347022, + "nauc_recall_at_100_diff1": 0.559343, + "nauc_recall_at_1000_max": 0.791958, + "nauc_recall_at_1000_std": 0.637138, + "nauc_recall_at_1000_diff1": 0.628326, + "nauc_precision_at_1_max": 0.494156, + "nauc_precision_at_1_std": 0.179823, + "nauc_precision_at_1_diff1": 0.701234, + "nauc_precision_at_3_max": 0.572273, + "nauc_precision_at_3_std": 0.270677, + "nauc_precision_at_3_diff1": 0.528038, + "nauc_precision_at_5_max": 0.514554, + "nauc_precision_at_5_std": 0.233615, + "nauc_precision_at_5_diff1": 0.441645, + "nauc_precision_at_10_max": 0.504284, + "nauc_precision_at_10_std": 0.284772, + "nauc_precision_at_10_diff1": 0.314772, + "nauc_precision_at_20_max": 0.425568, + "nauc_precision_at_20_std": 0.326138, + "nauc_precision_at_20_diff1": 0.234987, + "nauc_precision_at_100_max": 0.337983, + "nauc_precision_at_100_std": 0.448859, + "nauc_precision_at_100_diff1": 0.079106, + "nauc_precision_at_1000_max": 0.230914, + "nauc_precision_at_1000_std": 0.460357, + "nauc_precision_at_1000_diff1": -0.125412, + "nauc_mrr_at_1_max": 0.494156, + "nauc_mrr_at_1_std": 0.179823, + "nauc_mrr_at_1_diff1": 0.701234, + "nauc_mrr_at_3_max": 0.535852, + "nauc_mrr_at_3_std": 0.169294, + "nauc_mrr_at_3_diff1": 0.679143, + "nauc_mrr_at_5_max": 0.528663, + "nauc_mrr_at_5_std": 0.160492, + "nauc_mrr_at_5_diff1": 0.671386, + "nauc_mrr_at_10_max": 0.532132, + "nauc_mrr_at_10_std": 0.165224, + "nauc_mrr_at_10_diff1": 0.671661, + "nauc_mrr_at_20_max": 0.531545, + "nauc_mrr_at_20_std": 0.166536, + "nauc_mrr_at_20_diff1": 0.671882, + "nauc_mrr_at_100_max": 0.530415, + "nauc_mrr_at_100_std": 0.167896, + "nauc_mrr_at_100_diff1": 0.671524, + "nauc_mrr_at_1000_max": 0.53043, + "nauc_mrr_at_1000_std": 0.167823, + "nauc_mrr_at_1000_diff1": 0.671834, + "main_score": 0.60427, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 12.572911739349365, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/SprintDuplicateQuestions.json b/mteb/SprintDuplicateQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..d1adf6bb72551aa13f59d4d8ab56855f3b39b8f6 --- /dev/null +++ b/mteb/SprintDuplicateQuestions.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "task_name": "SprintDuplicateQuestions", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "similarity_accuracy": 0.99802, + "similarity_accuracy_threshold": 0.689054, + "similarity_f1": 0.898944, + "similarity_f1_threshold": 0.681838, + "similarity_precision": 0.903943, + "similarity_recall": 0.894, + "similarity_ap": 0.951114, + "cosine_accuracy": 0.99802, + "cosine_accuracy_threshold": 0.689053, + "cosine_f1": 0.898944, + "cosine_f1_threshold": 0.681838, + "cosine_precision": 0.903943, + "cosine_recall": 0.894, + "cosine_ap": 0.951114, + "manhattan_accuracy": 0.998, + "manhattan_accuracy_threshold": 565.695862, + "manhattan_f1": 0.898899, + "manhattan_f1_threshold": 565.695862, + "manhattan_precision": 0.8998, + "manhattan_recall": 0.898, + "manhattan_ap": 0.949934, + "euclidean_accuracy": 0.99799, + "euclidean_accuracy_threshold": 25.397804, + "euclidean_f1": 0.898348, + "euclidean_f1_threshold": 25.688805, + "euclidean_precision": 0.899699, + "euclidean_recall": 0.897, + "euclidean_ap": 0.9506, + "dot_accuracy": 0.997723, + "dot_accuracy_threshold": 723.987732, + "dot_f1": 0.880829, + "dot_f1_threshold": 723.987732, + "dot_precision": 0.913978, + "dot_recall": 0.85, + "dot_ap": 0.937404, + "max_accuracy": 0.99802, + "max_f1": 0.898944, + "max_precision": 0.913978, + "max_recall": 0.898, + "max_ap": 0.951114, + "main_score": 0.951114, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.09257197380066, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/StackExchangeClustering.json b/mteb/StackExchangeClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..66bf1d8814c375618129e336ead096090b1fc4a7 --- /dev/null +++ b/mteb/StackExchangeClustering.json @@ -0,0 +1,47 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "task_name": "StackExchangeClustering", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "v_measure": 0.56641, + "v_measure_std": 0.046061, + "v_measures": [ + 0.618282, + 0.600575, + 0.475684, + 0.602598, + 0.543083, + 0.507604, + 0.522691, + 0.621222, + 0.5887, + 0.553584, + 0.662264, + 0.637221, + 0.652978, + 0.565985, + 0.523508, + 0.560681, + 0.529304, + 0.577785, + 0.542509, + 0.533142, + 0.550816, + 0.520423, + 0.540373, + 0.57613, + 0.553114 + ], + "main_score": 0.56641, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 157.4830355644226, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/StackExchangeClusteringP2P.json b/mteb/StackExchangeClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..797e16ad7d0ab9fc4445e8ffff2387e77538fc9c --- /dev/null +++ b/mteb/StackExchangeClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "task_name": "StackExchangeClusteringP2P", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "v_measure": 0.318501, + "v_measure_std": 0.013043, + "v_measures": [ + 0.307064, + 0.308754, + 0.305133, + 0.305295, + 0.303372, + 0.338656, + 0.326189, + 0.327465, + 0.33349, + 0.329596 + ], + "main_score": 0.318501, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 164.03653264045715, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/StackOverflowDupQuestions.json b/mteb/StackOverflowDupQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..019946a722009be76eb01552632565799c67e442 --- /dev/null +++ b/mteb/StackOverflowDupQuestions.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69", + "task_name": "StackOverflowDupQuestions", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "map": 0.481493, + "mrr": 0.488603, + "nAUC_map_max": 0.143434, + "nAUC_map_std": 0.081154, + "nAUC_map_diff1": 0.350177, + "nAUC_mrr_max": 0.150698, + "nAUC_mrr_std": 0.089674, + "nAUC_mrr_diff1": 0.350292, + "main_score": 0.481493, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 35.406832695007324, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/SummEval.json b/mteb/SummEval.json new file mode 100644 index 0000000000000000000000000000000000000000..12e90b42b6ef0463abeb568040edaac1618f6e05 --- /dev/null +++ b/mteb/SummEval.json @@ -0,0 +1,24 @@ +{ + "dataset_revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c", + "task_name": "SummEval", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "pearson": 0.299904, + "spearman": 0.318365, + "cosine_spearman": 0.318365, + "cosine_pearson": 0.299904, + "dot_spearman": 0.297131, + "dot_pearson": 0.283169, + "main_score": 0.318365, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.6189992427825928, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/TRECCOVID.json b/mteb/TRECCOVID.json new file mode 100644 index 0000000000000000000000000000000000000000..36a27582588eab88d28a61dedb96b6a4a10c0af7 --- /dev/null +++ b/mteb/TRECCOVID.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bb9466bac8153a0349341eb1b22e06409e78ef4e", + "task_name": "TRECCOVID", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.81, + "ndcg_at_3": 0.75989, + "ndcg_at_5": 0.75205, + "ndcg_at_10": 0.71532, + "ndcg_at_20": 0.68284, + "ndcg_at_100": 0.52276, + "ndcg_at_1000": 0.45097, + "map_at_1": 0.00231, + "map_at_3": 0.00583, + "map_at_5": 0.00937, + "map_at_10": 0.01718, + "map_at_20": 0.03005, + "map_at_100": 0.09005, + "map_at_1000": 0.20935, + "recall_at_1": 0.00231, + "recall_at_3": 0.00619, + "recall_at_5": 0.01024, + "recall_at_10": 0.01961, + "recall_at_20": 0.03532, + "recall_at_100": 0.12046, + "recall_at_1000": 0.41831, + "precision_at_1": 0.88, + "precision_at_3": 0.8, + "precision_at_5": 0.8, + "precision_at_10": 0.758, + "precision_at_20": 0.715, + "precision_at_100": 0.5306, + "precision_at_1000": 0.20356, + "mrr_at_1": 0.88, + "mrr_at_3": 0.926667, + "mrr_at_5": 0.931667, + "mrr_at_10": 0.931667, + "mrr_at_20": 0.931667, + "mrr_at_100": 0.931667, + "mrr_at_1000": 0.931667, + "nauc_ndcg_at_1_max": 0.235033, + "nauc_ndcg_at_1_std": 0.425107, + "nauc_ndcg_at_1_diff1": -0.166331, + "nauc_ndcg_at_3_max": 0.457534, + "nauc_ndcg_at_3_std": 0.49449, + "nauc_ndcg_at_3_diff1": -0.218177, + "nauc_ndcg_at_5_max": 0.523915, + "nauc_ndcg_at_5_std": 0.540234, + "nauc_ndcg_at_5_diff1": -0.224601, + "nauc_ndcg_at_10_max": 0.489955, + "nauc_ndcg_at_10_std": 0.579181, + "nauc_ndcg_at_10_diff1": -0.318485, + "nauc_ndcg_at_20_max": 0.536688, + "nauc_ndcg_at_20_std": 0.690301, + "nauc_ndcg_at_20_diff1": -0.354435, + "nauc_ndcg_at_100_max": 0.518196, + "nauc_ndcg_at_100_std": 0.822754, + "nauc_ndcg_at_100_diff1": -0.397011, + "nauc_ndcg_at_1000_max": 0.654034, + "nauc_ndcg_at_1000_std": 0.817756, + "nauc_ndcg_at_1000_diff1": -0.301325, + "nauc_map_at_1_max": 0.030979, + "nauc_map_at_1_std": -0.112343, + "nauc_map_at_1_diff1": 0.207618, + "nauc_map_at_3_max": 0.286885, + "nauc_map_at_3_std": 0.029605, + "nauc_map_at_3_diff1": 0.113494, + "nauc_map_at_5_max": 0.317008, + "nauc_map_at_5_std": 0.056763, + "nauc_map_at_5_diff1": 0.091052, + "nauc_map_at_10_max": 0.299906, + "nauc_map_at_10_std": 0.065652, + "nauc_map_at_10_diff1": 0.054684, + "nauc_map_at_20_max": 0.386532, + "nauc_map_at_20_std": 0.228509, + "nauc_map_at_20_diff1": -0.026004, + "nauc_map_at_100_max": 0.589551, + "nauc_map_at_100_std": 0.673813, + "nauc_map_at_100_diff1": -0.237286, + "nauc_map_at_1000_max": 0.665071, + "nauc_map_at_1000_std": 0.876026, + "nauc_map_at_1000_diff1": -0.342005, + "nauc_recall_at_1_max": 0.030979, + "nauc_recall_at_1_std": -0.112343, + "nauc_recall_at_1_diff1": 0.207618, + "nauc_recall_at_3_max": 0.225952, + "nauc_recall_at_3_std": -0.050699, + "nauc_recall_at_3_diff1": 0.121543, + "nauc_recall_at_5_max": 0.231872, + "nauc_recall_at_5_std": -0.041483, + "nauc_recall_at_5_diff1": 0.119388, + "nauc_recall_at_10_max": 0.185173, + "nauc_recall_at_10_std": -0.063048, + "nauc_recall_at_10_diff1": 0.087213, + "nauc_recall_at_20_max": 0.271838, + "nauc_recall_at_20_std": 0.096419, + "nauc_recall_at_20_diff1": 0.036646, + "nauc_recall_at_100_max": 0.511496, + "nauc_recall_at_100_std": 0.527999, + "nauc_recall_at_100_diff1": -0.177583, + "nauc_recall_at_1000_max": 0.646146, + "nauc_recall_at_1000_std": 0.733405, + "nauc_recall_at_1000_diff1": -0.223217, + "nauc_precision_at_1_max": 0.4895, + "nauc_precision_at_1_std": 0.758988, + "nauc_precision_at_1_diff1": -0.034839, + "nauc_precision_at_3_max": 0.709443, + "nauc_precision_at_3_std": 0.6753, + "nauc_precision_at_3_diff1": -0.198922, + "nauc_precision_at_5_max": 0.730315, + "nauc_precision_at_5_std": 0.708924, + "nauc_precision_at_5_diff1": -0.243446, + "nauc_precision_at_10_max": 0.55359, + "nauc_precision_at_10_std": 0.627347, + "nauc_precision_at_10_diff1": -0.33091, + "nauc_precision_at_20_max": 0.591229, + "nauc_precision_at_20_std": 0.760406, + "nauc_precision_at_20_diff1": -0.35061, + "nauc_precision_at_100_max": 0.531281, + "nauc_precision_at_100_std": 0.858325, + "nauc_precision_at_100_diff1": -0.406393, + "nauc_precision_at_1000_max": 0.413845, + "nauc_precision_at_1000_std": 0.676367, + "nauc_precision_at_1000_diff1": -0.332047, + "nauc_mrr_at_1_max": 0.4895, + "nauc_mrr_at_1_std": 0.758988, + "nauc_mrr_at_1_diff1": -0.034839, + "nauc_mrr_at_3_max": 0.545437, + "nauc_mrr_at_3_std": 0.71448, + "nauc_mrr_at_3_diff1": -0.100942, + "nauc_mrr_at_5_max": 0.532249, + "nauc_mrr_at_5_std": 0.741067, + "nauc_mrr_at_5_diff1": -0.066437, + "nauc_mrr_at_10_max": 0.532249, + "nauc_mrr_at_10_std": 0.741067, + "nauc_mrr_at_10_diff1": -0.066437, + "nauc_mrr_at_20_max": 0.532249, + "nauc_mrr_at_20_std": 0.741067, + "nauc_mrr_at_20_diff1": -0.066437, + "nauc_mrr_at_100_max": 0.532249, + "nauc_mrr_at_100_std": 0.741067, + "nauc_mrr_at_100_diff1": -0.066437, + "nauc_mrr_at_1000_max": 0.532249, + "nauc_mrr_at_1000_std": 0.741067, + "nauc_mrr_at_1000_diff1": -0.066437, + "main_score": 0.71532, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 301.5285987854004, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/Touche2020.json b/mteb/Touche2020.json new file mode 100644 index 0000000000000000000000000000000000000000..3d24d410649b09898fbf58184cd09aac69cc686c --- /dev/null +++ b/mteb/Touche2020.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f", + "task_name": "Touche2020", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "ndcg_at_1": 0.2449, + "ndcg_at_3": 0.24729, + "ndcg_at_5": 0.23791, + "ndcg_at_10": 0.22788, + "ndcg_at_20": 0.25138, + "ndcg_at_100": 0.35506, + "ndcg_at_1000": 0.47535, + "map_at_1": 0.02094, + "map_at_3": 0.05037, + "map_at_5": 0.06667, + "map_at_10": 0.09567, + "map_at_20": 0.12238, + "map_at_100": 0.15456, + "map_at_1000": 0.17134, + "recall_at_1": 0.02094, + "recall_at_3": 0.06356, + "recall_at_5": 0.09517, + "recall_at_10": 0.15589, + "recall_at_20": 0.24322, + "recall_at_100": 0.46668, + "recall_at_1000": 0.83852, + "precision_at_1": 0.26531, + "precision_at_3": 0.26531, + "precision_at_5": 0.2449, + "precision_at_10": 0.20408, + "precision_at_20": 0.17143, + "precision_at_100": 0.0749, + "precision_at_1000": 0.01551, + "mrr_at_1": 0.265306, + "mrr_at_3": 0.380952, + "mrr_at_5": 0.409524, + "mrr_at_10": 0.42151, + "mrr_at_20": 0.431482, + "mrr_at_100": 0.433322, + "mrr_at_1000": 0.433603, + "nauc_ndcg_at_1_max": -0.21184, + "nauc_ndcg_at_1_std": 0.045078, + "nauc_ndcg_at_1_diff1": -0.127358, + "nauc_ndcg_at_3_max": -0.194252, + "nauc_ndcg_at_3_std": -0.062552, + "nauc_ndcg_at_3_diff1": 0.013456, + "nauc_ndcg_at_5_max": -0.21515, + "nauc_ndcg_at_5_std": -0.078813, + "nauc_ndcg_at_5_diff1": -0.012183, + "nauc_ndcg_at_10_max": -0.1812, + "nauc_ndcg_at_10_std": -0.055853, + "nauc_ndcg_at_10_diff1": 0.012866, + "nauc_ndcg_at_20_max": -0.279821, + "nauc_ndcg_at_20_std": -0.058553, + "nauc_ndcg_at_20_diff1": 0.031812, + "nauc_ndcg_at_100_max": -0.230923, + "nauc_ndcg_at_100_std": 0.126236, + "nauc_ndcg_at_100_diff1": 0.08099, + "nauc_ndcg_at_1000_max": -0.201611, + "nauc_ndcg_at_1000_std": 0.222888, + "nauc_ndcg_at_1000_diff1": 0.038947, + "nauc_map_at_1_max": -0.151336, + "nauc_map_at_1_std": -0.05686, + "nauc_map_at_1_diff1": -0.110982, + "nauc_map_at_3_max": -0.17957, + "nauc_map_at_3_std": -0.143613, + "nauc_map_at_3_diff1": -0.040538, + "nauc_map_at_5_max": -0.197211, + "nauc_map_at_5_std": -0.154328, + "nauc_map_at_5_diff1": -0.05491, + "nauc_map_at_10_max": -0.146394, + "nauc_map_at_10_std": -0.151459, + "nauc_map_at_10_diff1": -0.020209, + "nauc_map_at_20_max": -0.194593, + "nauc_map_at_20_std": -0.150695, + "nauc_map_at_20_diff1": 0.027577, + "nauc_map_at_100_max": -0.172374, + "nauc_map_at_100_std": -0.062214, + "nauc_map_at_100_diff1": 0.077665, + "nauc_map_at_1000_max": -0.164723, + "nauc_map_at_1000_std": -0.024473, + "nauc_map_at_1000_diff1": 0.071406, + "nauc_recall_at_1_max": -0.151336, + "nauc_recall_at_1_std": -0.05686, + "nauc_recall_at_1_diff1": -0.110982, + "nauc_recall_at_3_max": -0.216947, + "nauc_recall_at_3_std": -0.208214, + "nauc_recall_at_3_diff1": -0.021295, + "nauc_recall_at_5_max": -0.24176, + "nauc_recall_at_5_std": -0.213149, + "nauc_recall_at_5_diff1": -0.06896, + "nauc_recall_at_10_max": -0.179446, + "nauc_recall_at_10_std": -0.142058, + "nauc_recall_at_10_diff1": -0.022227, + "nauc_recall_at_20_max": -0.301632, + "nauc_recall_at_20_std": -0.119398, + "nauc_recall_at_20_diff1": 0.042804, + "nauc_recall_at_100_max": -0.21802, + "nauc_recall_at_100_std": 0.203004, + "nauc_recall_at_100_diff1": 0.081152, + "nauc_recall_at_1000_max": -0.154843, + "nauc_recall_at_1000_std": 0.610456, + "nauc_recall_at_1000_diff1": -0.223118, + "nauc_precision_at_1_max": -0.163555, + "nauc_precision_at_1_std": 0.014887, + "nauc_precision_at_1_diff1": -0.072894, + "nauc_precision_at_3_max": -0.185059, + "nauc_precision_at_3_std": -0.110688, + "nauc_precision_at_3_diff1": 0.082707, + "nauc_precision_at_5_max": -0.20949, + "nauc_precision_at_5_std": -0.135561, + "nauc_precision_at_5_diff1": 0.038719, + "nauc_precision_at_10_max": -0.115299, + "nauc_precision_at_10_std": -0.016832, + "nauc_precision_at_10_diff1": 0.116033, + "nauc_precision_at_20_max": -0.271497, + "nauc_precision_at_20_std": 0.057508, + "nauc_precision_at_20_diff1": 0.200541, + "nauc_precision_at_100_max": -0.02628, + "nauc_precision_at_100_std": 0.470634, + "nauc_precision_at_100_diff1": 0.298468, + "nauc_precision_at_1000_max": 0.308665, + "nauc_precision_at_1000_std": 0.338682, + "nauc_precision_at_1000_diff1": 0.113792, + "nauc_mrr_at_1_max": -0.163555, + "nauc_mrr_at_1_std": 0.014887, + "nauc_mrr_at_1_diff1": -0.072894, + "nauc_mrr_at_3_max": -0.201747, + "nauc_mrr_at_3_std": -0.098292, + "nauc_mrr_at_3_diff1": 0.023817, + "nauc_mrr_at_5_max": -0.202904, + "nauc_mrr_at_5_std": -0.081643, + "nauc_mrr_at_5_diff1": -0.001708, + "nauc_mrr_at_10_max": -0.19431, + "nauc_mrr_at_10_std": -0.046234, + "nauc_mrr_at_10_diff1": -0.009555, + "nauc_mrr_at_20_max": -0.197291, + "nauc_mrr_at_20_std": -0.044182, + "nauc_mrr_at_20_diff1": -0.007163, + "nauc_mrr_at_100_max": -0.19662, + "nauc_mrr_at_100_std": -0.044295, + "nauc_mrr_at_100_diff1": -0.009366, + "nauc_mrr_at_1000_max": -0.196591, + "nauc_mrr_at_1000_std": -0.045101, + "nauc_mrr_at_1000_diff1": -0.009622, + "main_score": 0.22788, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1095.016489982605, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/ToxicConversationsClassification.json b/mteb/ToxicConversationsClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..9331703d269813de1fce90c1bad0a0a3ba5699a5 --- /dev/null +++ b/mteb/ToxicConversationsClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "edfaf9da55d3dd50d43143d90c1ac476895ae6de", + "task_name": "ToxicConversationsClassification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.644141, + "f1": 0.485129, + "f1_weighted": 0.722769, + "ap": 0.104382, + "ap_weighted": 0.104382, + "scores_per_experiment": [ + { + "accuracy": 0.506836, + "f1": 0.416453, + "f1_weighted": 0.609778, + "ap": 0.09937, + "ap_weighted": 0.09937 + }, + { + "accuracy": 0.742676, + "f1": 0.543386, + "f1_weighted": 0.797322, + "ap": 0.119804, + "ap_weighted": 0.119804 + }, + { + "accuracy": 0.687012, + "f1": 0.509065, + "f1_weighted": 0.757873, + "ap": 0.108171, + "ap_weighted": 0.108171 + }, + { + "accuracy": 0.687988, + "f1": 0.513283, + "f1_weighted": 0.758753, + "ap": 0.112236, + "ap_weighted": 0.112236 + }, + { + "accuracy": 0.605957, + "f1": 0.459904, + "f1_weighted": 0.696332, + "ap": 0.094358, + "ap_weighted": 0.094358 + }, + { + "accuracy": 0.541016, + "f1": 0.427484, + "f1_weighted": 0.642099, + "ap": 0.092116, + "ap_weighted": 0.092116 + }, + { + "accuracy": 0.708496, + "f1": 0.505065, + "f1_weighted": 0.772175, + "ap": 0.0967, + "ap_weighted": 0.0967 + }, + { + "accuracy": 0.69043, + "f1": 0.513077, + "f1_weighted": 0.760452, + "ap": 0.110868, + "ap_weighted": 0.110868 + }, + { + "accuracy": 0.569336, + "f1": 0.455629, + "f1_weighted": 0.665064, + "ap": 0.108882, + "ap_weighted": 0.108882 + }, + { + "accuracy": 0.70166, + "f1": 0.507948, + "f1_weighted": 0.767839, + "ap": 0.101317, + "ap_weighted": 0.101317 + } + ], + "main_score": 0.644141, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.779565811157227, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/TweetSentimentExtractionClassification.json b/mteb/TweetSentimentExtractionClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..81e9d11278404be5704f69ee9ddb0f46a3236d11 --- /dev/null +++ b/mteb/TweetSentimentExtractionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a", + "task_name": "TweetSentimentExtractionClassification", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "accuracy": 0.526316, + "f1": 0.529212, + "f1_weighted": 0.523632, + "scores_per_experiment": [ + { + "accuracy": 0.532258, + "f1": 0.534863, + "f1_weighted": 0.527884 + }, + { + "accuracy": 0.533956, + "f1": 0.537524, + "f1_weighted": 0.529935 + }, + { + "accuracy": 0.522071, + "f1": 0.525822, + "f1_weighted": 0.521472 + }, + { + "accuracy": 0.523769, + "f1": 0.525527, + "f1_weighted": 0.524488 + }, + { + "accuracy": 0.54867, + "f1": 0.550189, + "f1_weighted": 0.54339 + }, + { + "accuracy": 0.55348, + "f1": 0.556835, + "f1_weighted": 0.549865 + }, + { + "accuracy": 0.517261, + "f1": 0.520434, + "f1_weighted": 0.515424 + }, + { + "accuracy": 0.526316, + "f1": 0.529428, + "f1_weighted": 0.521649 + }, + { + "accuracy": 0.50283, + "f1": 0.506101, + "f1_weighted": 0.501108 + }, + { + "accuracy": 0.502547, + "f1": 0.505401, + "f1_weighted": 0.501102 + } + ], + "main_score": 0.526316, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.326297998428345, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/TwentyNewsgroupsClustering.json b/mteb/TwentyNewsgroupsClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..886bbc77f53a22f23ef3ce876f483c69ec43bd2b --- /dev/null +++ b/mteb/TwentyNewsgroupsClustering.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "task_name": "TwentyNewsgroupsClustering", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "v_measure": 0.343169, + "v_measure_std": 0.01669, + "v_measures": [ + 0.371176, + 0.329113, + 0.358309, + 0.359354, + 0.321954, + 0.33045, + 0.34941, + 0.3566, + 0.327921, + 0.327403 + ], + "main_score": 0.343169, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 26.601304531097412, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/TwitterSemEval2015.json b/mteb/TwitterSemEval2015.json new file mode 100644 index 0000000000000000000000000000000000000000..04474fa810177ac1cfe769129fb11e05bee04116 --- /dev/null +++ b/mteb/TwitterSemEval2015.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "task_name": "TwitterSemEval2015", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "similarity_accuracy": 0.834595, + "similarity_accuracy_threshold": 0.724461, + "similarity_f1": 0.617155, + "similarity_f1_threshold": 0.625381, + "similarity_precision": 0.560681, + "similarity_recall": 0.68628, + "similarity_ap": 0.650861, + "cosine_accuracy": 0.834595, + "cosine_accuracy_threshold": 0.724461, + "cosine_f1": 0.617155, + "cosine_f1_threshold": 0.625381, + "cosine_precision": 0.560681, + "cosine_recall": 0.68628, + "cosine_ap": 0.650861, + "manhattan_accuracy": 0.833999, + "manhattan_accuracy_threshold": 534.933594, + "manhattan_f1": 0.610456, + "manhattan_f1_threshold": 607.510315, + "manhattan_precision": 0.576589, + "manhattan_recall": 0.648549, + "manhattan_ap": 0.645486, + "euclidean_accuracy": 0.834059, + "euclidean_accuracy_threshold": 24.222414, + "euclidean_f1": 0.609938, + "euclidean_f1_threshold": 27.656849, + "euclidean_precision": 0.576291, + "euclidean_recall": 0.647757, + "euclidean_ap": 0.646071, + "dot_accuracy": 0.826012, + "dot_accuracy_threshold": 771.538452, + "dot_f1": 0.599331, + "dot_f1_threshold": 671.008545, + "dot_precision": 0.547858, + "dot_recall": 0.661478, + "dot_ap": 0.621271, + "max_accuracy": 0.834595, + "max_f1": 0.617155, + "max_precision": 0.576589, + "max_recall": 0.68628, + "max_ap": 0.650861, + "main_score": 0.650861, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 8.805907487869263, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/TwitterURLCorpus.json b/mteb/TwitterURLCorpus.json new file mode 100644 index 0000000000000000000000000000000000000000..a2859d18c25419a49bbc24b5c7f55179d2bef2dd --- /dev/null +++ b/mteb/TwitterURLCorpus.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "task_name": "TwitterURLCorpus", + "mteb_version": "1.25.1", + "scores": { + "test": [ + { + "similarity_accuracy": 0.881612, + "similarity_accuracy_threshold": 0.638545, + "similarity_f1": 0.763737, + "similarity_f1_threshold": 0.604226, + "similarity_precision": 0.741711, + "similarity_recall": 0.787111, + "similarity_ap": 0.835651, + "cosine_accuracy": 0.881612, + "cosine_accuracy_threshold": 0.638545, + "cosine_f1": 0.763737, + "cosine_f1_threshold": 0.604226, + "cosine_precision": 0.741711, + "cosine_recall": 0.787111, + "cosine_ap": 0.835651, + "manhattan_accuracy": 0.879652, + "manhattan_accuracy_threshold": 578.377014, + "manhattan_f1": 0.761352, + "manhattan_f1_threshold": 617.466187, + "manhattan_precision": 0.723335, + "manhattan_recall": 0.803588, + "manhattan_ap": 0.833006, + "euclidean_accuracy": 0.880002, + "euclidean_accuracy_threshold": 26.073254, + "euclidean_f1": 0.762589, + "euclidean_f1_threshold": 27.932058, + "euclidean_precision": 0.731054, + "euclidean_recall": 0.796966, + "euclidean_ap": 0.833352, + "dot_accuracy": 0.876994, + "dot_accuracy_threshold": 621.233459, + "dot_f1": 0.75509, + "dot_f1_threshold": 561.458374, + "dot_precision": 0.709651, + "dot_recall": 0.806745, + "dot_ap": 0.817367, + "max_accuracy": 0.881612, + "max_f1": 0.763737, + "max_precision": 0.741711, + "max_recall": 0.806745, + "max_ap": 0.835651, + "main_score": 0.835651, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 22.606329441070557, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/mteb/model_meta.json b/mteb/model_meta.json new file mode 100644 index 0000000000000000000000000000000000000000..94698b49c9810ffa2fda32895b06b63d8dea58c9 --- /dev/null +++ b/mteb/model_meta.json @@ -0,0 +1 @@ +{"name": "joe32140/ModernBERT-base-msmarco", "revision": "c681d6fd230415ac36cb6468d0d98571fef81d58", "release_date": null, "languages": [], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "loader": null} \ No newline at end of file