diff --git a/beir/AmazonCounterfactualClassification.json b/beir/AmazonCounterfactualClassification.json deleted file mode 100644 index 1ed3a7a4a4f53f8351866f1dfac5cbbc482e7792..0000000000000000000000000000000000000000 --- a/beir/AmazonCounterfactualClassification.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205", - "task_name": "AmazonCounterfactualClassification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.65997, - "f1": 0.535951, - "f1_weighted": 0.725304, - "ap": 0.163093, - "ap_weighted": 0.163093, - "scores_per_experiment": [ - { - "accuracy": 0.644678, - "f1": 0.531347, - "f1_weighted": 0.713782, - "ap": 0.166602, - "ap_weighted": 0.166602 - }, - { - "accuracy": 0.608696, - "f1": 0.518256, - "f1_weighted": 0.683489, - "ap": 0.176106, - "ap_weighted": 0.176106 - }, - { - "accuracy": 0.643928, - "f1": 0.529901, - "f1_weighted": 0.713177, - "ap": 0.164896, - "ap_weighted": 0.164896 - }, - { - "accuracy": 0.641679, - "f1": 0.51983, - "f1_weighted": 0.711307, - "ap": 0.151661, - "ap_weighted": 0.151661 - }, - { - "accuracy": 0.668666, - "f1": 0.544738, - "f1_weighted": 0.732766, - "ap": 0.169199, - "ap_weighted": 0.169199 - }, - { - "accuracy": 0.707646, - "f1": 0.550963, - "f1_weighted": 0.760934, - "ap": 0.15327, - "ap_weighted": 0.15327 - }, - { - "accuracy": 0.654423, - "f1": 0.525677, - "f1_weighted": 0.721296, - "ap": 0.151452, - "ap_weighted": 0.151452 - }, - { - "accuracy": 0.664918, - "f1": 0.539092, - "f1_weighted": 0.729726, - "ap": 0.163361, - "ap_weighted": 0.163361 - }, - { - "accuracy": 0.696402, - "f1": 0.563025, - "f1_weighted": 0.754132, - "ap": 0.176998, - "ap_weighted": 0.176998 - }, - { - "accuracy": 0.668666, - "f1": 0.536679, - "f1_weighted": 0.732435, - "ap": 0.15738, - "ap_weighted": 0.15738 - } - ], - "main_score": 0.65997, - "hf_subset": "en-ext", - "languages": [ - "eng-Latn" - ] - }, - { - "accuracy": 0.64194, - "f1": 0.585969, - "f1_weighted": 0.677482, - "ap": 0.280748, - "ap_weighted": 0.280748, - "scores_per_experiment": [ - { - "accuracy": 0.659701, - "f1": 0.606678, - "f1_weighted": 0.694619, - "ap": 0.299469, - "ap_weighted": 0.299469 - }, - { - "accuracy": 0.61194, - "f1": 0.564082, - "f1_weighted": 0.652038, - "ap": 0.268001, - "ap_weighted": 0.268001 - }, - { - "accuracy": 0.592537, - "f1": 0.553977, - "f1_weighted": 0.633838, - "ap": 0.269473, - "ap_weighted": 0.269473 - }, - { - "accuracy": 0.61194, - "f1": 0.565044, - "f1_weighted": 0.652016, - "ap": 0.269697, - "ap_weighted": 0.269697 - }, - { - "accuracy": 0.668657, - "f1": 0.603271, - "f1_weighted": 0.70135, - "ap": 0.285582, - "ap_weighted": 0.285582 - }, - { - "accuracy": 0.625373, - "f1": 0.56863, - "f1_weighted": 0.663902, - "ap": 0.26389, - "ap_weighted": 0.26389 - }, - { - "accuracy": 0.744776, - "f1": 0.648233, - "f1_weighted": 0.760454, - "ap": 0.307423, - "ap_weighted": 0.307423 - }, - { - "accuracy": 0.667164, - "f1": 0.613827, - "f1_weighted": 0.701223, - "ap": 0.30575, - "ap_weighted": 0.30575 - }, - { - "accuracy": 0.591045, - "f1": 0.547427, - "f1_weighted": 0.632985, - "ap": 0.259001, - "ap_weighted": 0.259001 - }, - { - "accuracy": 0.646269, - "f1": 0.588519, - "f1_weighted": 0.682391, - "ap": 0.279194, - "ap_weighted": 0.279194 - } - ], - "main_score": 0.64194, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 18.05203342437744, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/AmazonPolarityClassification.json b/beir/AmazonPolarityClassification.json deleted file mode 100644 index f0f3a65bee36dd97551ff68a3acb74fa9b3a8c77..0000000000000000000000000000000000000000 --- a/beir/AmazonPolarityClassification.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "dataset_revision": "e2d317d38cd51312af73b3d32a06d1a08b442046", - "task_name": "AmazonPolarityClassification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.671713, - "f1": 0.669443, - "f1_weighted": 0.669443, - "ap": 0.617296, - "ap_weighted": 0.617296, - "scores_per_experiment": [ - { - "accuracy": 0.730195, - "f1": 0.729644, - "f1_weighted": 0.729644, - "ap": 0.673348, - "ap_weighted": 0.673348 - }, - { - "accuracy": 0.648605, - "f1": 0.647126, - "f1_weighted": 0.647126, - "ap": 0.593854, - "ap_weighted": 0.593854 - }, - { - "accuracy": 0.720182, - "f1": 0.716345, - "f1_weighted": 0.716345, - "ap": 0.649422, - "ap_weighted": 0.649422 - }, - { - "accuracy": 0.717512, - "f1": 0.715561, - "f1_weighted": 0.715561, - "ap": 0.665462, - "ap_weighted": 0.665462 - }, - { - "accuracy": 0.660528, - "f1": 0.659856, - "f1_weighted": 0.659856, - "ap": 0.60393, - "ap_weighted": 0.60393 - }, - { - "accuracy": 0.66763, - "f1": 0.66655, - "f1_weighted": 0.66655, - "ap": 0.609044, - "ap_weighted": 0.609044 - }, - { - "accuracy": 0.67263, - "f1": 0.670559, - "f1_weighted": 0.670559, - "ap": 0.612038, - "ap_weighted": 0.612038 - }, - { - "accuracy": 0.677455, - "f1": 0.674938, - "f1_weighted": 0.674938, - "ap": 0.626943, - "ap_weighted": 0.626943 - }, - { - "accuracy": 0.582143, - "f1": 0.574911, - "f1_weighted": 0.574911, - "ap": 0.546423, - "ap_weighted": 0.546423 - }, - { - "accuracy": 0.64025, - "f1": 0.638938, - "f1_weighted": 0.638938, - "ap": 0.592492, - "ap_weighted": 0.592492 - } - ], - "main_score": 0.671713, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 439.54625058174133, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/AmazonReviewsClassification.json b/beir/AmazonReviewsClassification.json deleted file mode 100644 index 28a4b86fa77fcfccf6346ae904fa8eab9a0e4e4c..0000000000000000000000000000000000000000 --- a/beir/AmazonReviewsClassification.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d", - "task_name": "AmazonReviewsClassification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.32318, - "f1": 0.319973, - "f1_weighted": 0.319973, - "scores_per_experiment": [ - { - "accuracy": 0.341, - "f1": 0.333855, - "f1_weighted": 0.333855 - }, - { - "accuracy": 0.3368, - "f1": 0.338041, - "f1_weighted": 0.338041 - }, - { - "accuracy": 0.3336, - "f1": 0.330524, - "f1_weighted": 0.330524 - }, - { - "accuracy": 0.3086, - "f1": 0.309647, - "f1_weighted": 0.309647 - }, - { - "accuracy": 0.3114, - "f1": 0.305097, - "f1_weighted": 0.305097 - }, - { - "accuracy": 0.3334, - "f1": 0.330257, - "f1_weighted": 0.330257 - }, - { - "accuracy": 0.299, - "f1": 0.292306, - "f1_weighted": 0.292306 - }, - { - "accuracy": 0.3384, - "f1": 0.336788, - "f1_weighted": 0.336788 - }, - { - "accuracy": 0.3242, - "f1": 0.31879, - "f1_weighted": 0.31879 - }, - { - "accuracy": 0.3054, - "f1": 0.304421, - "f1_weighted": 0.304421 - } - ], - "main_score": 0.32318, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 9.712511777877808, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/ArguAna.json b/beir/ArguAna.json deleted file mode 100644 index 22daa51005ec9b728f797b73f9eeae1e1566003b..0000000000000000000000000000000000000000 --- a/beir/ArguAna.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a", - "task_name": "ArguAna", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.24609, - "ndcg_at_3": 0.36278, - "ndcg_at_5": 0.40903, - "ndcg_at_10": 0.46381, - "ndcg_at_20": 0.49571, - "ndcg_at_100": 0.51666, - "ndcg_at_1000": 0.52098, - "map_at_1": 0.24609, - "map_at_3": 0.33404, - "map_at_5": 0.35968, - "map_at_10": 0.38204, - "map_at_20": 0.39113, - "map_at_100": 0.39432, - "map_at_1000": 0.3945, - "recall_at_1": 0.24609, - "recall_at_3": 0.44595, - "recall_at_5": 0.55832, - "recall_at_10": 0.72902, - "recall_at_20": 0.85277, - "recall_at_100": 0.96159, - "recall_at_1000": 0.99431, - "precision_at_1": 0.24609, - "precision_at_3": 0.14865, - "precision_at_5": 0.11166, - "precision_at_10": 0.0729, - "precision_at_20": 0.04264, - "precision_at_100": 0.00962, - "precision_at_1000": 0.00099, - "mrr_at_1": 0.251067, - "mrr_at_3": 0.335941, - "mrr_at_5": 0.361546, - "mrr_at_10": 0.383849, - "mrr_at_20": 0.393004, - "mrr_at_100": 0.396189, - "mrr_at_1000": 0.396374, - "nauc_ndcg_at_1_max": -0.056006, - "nauc_ndcg_at_1_std": -0.048049, - "nauc_ndcg_at_1_diff1": 0.14903, - "nauc_ndcg_at_3_max": -0.00408, - "nauc_ndcg_at_3_std": -0.019375, - "nauc_ndcg_at_3_diff1": 0.121454, - "nauc_ndcg_at_5_max": -0.004685, - "nauc_ndcg_at_5_std": -0.020393, - "nauc_ndcg_at_5_diff1": 0.107318, - "nauc_ndcg_at_10_max": 0.030675, - "nauc_ndcg_at_10_std": -0.003638, - "nauc_ndcg_at_10_diff1": 0.113071, - "nauc_ndcg_at_20_max": 0.025101, - "nauc_ndcg_at_20_std": -0.003348, - "nauc_ndcg_at_20_diff1": 0.115383, - "nauc_ndcg_at_100_max": 0.010754, - "nauc_ndcg_at_100_std": 0.001077, - "nauc_ndcg_at_100_diff1": 0.118117, - "nauc_ndcg_at_1000_max": 0.005791, - "nauc_ndcg_at_1000_std": -0.006433, - "nauc_ndcg_at_1000_diff1": 0.119586, - "nauc_map_at_1_max": -0.056006, - "nauc_map_at_1_std": -0.048049, - "nauc_map_at_1_diff1": 0.14903, - "nauc_map_at_3_max": -0.015511, - "nauc_map_at_3_std": -0.024433, - "nauc_map_at_3_diff1": 0.126974, - "nauc_map_at_5_max": -0.015512, - "nauc_map_at_5_std": -0.024537, - "nauc_map_at_5_diff1": 0.11982, - "nauc_map_at_10_max": -0.00244, - "nauc_map_at_10_std": -0.017698, - "nauc_map_at_10_diff1": 0.122534, - "nauc_map_at_20_max": -0.00477, - "nauc_map_at_20_std": -0.018189, - "nauc_map_at_20_diff1": 0.123218, - "nauc_map_at_100_max": -0.006815, - "nauc_map_at_100_std": -0.017488, - "nauc_map_at_100_diff1": 0.123542, - "nauc_map_at_1000_max": -0.006971, - "nauc_map_at_1000_std": -0.017692, - "nauc_map_at_1000_diff1": 0.123594, - "nauc_recall_at_1_max": -0.056006, - "nauc_recall_at_1_std": -0.048049, - "nauc_recall_at_1_diff1": 0.14903, - "nauc_recall_at_3_max": 0.026873, - "nauc_recall_at_3_std": -0.006182, - "nauc_recall_at_3_diff1": 0.106725, - "nauc_recall_at_5_max": 0.025734, - "nauc_recall_at_5_std": -0.009853, - "nauc_recall_at_5_diff1": 0.068572, - "nauc_recall_at_10_max": 0.178111, - "nauc_recall_at_10_std": 0.056516, - "nauc_recall_at_10_diff1": 0.07601, - "nauc_recall_at_20_max": 0.239859, - "nauc_recall_at_20_std": 0.104235, - "nauc_recall_at_20_diff1": 0.070172, - "nauc_recall_at_100_max": 0.307669, - "nauc_recall_at_100_std": 0.509552, - "nauc_recall_at_100_diff1": 0.051657, - "nauc_recall_at_1000_max": 0.254176, - "nauc_recall_at_1000_std": 0.730444, - "nauc_recall_at_1000_diff1": 0.166349, - "nauc_precision_at_1_max": -0.056006, - "nauc_precision_at_1_std": -0.048049, - "nauc_precision_at_1_diff1": 0.14903, - "nauc_precision_at_3_max": 0.026873, - "nauc_precision_at_3_std": -0.006182, - "nauc_precision_at_3_diff1": 0.106725, - "nauc_precision_at_5_max": 0.025734, - "nauc_precision_at_5_std": -0.009853, - "nauc_precision_at_5_diff1": 0.068572, - "nauc_precision_at_10_max": 0.178111, - "nauc_precision_at_10_std": 0.056516, - "nauc_precision_at_10_diff1": 0.07601, - "nauc_precision_at_20_max": 0.239859, - "nauc_precision_at_20_std": 0.104235, - "nauc_precision_at_20_diff1": 0.070172, - "nauc_precision_at_100_max": 0.307669, - "nauc_precision_at_100_std": 0.509552, - "nauc_precision_at_100_diff1": 0.051657, - "nauc_precision_at_1000_max": 0.254176, - "nauc_precision_at_1000_std": 0.730444, - "nauc_precision_at_1000_diff1": 0.166349, - "nauc_mrr_at_1_max": -0.051381, - "nauc_mrr_at_1_std": -0.041855, - "nauc_mrr_at_1_diff1": 0.132056, - "nauc_mrr_at_3_max": -0.017796, - "nauc_mrr_at_3_std": -0.023432, - "nauc_mrr_at_3_diff1": 0.114369, - "nauc_mrr_at_5_max": -0.017394, - "nauc_mrr_at_5_std": -0.023168, - "nauc_mrr_at_5_diff1": 0.107454, - "nauc_mrr_at_10_max": -0.005076, - "nauc_mrr_at_10_std": -0.016223, - "nauc_mrr_at_10_diff1": 0.109067, - "nauc_mrr_at_20_max": -0.00715, - "nauc_mrr_at_20_std": -0.016808, - "nauc_mrr_at_20_diff1": 0.109766, - "nauc_mrr_at_100_max": -0.009223, - "nauc_mrr_at_100_std": -0.016097, - "nauc_mrr_at_100_diff1": 0.109945, - "nauc_mrr_at_1000_max": -0.00938, - "nauc_mrr_at_1000_std": -0.0163, - "nauc_mrr_at_1000_diff1": 0.109988, - "main_score": 0.46381, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 18.032260179519653, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/ArxivClusteringP2P.json b/beir/ArxivClusteringP2P.json deleted file mode 100644 index 01ed4f0ebb6ed0158d42b11a27521662f91780b0..0000000000000000000000000000000000000000 --- a/beir/ArxivClusteringP2P.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "dataset_revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d", - "task_name": "ArxivClusteringP2P", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "v_measure": 0.389781, - "v_measure_std": 0.140064, - "v_measures": [ - 0.397318, - 0.395931, - 0.398734, - 0.421181, - 0.402112, - 0.39659, - 0.40027, - 0.4156, - 0.418672, - 0.408705, - 0.442656, - 0.441182, - 0.438416, - 0.447467, - 0.440814, - 0.442237, - 0.445092, - 0.440632, - 0.445347, - 0.437578, - 0.391066, - 0.265296, - 0.30278, - 0.296997, - 0.291818, - 0.212322, - 0.256389, - 0.159543, - 0.200181, - 1.0, - 0.230291 - ], - "main_score": 0.389781, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 1234.510992050171, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/ArxivClusteringS2S.json b/beir/ArxivClusteringS2S.json deleted file mode 100644 index 239503c80a116a05ccc215e9f6a9acf8766115c1..0000000000000000000000000000000000000000 --- a/beir/ArxivClusteringS2S.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "dataset_revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53", - "task_name": "ArxivClusteringS2S", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "v_measure": 0.289645, - "v_measure_std": 0.142769, - "v_measures": [ - 0.284647, - 0.288473, - 0.27734, - 0.269279, - 0.263681, - 0.266674, - 0.272582, - 0.251594, - 0.287376, - 0.275816, - 0.326475, - 0.330363, - 0.320361, - 0.330296, - 0.328716, - 0.325683, - 0.321654, - 0.324298, - 0.326664, - 0.316507, - 0.304434, - 0.188541, - 0.204597, - 0.259977, - 0.237603, - 0.164765, - 0.199714, - 0.108873, - 0.155307, - 1.0, - 0.166707 - ], - "main_score": 0.289645, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 300.5607361793518, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/AskUbuntuDupQuestions.json b/beir/AskUbuntuDupQuestions.json deleted file mode 100644 index 0c4b75b757d4abfbf7bc366f23b3b5ba70f774a4..0000000000000000000000000000000000000000 --- a/beir/AskUbuntuDupQuestions.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "2000358ca161889fa9c082cb41daa8dcfb161a54", - "task_name": "AskUbuntuDupQuestions", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "map": 0.579553, - "mrr": 0.708293, - "nAUC_map_max": 0.232041, - "nAUC_map_std": 0.210013, - "nAUC_map_diff1": 0.067763, - "nAUC_mrr_max": 0.331435, - "nAUC_mrr_std": 0.258722, - "nAUC_mrr_diff1": 0.102834, - "main_score": 0.579553, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 2.8852455615997314, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/BIOSSES.json b/beir/BIOSSES.json deleted file mode 100644 index 65507bf63a5f71cbedc18786149507ed7dea53b0..0000000000000000000000000000000000000000 --- a/beir/BIOSSES.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a", - "task_name": "BIOSSES", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "pearson": 0.850909, - "spearman": 0.807386, - "cosine_pearson": 0.850909, - "cosine_spearman": 0.807386, - "manhattan_pearson": 0.820327, - "manhattan_spearman": 0.78562, - "euclidean_pearson": 0.826159, - "euclidean_spearman": 0.793542, - "main_score": 0.807386, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 0.15388250350952148, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/Banking77Classification.json b/beir/Banking77Classification.json deleted file mode 100644 index ffe1bac7d5457ccac26ee41da879480254cc1b85..0000000000000000000000000000000000000000 --- a/beir/Banking77Classification.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "dataset_revision": "0fd18e25b25c072e09e0d92ab615fda904d66300", - "task_name": "Banking77Classification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.82026, - "f1": 0.819738, - "f1_weighted": 0.819738, - "scores_per_experiment": [ - { - "accuracy": 0.820779, - "f1": 0.820659, - "f1_weighted": 0.820659 - }, - { - "accuracy": 0.827273, - "f1": 0.827807, - "f1_weighted": 0.827807 - }, - { - "accuracy": 0.837013, - "f1": 0.836189, - "f1_weighted": 0.836189 - }, - { - "accuracy": 0.827273, - "f1": 0.827111, - "f1_weighted": 0.827111 - }, - { - "accuracy": 0.816558, - "f1": 0.816573, - "f1_weighted": 0.816573 - }, - { - "accuracy": 0.800649, - "f1": 0.79907, - "f1_weighted": 0.79907 - }, - { - "accuracy": 0.816558, - "f1": 0.815777, - "f1_weighted": 0.815777 - }, - { - "accuracy": 0.812338, - "f1": 0.811554, - "f1_weighted": 0.811554 - }, - { - "accuracy": 0.828247, - "f1": 0.828126, - "f1_weighted": 0.828126 - }, - { - "accuracy": 0.815909, - "f1": 0.814514, - "f1_weighted": 0.814514 - } - ], - "main_score": 0.82026, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 11.350855588912964, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/BiorxivClusteringP2P.json b/beir/BiorxivClusteringP2P.json deleted file mode 100644 index b61f99419e7617898e532c88153c6b611e2cf630..0000000000000000000000000000000000000000 --- a/beir/BiorxivClusteringP2P.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "dataset_revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40", - "task_name": "BiorxivClusteringP2P", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "v_measure": 0.30787, - "v_measure_std": 0.009286, - "v_measures": [ - 0.294563, - 0.300806, - 0.300434, - 0.298455, - 0.313446, - 0.313647, - 0.310572, - 0.303024, - 0.3205, - 0.323252 - ], - "main_score": 0.30787, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 183.2675678730011, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/BiorxivClusteringS2S.json b/beir/BiorxivClusteringS2S.json deleted file mode 100644 index edb6bf759830e41e1242e1837e00a01a70cf558c..0000000000000000000000000000000000000000 --- a/beir/BiorxivClusteringS2S.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "dataset_revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908", - "task_name": "BiorxivClusteringS2S", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "v_measure": 0.270998, - "v_measure_std": 0.007908, - "v_measures": [ - 0.269737, - 0.264231, - 0.263528, - 0.267448, - 0.268571, - 0.273179, - 0.261491, - 0.27174, - 0.282342, - 0.287709 - ], - "main_score": 0.270998, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 32.66898798942566, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackAndroidRetrieval.json b/beir/CQADupstackAndroidRetrieval.json deleted file mode 100644 index 8fdcd4cb062fde35c982679b5f188e2062fac656..0000000000000000000000000000000000000000 --- a/beir/CQADupstackAndroidRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "f46a197baaae43b4f621051089b82a364682dfeb", - "task_name": "CQADupstackAndroidRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.29757, - "ndcg_at_3": 0.34736, - "ndcg_at_5": 0.36529, - "ndcg_at_10": 0.39114, - "ndcg_at_20": 0.4105, - "ndcg_at_100": 0.44359, - "ndcg_at_1000": 0.47015, - "map_at_1": 0.24501, - "map_at_3": 0.30857, - "map_at_5": 0.32307, - "map_at_10": 0.33582, - "map_at_20": 0.34229, - "map_at_100": 0.34801, - "map_at_1000": 0.34936, - "recall_at_1": 0.24501, - "recall_at_3": 0.37393, - "recall_at_5": 0.42701, - "recall_at_10": 0.5047, - "recall_at_20": 0.57451, - "recall_at_100": 0.72912, - "recall_at_1000": 0.90643, - "precision_at_1": 0.29757, - "precision_at_3": 0.16738, - "precision_at_5": 0.11788, - "precision_at_10": 0.07282, - "precision_at_20": 0.04349, - "precision_at_100": 0.01209, - "precision_at_1000": 0.00172, - "mrr_at_1": 0.297568, - "mrr_at_3": 0.360515, - "mrr_at_5": 0.37432, - "mrr_at_10": 0.384942, - "mrr_at_20": 0.389932, - "mrr_at_100": 0.393632, - "mrr_at_1000": 0.394271, - "nauc_ndcg_at_1_max": 0.35163, - "nauc_ndcg_at_1_std": -0.04949, - "nauc_ndcg_at_1_diff1": 0.506784, - "nauc_ndcg_at_3_max": 0.360879, - "nauc_ndcg_at_3_std": -0.032672, - "nauc_ndcg_at_3_diff1": 0.472847, - "nauc_ndcg_at_5_max": 0.364638, - "nauc_ndcg_at_5_std": -0.016767, - "nauc_ndcg_at_5_diff1": 0.466553, - "nauc_ndcg_at_10_max": 0.373779, - "nauc_ndcg_at_10_std": -0.007015, - "nauc_ndcg_at_10_diff1": 0.468114, - "nauc_ndcg_at_20_max": 0.374599, - "nauc_ndcg_at_20_std": -0.000285, - "nauc_ndcg_at_20_diff1": 0.460127, - "nauc_ndcg_at_100_max": 0.376178, - "nauc_ndcg_at_100_std": 0.003917, - "nauc_ndcg_at_100_diff1": 0.459498, - "nauc_ndcg_at_1000_max": 0.378896, - "nauc_ndcg_at_1000_std": 0.011141, - "nauc_ndcg_at_1000_diff1": 0.45977, - "nauc_map_at_1_max": 0.329735, - "nauc_map_at_1_std": -0.06633, - "nauc_map_at_1_diff1": 0.52641, - "nauc_map_at_3_max": 0.352305, - "nauc_map_at_3_std": -0.050847, - "nauc_map_at_3_diff1": 0.496157, - "nauc_map_at_5_max": 0.356133, - "nauc_map_at_5_std": -0.040399, - "nauc_map_at_5_diff1": 0.488895, - "nauc_map_at_10_max": 0.361679, - "nauc_map_at_10_std": -0.03574, - "nauc_map_at_10_diff1": 0.487396, - "nauc_map_at_20_max": 0.362918, - "nauc_map_at_20_std": -0.032796, - "nauc_map_at_20_diff1": 0.484663, - "nauc_map_at_100_max": 0.364498, - "nauc_map_at_100_std": -0.030688, - "nauc_map_at_100_diff1": 0.484524, - "nauc_map_at_1000_max": 0.364623, - "nauc_map_at_1000_std": -0.030062, - "nauc_map_at_1000_diff1": 0.484259, - "nauc_recall_at_1_max": 0.329735, - "nauc_recall_at_1_std": -0.06633, - "nauc_recall_at_1_diff1": 0.52641, - "nauc_recall_at_3_max": 0.34925, - "nauc_recall_at_3_std": -0.020609, - "nauc_recall_at_3_diff1": 0.441485, - "nauc_recall_at_5_max": 0.347606, - "nauc_recall_at_5_std": 0.016887, - "nauc_recall_at_5_diff1": 0.414227, - "nauc_recall_at_10_max": 0.3657, - "nauc_recall_at_10_std": 0.055427, - "nauc_recall_at_10_diff1": 0.409419, - "nauc_recall_at_20_max": 0.366151, - "nauc_recall_at_20_std": 0.091305, - "nauc_recall_at_20_diff1": 0.372562, - "nauc_recall_at_100_max": 0.375285, - "nauc_recall_at_100_std": 0.138761, - "nauc_recall_at_100_diff1": 0.355754, - "nauc_recall_at_1000_max": 0.485408, - "nauc_recall_at_1000_std": 0.470295, - "nauc_recall_at_1000_diff1": 0.311568, - "nauc_precision_at_1_max": 0.35163, - "nauc_precision_at_1_std": -0.04949, - "nauc_precision_at_1_diff1": 0.506784, - "nauc_precision_at_3_max": 0.332905, - "nauc_precision_at_3_std": -0.016148, - "nauc_precision_at_3_diff1": 0.331776, - "nauc_precision_at_5_max": 0.333257, - "nauc_precision_at_5_std": 0.04402, - "nauc_precision_at_5_diff1": 0.272371, - "nauc_precision_at_10_max": 0.307887, - "nauc_precision_at_10_std": 0.070138, - "nauc_precision_at_10_diff1": 0.20856, - "nauc_precision_at_20_max": 0.27621, - "nauc_precision_at_20_std": 0.099861, - "nauc_precision_at_20_diff1": 0.131662, - "nauc_precision_at_100_max": 0.176654, - "nauc_precision_at_100_std": 0.126523, - "nauc_precision_at_100_diff1": 0.007577, - "nauc_precision_at_1000_max": -0.000854, - "nauc_precision_at_1000_std": 0.095769, - "nauc_precision_at_1000_diff1": -0.16308, - "nauc_mrr_at_1_max": 0.35163, - "nauc_mrr_at_1_std": -0.04949, - "nauc_mrr_at_1_diff1": 0.506784, - "nauc_mrr_at_3_max": 0.363793, - "nauc_mrr_at_3_std": -0.03288, - "nauc_mrr_at_3_diff1": 0.469367, - "nauc_mrr_at_5_max": 0.364904, - "nauc_mrr_at_5_std": -0.021184, - "nauc_mrr_at_5_diff1": 0.463741, - "nauc_mrr_at_10_max": 0.368009, - "nauc_mrr_at_10_std": -0.016722, - "nauc_mrr_at_10_diff1": 0.464536, - "nauc_mrr_at_20_max": 0.368492, - "nauc_mrr_at_20_std": -0.015649, - "nauc_mrr_at_20_diff1": 0.462973, - "nauc_mrr_at_100_max": 0.367136, - "nauc_mrr_at_100_std": -0.016903, - "nauc_mrr_at_100_diff1": 0.463234, - "nauc_mrr_at_1000_max": 0.367098, - "nauc_mrr_at_1000_std": -0.016532, - "nauc_mrr_at_1000_diff1": 0.463207, - "main_score": 0.39114, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 27.28526473045349, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackEnglishRetrieval.json b/beir/CQADupstackEnglishRetrieval.json deleted file mode 100644 index 1eca0d7a0bec63dbefc8aec97afb8461cdf281ae..0000000000000000000000000000000000000000 --- a/beir/CQADupstackEnglishRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0", - "task_name": "CQADupstackEnglishRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.32229, - "ndcg_at_3": 0.35352, - "ndcg_at_5": 0.36907, - "ndcg_at_10": 0.38867, - "ndcg_at_20": 0.40624, - "ndcg_at_100": 0.43169, - "ndcg_at_1000": 0.45411, - "map_at_1": 0.25286, - "map_at_3": 0.31391, - "map_at_5": 0.3276, - "map_at_10": 0.33835, - "map_at_20": 0.34453, - "map_at_100": 0.34926, - "map_at_1000": 0.35039, - "recall_at_1": 0.25286, - "recall_at_3": 0.36961, - "recall_at_5": 0.4154, - "recall_at_10": 0.47543, - "recall_at_20": 0.53979, - "recall_at_100": 0.65744, - "recall_at_1000": 0.80437, - "precision_at_1": 0.32229, - "precision_at_3": 0.17113, - "precision_at_5": 0.11962, - "precision_at_10": 0.0721, - "precision_at_20": 0.04287, - "precision_at_100": 0.01177, - "precision_at_1000": 0.00165, - "mrr_at_1": 0.322293, - "mrr_at_3": 0.377707, - "mrr_at_5": 0.388917, - "mrr_at_10": 0.398009, - "mrr_at_20": 0.402384, - "mrr_at_100": 0.40498, - "mrr_at_1000": 0.40545, - "nauc_ndcg_at_1_max": 0.386829, - "nauc_ndcg_at_1_std": 0.074424, - "nauc_ndcg_at_1_diff1": 0.528493, - "nauc_ndcg_at_3_max": 0.370654, - "nauc_ndcg_at_3_std": 0.054984, - "nauc_ndcg_at_3_diff1": 0.466131, - "nauc_ndcg_at_5_max": 0.36923, - "nauc_ndcg_at_5_std": 0.061408, - "nauc_ndcg_at_5_diff1": 0.457153, - "nauc_ndcg_at_10_max": 0.365462, - "nauc_ndcg_at_10_std": 0.062711, - "nauc_ndcg_at_10_diff1": 0.455885, - "nauc_ndcg_at_20_max": 0.364087, - "nauc_ndcg_at_20_std": 0.064921, - "nauc_ndcg_at_20_diff1": 0.453433, - "nauc_ndcg_at_100_max": 0.366238, - "nauc_ndcg_at_100_std": 0.083159, - "nauc_ndcg_at_100_diff1": 0.447424, - "nauc_ndcg_at_1000_max": 0.37042, - "nauc_ndcg_at_1000_std": 0.089336, - "nauc_ndcg_at_1000_diff1": 0.448673, - "nauc_map_at_1_max": 0.332074, - "nauc_map_at_1_std": 0.021052, - "nauc_map_at_1_diff1": 0.546382, - "nauc_map_at_3_max": 0.352336, - "nauc_map_at_3_std": 0.025225, - "nauc_map_at_3_diff1": 0.49159, - "nauc_map_at_5_max": 0.355997, - "nauc_map_at_5_std": 0.037594, - "nauc_map_at_5_diff1": 0.484115, - "nauc_map_at_10_max": 0.358431, - "nauc_map_at_10_std": 0.042814, - "nauc_map_at_10_diff1": 0.482818, - "nauc_map_at_20_max": 0.359632, - "nauc_map_at_20_std": 0.046509, - "nauc_map_at_20_diff1": 0.481984, - "nauc_map_at_100_max": 0.361803, - "nauc_map_at_100_std": 0.051838, - "nauc_map_at_100_diff1": 0.480556, - "nauc_map_at_1000_max": 0.362442, - "nauc_map_at_1000_std": 0.052821, - "nauc_map_at_1000_diff1": 0.480396, - "nauc_recall_at_1_max": 0.332074, - "nauc_recall_at_1_std": 0.021052, - "nauc_recall_at_1_diff1": 0.546382, - "nauc_recall_at_3_max": 0.33786, - "nauc_recall_at_3_std": 0.022159, - "nauc_recall_at_3_diff1": 0.421871, - "nauc_recall_at_5_max": 0.336662, - "nauc_recall_at_5_std": 0.046278, - "nauc_recall_at_5_diff1": 0.393118, - "nauc_recall_at_10_max": 0.322503, - "nauc_recall_at_10_std": 0.05947, - "nauc_recall_at_10_diff1": 0.37952, - "nauc_recall_at_20_max": 0.312259, - "nauc_recall_at_20_std": 0.068895, - "nauc_recall_at_20_diff1": 0.355049, - "nauc_recall_at_100_max": 0.303406, - "nauc_recall_at_100_std": 0.156142, - "nauc_recall_at_100_diff1": 0.295623, - "nauc_recall_at_1000_max": 0.305454, - "nauc_recall_at_1000_std": 0.214645, - "nauc_recall_at_1000_diff1": 0.271848, - "nauc_precision_at_1_max": 0.386829, - "nauc_precision_at_1_std": 0.074424, - "nauc_precision_at_1_diff1": 0.528493, - "nauc_precision_at_3_max": 0.370629, - "nauc_precision_at_3_std": 0.114377, - "nauc_precision_at_3_diff1": 0.322216, - "nauc_precision_at_5_max": 0.356068, - "nauc_precision_at_5_std": 0.161786, - "nauc_precision_at_5_diff1": 0.256865, - "nauc_precision_at_10_max": 0.336078, - "nauc_precision_at_10_std": 0.188689, - "nauc_precision_at_10_diff1": 0.195934, - "nauc_precision_at_20_max": 0.294667, - "nauc_precision_at_20_std": 0.210742, - "nauc_precision_at_20_diff1": 0.13482, - "nauc_precision_at_100_max": 0.229317, - "nauc_precision_at_100_std": 0.273075, - "nauc_precision_at_100_diff1": -0.000666, - "nauc_precision_at_1000_max": 0.139545, - "nauc_precision_at_1000_std": 0.264182, - "nauc_precision_at_1000_diff1": -0.103752, - "nauc_mrr_at_1_max": 0.386829, - "nauc_mrr_at_1_std": 0.074424, - "nauc_mrr_at_1_diff1": 0.528493, - "nauc_mrr_at_3_max": 0.387082, - "nauc_mrr_at_3_std": 0.075774, - "nauc_mrr_at_3_diff1": 0.478237, - "nauc_mrr_at_5_max": 0.385714, - "nauc_mrr_at_5_std": 0.078757, - "nauc_mrr_at_5_diff1": 0.46983, - "nauc_mrr_at_10_max": 0.385054, - "nauc_mrr_at_10_std": 0.079545, - "nauc_mrr_at_10_diff1": 0.469465, - "nauc_mrr_at_20_max": 0.383847, - "nauc_mrr_at_20_std": 0.079177, - "nauc_mrr_at_20_diff1": 0.468517, - "nauc_mrr_at_100_max": 0.38363, - "nauc_mrr_at_100_std": 0.081133, - "nauc_mrr_at_100_diff1": 0.467934, - "nauc_mrr_at_1000_max": 0.383753, - "nauc_mrr_at_1000_std": 0.08116, - "nauc_mrr_at_1000_diff1": 0.468037, - "main_score": 0.38867, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 41.29191493988037, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackGamingRetrieval.json b/beir/CQADupstackGamingRetrieval.json deleted file mode 100644 index dbd255293cbcefcb9d72290ec3ceda0d5519b53b..0000000000000000000000000000000000000000 --- a/beir/CQADupstackGamingRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "4885aa143210c98657558c04aaf3dc47cfb54340", - "task_name": "CQADupstackGamingRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.37555, - "ndcg_at_3": 0.43752, - "ndcg_at_5": 0.4595, - "ndcg_at_10": 0.48593, - "ndcg_at_20": 0.50274, - "ndcg_at_100": 0.52922, - "ndcg_at_1000": 0.5442, - "map_at_1": 0.32732, - "map_at_3": 0.40392, - "map_at_5": 0.41928, - "map_at_10": 0.43189, - "map_at_20": 0.4374, - "map_at_100": 0.44174, - "map_at_1000": 0.44242, - "recall_at_1": 0.32732, - "recall_at_3": 0.48, - "recall_at_5": 0.53475, - "recall_at_10": 0.61265, - "recall_at_20": 0.67586, - "recall_at_100": 0.80604, - "recall_at_1000": 0.91408, - "precision_at_1": 0.37555, - "precision_at_3": 0.19645, - "precision_at_5": 0.13455, - "precision_at_10": 0.079, - "precision_at_20": 0.04436, - "precision_at_100": 0.0109, - "precision_at_1000": 0.00127, - "mrr_at_1": 0.375549, - "mrr_at_3": 0.443574, - "mrr_at_5": 0.455455, - "mrr_at_10": 0.465721, - "mrr_at_20": 0.469989, - "mrr_at_100": 0.473172, - "mrr_at_1000": 0.473553, - "nauc_ndcg_at_1_max": 0.34123, - "nauc_ndcg_at_1_std": -0.017841, - "nauc_ndcg_at_1_diff1": 0.518322, - "nauc_ndcg_at_3_max": 0.35477, - "nauc_ndcg_at_3_std": -0.012643, - "nauc_ndcg_at_3_diff1": 0.475991, - "nauc_ndcg_at_5_max": 0.362862, - "nauc_ndcg_at_5_std": -3.8e-05, - "nauc_ndcg_at_5_diff1": 0.465251, - "nauc_ndcg_at_10_max": 0.367788, - "nauc_ndcg_at_10_std": 0.008117, - "nauc_ndcg_at_10_diff1": 0.467285, - "nauc_ndcg_at_20_max": 0.375701, - "nauc_ndcg_at_20_std": 0.016506, - "nauc_ndcg_at_20_diff1": 0.46824, - "nauc_ndcg_at_100_max": 0.378108, - "nauc_ndcg_at_100_std": 0.029329, - "nauc_ndcg_at_100_diff1": 0.466243, - "nauc_ndcg_at_1000_max": 0.378672, - "nauc_ndcg_at_1000_std": 0.029359, - "nauc_ndcg_at_1000_diff1": 0.467787, - "nauc_map_at_1_max": 0.309187, - "nauc_map_at_1_std": -0.026286, - "nauc_map_at_1_diff1": 0.51255, - "nauc_map_at_3_max": 0.339493, - "nauc_map_at_3_std": -0.024583, - "nauc_map_at_3_diff1": 0.485462, - "nauc_map_at_5_max": 0.34567, - "nauc_map_at_5_std": -0.015681, - "nauc_map_at_5_diff1": 0.478419, - "nauc_map_at_10_max": 0.350466, - "nauc_map_at_10_std": -0.010182, - "nauc_map_at_10_diff1": 0.479976, - "nauc_map_at_20_max": 0.354014, - "nauc_map_at_20_std": -0.006443, - "nauc_map_at_20_diff1": 0.480216, - "nauc_map_at_100_max": 0.355061, - "nauc_map_at_100_std": -0.003215, - "nauc_map_at_100_diff1": 0.479392, - "nauc_map_at_1000_max": 0.35521, - "nauc_map_at_1000_std": -0.003054, - "nauc_map_at_1000_diff1": 0.479491, - "nauc_recall_at_1_max": 0.309187, - "nauc_recall_at_1_std": -0.026286, - "nauc_recall_at_1_diff1": 0.51255, - "nauc_recall_at_3_max": 0.34792, - "nauc_recall_at_3_std": -0.017638, - "nauc_recall_at_3_diff1": 0.440913, - "nauc_recall_at_5_max": 0.364564, - "nauc_recall_at_5_std": 0.014875, - "nauc_recall_at_5_diff1": 0.409842, - "nauc_recall_at_10_max": 0.372517, - "nauc_recall_at_10_std": 0.038871, - "nauc_recall_at_10_diff1": 0.405621, - "nauc_recall_at_20_max": 0.408612, - "nauc_recall_at_20_std": 0.080092, - "nauc_recall_at_20_diff1": 0.400661, - "nauc_recall_at_100_max": 0.431074, - "nauc_recall_at_100_std": 0.195316, - "nauc_recall_at_100_diff1": 0.363097, - "nauc_recall_at_1000_max": 0.493019, - "nauc_recall_at_1000_std": 0.343645, - "nauc_recall_at_1000_diff1": 0.316154, - "nauc_precision_at_1_max": 0.34123, - "nauc_precision_at_1_std": -0.017841, - "nauc_precision_at_1_diff1": 0.518322, - "nauc_precision_at_3_max": 0.358498, - "nauc_precision_at_3_std": 0.027106, - "nauc_precision_at_3_diff1": 0.347859, - "nauc_precision_at_5_max": 0.34631, - "nauc_precision_at_5_std": 0.066776, - "nauc_precision_at_5_diff1": 0.274746, - "nauc_precision_at_10_max": 0.338077, - "nauc_precision_at_10_std": 0.112782, - "nauc_precision_at_10_diff1": 0.220539, - "nauc_precision_at_20_max": 0.340661, - "nauc_precision_at_20_std": 0.161584, - "nauc_precision_at_20_diff1": 0.172042, - "nauc_precision_at_100_max": 0.287824, - "nauc_precision_at_100_std": 0.236474, - "nauc_precision_at_100_diff1": 0.044878, - "nauc_precision_at_1000_max": 0.236745, - "nauc_precision_at_1000_std": 0.224593, - "nauc_precision_at_1000_diff1": -0.026638, - "nauc_mrr_at_1_max": 0.34123, - "nauc_mrr_at_1_std": -0.017841, - "nauc_mrr_at_1_diff1": 0.518322, - "nauc_mrr_at_3_max": 0.36619, - "nauc_mrr_at_3_std": -0.003463, - "nauc_mrr_at_3_diff1": 0.487033, - "nauc_mrr_at_5_max": 0.369399, - "nauc_mrr_at_5_std": 0.001776, - "nauc_mrr_at_5_diff1": 0.481842, - "nauc_mrr_at_10_max": 0.370071, - "nauc_mrr_at_10_std": 0.003426, - "nauc_mrr_at_10_diff1": 0.483223, - "nauc_mrr_at_20_max": 0.371624, - "nauc_mrr_at_20_std": 0.005601, - "nauc_mrr_at_20_diff1": 0.483756, - "nauc_mrr_at_100_max": 0.371384, - "nauc_mrr_at_100_std": 0.006386, - "nauc_mrr_at_100_diff1": 0.483777, - "nauc_mrr_at_1000_max": 0.371237, - "nauc_mrr_at_1000_std": 0.00627, - "nauc_mrr_at_1000_diff1": 0.483789, - "main_score": 0.48593, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 45.477328062057495, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackGisRetrieval.json b/beir/CQADupstackGisRetrieval.json deleted file mode 100644 index 5f1ce55b5d826343a6244d9f204ef35ace41f520..0000000000000000000000000000000000000000 --- a/beir/CQADupstackGisRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "5003b3064772da1887988e05400cf3806fe491f2", - "task_name": "CQADupstackGisRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.19322, - "ndcg_at_3": 0.2464, - "ndcg_at_5": 0.26398, - "ndcg_at_10": 0.28629, - "ndcg_at_20": 0.30424, - "ndcg_at_100": 0.33635, - "ndcg_at_1000": 0.36296, - "map_at_1": 0.1796, - "map_at_3": 0.22722, - "map_at_5": 0.23737, - "map_at_10": 0.24671, - "map_at_20": 0.25173, - "map_at_100": 0.25633, - "map_at_1000": 0.25725, - "recall_at_1": 0.1796, - "recall_at_3": 0.28451, - "recall_at_5": 0.32652, - "recall_at_10": 0.39283, - "recall_at_20": 0.46066, - "recall_at_100": 0.62619, - "recall_at_1000": 0.83354, - "precision_at_1": 0.19322, - "precision_at_3": 0.10395, - "precision_at_5": 0.07254, - "precision_at_10": 0.04475, - "precision_at_20": 0.02672, - "precision_at_100": 0.0074, - "precision_at_1000": 0.00101, - "mrr_at_1": 0.19322, - "mrr_at_3": 0.243315, - "mrr_at_5": 0.25371, - "mrr_at_10": 0.263167, - "mrr_at_20": 0.268103, - "mrr_at_100": 0.27234, - "mrr_at_1000": 0.273059, - "nauc_ndcg_at_1_max": 0.3509, - "nauc_ndcg_at_1_std": -0.028343, - "nauc_ndcg_at_1_diff1": 0.4556, - "nauc_ndcg_at_3_max": 0.332288, - "nauc_ndcg_at_3_std": -0.001513, - "nauc_ndcg_at_3_diff1": 0.383786, - "nauc_ndcg_at_5_max": 0.323156, - "nauc_ndcg_at_5_std": -0.002329, - "nauc_ndcg_at_5_diff1": 0.375401, - "nauc_ndcg_at_10_max": 0.319542, - "nauc_ndcg_at_10_std": 0.001314, - "nauc_ndcg_at_10_diff1": 0.371061, - "nauc_ndcg_at_20_max": 0.322781, - "nauc_ndcg_at_20_std": 0.007944, - "nauc_ndcg_at_20_diff1": 0.359749, - "nauc_ndcg_at_100_max": 0.321408, - "nauc_ndcg_at_100_std": 0.021193, - "nauc_ndcg_at_100_diff1": 0.353214, - "nauc_ndcg_at_1000_max": 0.333213, - "nauc_ndcg_at_1000_std": 0.028037, - "nauc_ndcg_at_1000_diff1": 0.362077, - "nauc_map_at_1_max": 0.334646, - "nauc_map_at_1_std": -0.046069, - "nauc_map_at_1_diff1": 0.475264, - "nauc_map_at_3_max": 0.330952, - "nauc_map_at_3_std": -0.014788, - "nauc_map_at_3_diff1": 0.405655, - "nauc_map_at_5_max": 0.326424, - "nauc_map_at_5_std": -0.015209, - "nauc_map_at_5_diff1": 0.400335, - "nauc_map_at_10_max": 0.325464, - "nauc_map_at_10_std": -0.014892, - "nauc_map_at_10_diff1": 0.398555, - "nauc_map_at_20_max": 0.326796, - "nauc_map_at_20_std": -0.013008, - "nauc_map_at_20_diff1": 0.395344, - "nauc_map_at_100_max": 0.326354, - "nauc_map_at_100_std": -0.01094, - "nauc_map_at_100_diff1": 0.394208, - "nauc_map_at_1000_max": 0.326999, - "nauc_map_at_1000_std": -0.01055, - "nauc_map_at_1000_diff1": 0.394564, - "nauc_recall_at_1_max": 0.334646, - "nauc_recall_at_1_std": -0.046069, - "nauc_recall_at_1_diff1": 0.475264, - "nauc_recall_at_3_max": 0.312987, - "nauc_recall_at_3_std": 0.018809, - "nauc_recall_at_3_diff1": 0.329535, - "nauc_recall_at_5_max": 0.291171, - "nauc_recall_at_5_std": 0.015371, - "nauc_recall_at_5_diff1": 0.310569, - "nauc_recall_at_10_max": 0.278967, - "nauc_recall_at_10_std": 0.027129, - "nauc_recall_at_10_diff1": 0.297162, - "nauc_recall_at_20_max": 0.285044, - "nauc_recall_at_20_std": 0.048128, - "nauc_recall_at_20_diff1": 0.255835, - "nauc_recall_at_100_max": 0.265933, - "nauc_recall_at_100_std": 0.119517, - "nauc_recall_at_100_diff1": 0.202748, - "nauc_recall_at_1000_max": 0.373161, - "nauc_recall_at_1000_std": 0.285925, - "nauc_recall_at_1000_diff1": 0.204139, - "nauc_precision_at_1_max": 0.3509, - "nauc_precision_at_1_std": -0.028343, - "nauc_precision_at_1_diff1": 0.4556, - "nauc_precision_at_3_max": 0.349653, - "nauc_precision_at_3_std": 0.037263, - "nauc_precision_at_3_diff1": 0.316443, - "nauc_precision_at_5_max": 0.337659, - "nauc_precision_at_5_std": 0.038751, - "nauc_precision_at_5_diff1": 0.294004, - "nauc_precision_at_10_max": 0.327502, - "nauc_precision_at_10_std": 0.043505, - "nauc_precision_at_10_diff1": 0.285592, - "nauc_precision_at_20_max": 0.331996, - "nauc_precision_at_20_std": 0.076739, - "nauc_precision_at_20_diff1": 0.232499, - "nauc_precision_at_100_max": 0.299848, - "nauc_precision_at_100_std": 0.144632, - "nauc_precision_at_100_diff1": 0.154486, - "nauc_precision_at_1000_max": 0.28517, - "nauc_precision_at_1000_std": 0.204971, - "nauc_precision_at_1000_diff1": 0.090991, - "nauc_mrr_at_1_max": 0.3509, - "nauc_mrr_at_1_std": -0.028343, - "nauc_mrr_at_1_diff1": 0.4556, - "nauc_mrr_at_3_max": 0.343039, - "nauc_mrr_at_3_std": 0.003194, - "nauc_mrr_at_3_diff1": 0.395025, - "nauc_mrr_at_5_max": 0.338967, - "nauc_mrr_at_5_std": 0.002291, - "nauc_mrr_at_5_diff1": 0.389964, - "nauc_mrr_at_10_max": 0.337191, - "nauc_mrr_at_10_std": 0.004711, - "nauc_mrr_at_10_diff1": 0.388168, - "nauc_mrr_at_20_max": 0.338323, - "nauc_mrr_at_20_std": 0.006217, - "nauc_mrr_at_20_diff1": 0.385364, - "nauc_mrr_at_100_max": 0.337901, - "nauc_mrr_at_100_std": 0.007768, - "nauc_mrr_at_100_diff1": 0.384638, - "nauc_mrr_at_1000_max": 0.338161, - "nauc_mrr_at_1000_std": 0.00777, - "nauc_mrr_at_1000_diff1": 0.384956, - "main_score": 0.28629, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 75.08722925186157, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackMathematicaRetrieval.json b/beir/CQADupstackMathematicaRetrieval.json deleted file mode 100644 index 1fe0a3691947e08b4146f4c8c93eedfa40b06b0f..0000000000000000000000000000000000000000 --- a/beir/CQADupstackMathematicaRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "90fceea13679c63fe563ded68f3b6f06e50061de", - "task_name": "CQADupstackMathematicaRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.1592, - "ndcg_at_3": 0.18727, - "ndcg_at_5": 0.20573, - "ndcg_at_10": 0.23092, - "ndcg_at_20": 0.24856, - "ndcg_at_100": 0.28661, - "ndcg_at_1000": 0.31839, - "map_at_1": 0.126, - "map_at_3": 0.1645, - "map_at_5": 0.17543, - "map_at_10": 0.18641, - "map_at_20": 0.19162, - "map_at_100": 0.19739, - "map_at_1000": 0.19864, - "recall_at_1": 0.126, - "recall_at_3": 0.20946, - "recall_at_5": 0.2564, - "recall_at_10": 0.32953, - "recall_at_20": 0.392, - "recall_at_100": 0.57855, - "recall_at_1000": 0.80748, - "precision_at_1": 0.1592, - "precision_at_3": 0.08955, - "precision_at_5": 0.06592, - "precision_at_10": 0.04366, - "precision_at_20": 0.02668, - "precision_at_100": 0.00832, - "precision_at_1000": 0.00124, - "mrr_at_1": 0.159204, - "mrr_at_3": 0.204187, - "mrr_at_5": 0.216563, - "mrr_at_10": 0.228361, - "mrr_at_20": 0.233146, - "mrr_at_100": 0.237804, - "mrr_at_1000": 0.238608, - "nauc_ndcg_at_1_max": 0.173331, - "nauc_ndcg_at_1_std": -0.070922, - "nauc_ndcg_at_1_diff1": 0.313574, - "nauc_ndcg_at_3_max": 0.149226, - "nauc_ndcg_at_3_std": -0.044527, - "nauc_ndcg_at_3_diff1": 0.231067, - "nauc_ndcg_at_5_max": 0.147868, - "nauc_ndcg_at_5_std": -0.033304, - "nauc_ndcg_at_5_diff1": 0.21667, - "nauc_ndcg_at_10_max": 0.136413, - "nauc_ndcg_at_10_std": -0.030496, - "nauc_ndcg_at_10_diff1": 0.19898, - "nauc_ndcg_at_20_max": 0.140685, - "nauc_ndcg_at_20_std": -0.020516, - "nauc_ndcg_at_20_diff1": 0.199934, - "nauc_ndcg_at_100_max": 0.159372, - "nauc_ndcg_at_100_std": 0.004912, - "nauc_ndcg_at_100_diff1": 0.205706, - "nauc_ndcg_at_1000_max": 0.158797, - "nauc_ndcg_at_1000_std": 0.000364, - "nauc_ndcg_at_1000_diff1": 0.203824, - "nauc_map_at_1_max": 0.176756, - "nauc_map_at_1_std": -0.061246, - "nauc_map_at_1_diff1": 0.294157, - "nauc_map_at_3_max": 0.151356, - "nauc_map_at_3_std": -0.048515, - "nauc_map_at_3_diff1": 0.241311, - "nauc_map_at_5_max": 0.151841, - "nauc_map_at_5_std": -0.042908, - "nauc_map_at_5_diff1": 0.23564, - "nauc_map_at_10_max": 0.146342, - "nauc_map_at_10_std": -0.040837, - "nauc_map_at_10_diff1": 0.226564, - "nauc_map_at_20_max": 0.148085, - "nauc_map_at_20_std": -0.036882, - "nauc_map_at_20_diff1": 0.226222, - "nauc_map_at_100_max": 0.151301, - "nauc_map_at_100_std": -0.032596, - "nauc_map_at_100_diff1": 0.226917, - "nauc_map_at_1000_max": 0.151171, - "nauc_map_at_1000_std": -0.032835, - "nauc_map_at_1000_diff1": 0.226596, - "nauc_recall_at_1_max": 0.176756, - "nauc_recall_at_1_std": -0.061246, - "nauc_recall_at_1_diff1": 0.294157, - "nauc_recall_at_3_max": 0.129382, - "nauc_recall_at_3_std": -0.026964, - "nauc_recall_at_3_diff1": 0.182064, - "nauc_recall_at_5_max": 0.126607, - "nauc_recall_at_5_std": -0.010177, - "nauc_recall_at_5_diff1": 0.159092, - "nauc_recall_at_10_max": 0.100506, - "nauc_recall_at_10_std": -0.007447, - "nauc_recall_at_10_diff1": 0.123114, - "nauc_recall_at_20_max": 0.109598, - "nauc_recall_at_20_std": 0.022768, - "nauc_recall_at_20_diff1": 0.126061, - "nauc_recall_at_100_max": 0.192576, - "nauc_recall_at_100_std": 0.141059, - "nauc_recall_at_100_diff1": 0.148286, - "nauc_recall_at_1000_max": 0.2255, - "nauc_recall_at_1000_std": 0.2101, - "nauc_recall_at_1000_diff1": 0.097776, - "nauc_precision_at_1_max": 0.173331, - "nauc_precision_at_1_std": -0.070922, - "nauc_precision_at_1_diff1": 0.313574, - "nauc_precision_at_3_max": 0.128165, - "nauc_precision_at_3_std": -0.037662, - "nauc_precision_at_3_diff1": 0.186901, - "nauc_precision_at_5_max": 0.135809, - "nauc_precision_at_5_std": -0.01395, - "nauc_precision_at_5_diff1": 0.16978, - "nauc_precision_at_10_max": 0.111158, - "nauc_precision_at_10_std": -0.011867, - "nauc_precision_at_10_diff1": 0.126989, - "nauc_precision_at_20_max": 0.111932, - "nauc_precision_at_20_std": 0.005621, - "nauc_precision_at_20_diff1": 0.115231, - "nauc_precision_at_100_max": 0.106532, - "nauc_precision_at_100_std": 0.058503, - "nauc_precision_at_100_diff1": 0.072954, - "nauc_precision_at_1000_max": 0.065429, - "nauc_precision_at_1000_std": -0.003839, - "nauc_precision_at_1000_diff1": 0.017772, - "nauc_mrr_at_1_max": 0.173331, - "nauc_mrr_at_1_std": -0.070922, - "nauc_mrr_at_1_diff1": 0.313574, - "nauc_mrr_at_3_max": 0.156386, - "nauc_mrr_at_3_std": -0.049645, - "nauc_mrr_at_3_diff1": 0.249522, - "nauc_mrr_at_5_max": 0.154441, - "nauc_mrr_at_5_std": -0.043804, - "nauc_mrr_at_5_diff1": 0.240661, - "nauc_mrr_at_10_max": 0.14987, - "nauc_mrr_at_10_std": -0.044188, - "nauc_mrr_at_10_diff1": 0.23307, - "nauc_mrr_at_20_max": 0.151533, - "nauc_mrr_at_20_std": -0.042259, - "nauc_mrr_at_20_diff1": 0.234266, - "nauc_mrr_at_100_max": 0.153064, - "nauc_mrr_at_100_std": -0.039756, - "nauc_mrr_at_100_diff1": 0.234561, - "nauc_mrr_at_1000_max": 0.152904, - "nauc_mrr_at_1000_std": -0.039874, - "nauc_mrr_at_1000_diff1": 0.234452, - "main_score": 0.23092, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 45.40352916717529, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackPhysicsRetrieval.json b/beir/CQADupstackPhysicsRetrieval.json deleted file mode 100644 index e5a5df6f6783c94fc72152a40fb39b44f9e67e13..0000000000000000000000000000000000000000 --- a/beir/CQADupstackPhysicsRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4", - "task_name": "CQADupstackPhysicsRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.27238, - "ndcg_at_3": 0.32157, - "ndcg_at_5": 0.34044, - "ndcg_at_10": 0.37013, - "ndcg_at_20": 0.39337, - "ndcg_at_100": 0.42811, - "ndcg_at_1000": 0.45276, - "map_at_1": 0.22537, - "map_at_3": 0.2879, - "map_at_5": 0.3009, - "map_at_10": 0.31509, - "map_at_20": 0.32265, - "map_at_100": 0.32835, - "map_at_1000": 0.3296, - "recall_at_1": 0.22537, - "recall_at_3": 0.35122, - "recall_at_5": 0.39946, - "recall_at_10": 0.48803, - "recall_at_20": 0.56927, - "recall_at_100": 0.73288, - "recall_at_1000": 0.89725, - "precision_at_1": 0.27238, - "precision_at_3": 0.15271, - "precision_at_5": 0.1078, - "precision_at_10": 0.06756, - "precision_at_20": 0.04139, - "precision_at_100": 0.01153, - "precision_at_1000": 0.00154, - "mrr_at_1": 0.272377, - "mrr_at_3": 0.336702, - "mrr_at_5": 0.350369, - "mrr_at_10": 0.36334, - "mrr_at_20": 0.368893, - "mrr_at_100": 0.372666, - "mrr_at_1000": 0.373281, - "nauc_ndcg_at_1_max": 0.434963, - "nauc_ndcg_at_1_std": -0.016834, - "nauc_ndcg_at_1_diff1": 0.587191, - "nauc_ndcg_at_3_max": 0.376659, - "nauc_ndcg_at_3_std": -0.021128, - "nauc_ndcg_at_3_diff1": 0.498754, - "nauc_ndcg_at_5_max": 0.371104, - "nauc_ndcg_at_5_std": -0.006056, - "nauc_ndcg_at_5_diff1": 0.484702, - "nauc_ndcg_at_10_max": 0.371137, - "nauc_ndcg_at_10_std": 0.000755, - "nauc_ndcg_at_10_diff1": 0.472322, - "nauc_ndcg_at_20_max": 0.374454, - "nauc_ndcg_at_20_std": 0.009248, - "nauc_ndcg_at_20_diff1": 0.471307, - "nauc_ndcg_at_100_max": 0.385982, - "nauc_ndcg_at_100_std": 0.032502, - "nauc_ndcg_at_100_diff1": 0.473894, - "nauc_ndcg_at_1000_max": 0.39129, - "nauc_ndcg_at_1000_std": 0.036103, - "nauc_ndcg_at_1000_diff1": 0.475898, - "nauc_map_at_1_max": 0.387388, - "nauc_map_at_1_std": -0.063459, - "nauc_map_at_1_diff1": 0.595917, - "nauc_map_at_3_max": 0.368626, - "nauc_map_at_3_std": -0.041075, - "nauc_map_at_3_diff1": 0.520431, - "nauc_map_at_5_max": 0.369652, - "nauc_map_at_5_std": -0.028134, - "nauc_map_at_5_diff1": 0.512029, - "nauc_map_at_10_max": 0.373143, - "nauc_map_at_10_std": -0.021212, - "nauc_map_at_10_diff1": 0.505536, - "nauc_map_at_20_max": 0.375873, - "nauc_map_at_20_std": -0.017235, - "nauc_map_at_20_diff1": 0.50533, - "nauc_map_at_100_max": 0.378729, - "nauc_map_at_100_std": -0.013049, - "nauc_map_at_100_diff1": 0.505814, - "nauc_map_at_1000_max": 0.379222, - "nauc_map_at_1000_std": -0.012218, - "nauc_map_at_1000_diff1": 0.505796, - "nauc_recall_at_1_max": 0.387388, - "nauc_recall_at_1_std": -0.063459, - "nauc_recall_at_1_diff1": 0.595917, - "nauc_recall_at_3_max": 0.317313, - "nauc_recall_at_3_std": -0.038849, - "nauc_recall_at_3_diff1": 0.442818, - "nauc_recall_at_5_max": 0.31284, - "nauc_recall_at_5_std": 0.002032, - "nauc_recall_at_5_diff1": 0.407502, - "nauc_recall_at_10_max": 0.310181, - "nauc_recall_at_10_std": 0.029815, - "nauc_recall_at_10_diff1": 0.357645, - "nauc_recall_at_20_max": 0.305273, - "nauc_recall_at_20_std": 0.052793, - "nauc_recall_at_20_diff1": 0.339285, - "nauc_recall_at_100_max": 0.336613, - "nauc_recall_at_100_std": 0.198697, - "nauc_recall_at_100_diff1": 0.314998, - "nauc_recall_at_1000_max": 0.399223, - "nauc_recall_at_1000_std": 0.384869, - "nauc_recall_at_1000_diff1": 0.223792, - "nauc_precision_at_1_max": 0.434963, - "nauc_precision_at_1_std": -0.016834, - "nauc_precision_at_1_diff1": 0.587191, - "nauc_precision_at_3_max": 0.379844, - "nauc_precision_at_3_std": 0.058961, - "nauc_precision_at_3_diff1": 0.369786, - "nauc_precision_at_5_max": 0.367037, - "nauc_precision_at_5_std": 0.113331, - "nauc_precision_at_5_diff1": 0.304295, - "nauc_precision_at_10_max": 0.353315, - "nauc_precision_at_10_std": 0.159411, - "nauc_precision_at_10_diff1": 0.216986, - "nauc_precision_at_20_max": 0.321937, - "nauc_precision_at_20_std": 0.210608, - "nauc_precision_at_20_diff1": 0.15191, - "nauc_precision_at_100_max": 0.261556, - "nauc_precision_at_100_std": 0.287677, - "nauc_precision_at_100_diff1": 0.038748, - "nauc_precision_at_1000_max": 0.144137, - "nauc_precision_at_1000_std": 0.283117, - "nauc_precision_at_1000_diff1": -0.06849, - "nauc_mrr_at_1_max": 0.434963, - "nauc_mrr_at_1_std": -0.016834, - "nauc_mrr_at_1_diff1": 0.587191, - "nauc_mrr_at_3_max": 0.401226, - "nauc_mrr_at_3_std": -0.007172, - "nauc_mrr_at_3_diff1": 0.516348, - "nauc_mrr_at_5_max": 0.402557, - "nauc_mrr_at_5_std": 0.003495, - "nauc_mrr_at_5_diff1": 0.506254, - "nauc_mrr_at_10_max": 0.403164, - "nauc_mrr_at_10_std": 0.006847, - "nauc_mrr_at_10_diff1": 0.501766, - "nauc_mrr_at_20_max": 0.40341, - "nauc_mrr_at_20_std": 0.008421, - "nauc_mrr_at_20_diff1": 0.502319, - "nauc_mrr_at_100_max": 0.403681, - "nauc_mrr_at_100_std": 0.009655, - "nauc_mrr_at_100_diff1": 0.503195, - "nauc_mrr_at_1000_max": 0.403715, - "nauc_mrr_at_1000_std": 0.00956, - "nauc_mrr_at_1000_diff1": 0.503187, - "main_score": 0.37013, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 59.619898080825806, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackProgrammersRetrieval.json b/beir/CQADupstackProgrammersRetrieval.json deleted file mode 100644 index a9c2cae584f43419440cb7a23f6553c3b6d47b4a..0000000000000000000000000000000000000000 --- a/beir/CQADupstackProgrammersRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "6184bc1440d2dbc7612be22b50686b8826d22b32", - "task_name": "CQADupstackProgrammersRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.25228, - "ndcg_at_3": 0.29072, - "ndcg_at_5": 0.31289, - "ndcg_at_10": 0.33489, - "ndcg_at_20": 0.35566, - "ndcg_at_100": 0.39297, - "ndcg_at_1000": 0.41996, - "map_at_1": 0.20288, - "map_at_3": 0.25911, - "map_at_5": 0.27423, - "map_at_10": 0.2851, - "map_at_20": 0.2914, - "map_at_100": 0.29736, - "map_at_1000": 0.29862, - "recall_at_1": 0.20288, - "recall_at_3": 0.31413, - "recall_at_5": 0.37333, - "recall_at_10": 0.43861, - "recall_at_20": 0.51217, - "recall_at_100": 0.6923, - "recall_at_1000": 0.87747, - "precision_at_1": 0.25228, - "precision_at_3": 0.13889, - "precision_at_5": 0.10137, - "precision_at_10": 0.06084, - "precision_at_20": 0.03687, - "precision_at_100": 0.01065, - "precision_at_1000": 0.00146, - "mrr_at_1": 0.252283, - "mrr_at_3": 0.307078, - "mrr_at_5": 0.323002, - "mrr_at_10": 0.332576, - "mrr_at_20": 0.337973, - "mrr_at_100": 0.342415, - "mrr_at_1000": 0.343064, - "nauc_ndcg_at_1_max": 0.401716, - "nauc_ndcg_at_1_std": 0.065067, - "nauc_ndcg_at_1_diff1": 0.392721, - "nauc_ndcg_at_3_max": 0.383229, - "nauc_ndcg_at_3_std": 0.065445, - "nauc_ndcg_at_3_diff1": 0.331411, - "nauc_ndcg_at_5_max": 0.388223, - "nauc_ndcg_at_5_std": 0.079156, - "nauc_ndcg_at_5_diff1": 0.321325, - "nauc_ndcg_at_10_max": 0.382528, - "nauc_ndcg_at_10_std": 0.076964, - "nauc_ndcg_at_10_diff1": 0.319019, - "nauc_ndcg_at_20_max": 0.38324, - "nauc_ndcg_at_20_std": 0.088949, - "nauc_ndcg_at_20_diff1": 0.315701, - "nauc_ndcg_at_100_max": 0.394976, - "nauc_ndcg_at_100_std": 0.112611, - "nauc_ndcg_at_100_diff1": 0.318071, - "nauc_ndcg_at_1000_max": 0.400048, - "nauc_ndcg_at_1000_std": 0.116156, - "nauc_ndcg_at_1000_diff1": 0.320789, - "nauc_map_at_1_max": 0.357305, - "nauc_map_at_1_std": 0.015761, - "nauc_map_at_1_diff1": 0.384366, - "nauc_map_at_3_max": 0.373442, - "nauc_map_at_3_std": 0.047477, - "nauc_map_at_3_diff1": 0.342786, - "nauc_map_at_5_max": 0.37963, - "nauc_map_at_5_std": 0.058431, - "nauc_map_at_5_diff1": 0.336109, - "nauc_map_at_10_max": 0.379757, - "nauc_map_at_10_std": 0.059797, - "nauc_map_at_10_diff1": 0.336136, - "nauc_map_at_20_max": 0.381347, - "nauc_map_at_20_std": 0.064297, - "nauc_map_at_20_diff1": 0.335546, - "nauc_map_at_100_max": 0.383476, - "nauc_map_at_100_std": 0.069208, - "nauc_map_at_100_diff1": 0.335143, - "nauc_map_at_1000_max": 0.383756, - "nauc_map_at_1000_std": 0.069628, - "nauc_map_at_1000_diff1": 0.335194, - "nauc_recall_at_1_max": 0.357305, - "nauc_recall_at_1_std": 0.015761, - "nauc_recall_at_1_diff1": 0.384366, - "nauc_recall_at_3_max": 0.358924, - "nauc_recall_at_3_std": 0.056564, - "nauc_recall_at_3_diff1": 0.303315, - "nauc_recall_at_5_max": 0.367283, - "nauc_recall_at_5_std": 0.09274, - "nauc_recall_at_5_diff1": 0.273828, - "nauc_recall_at_10_max": 0.35053, - "nauc_recall_at_10_std": 0.089863, - "nauc_recall_at_10_diff1": 0.262794, - "nauc_recall_at_20_max": 0.339605, - "nauc_recall_at_20_std": 0.126648, - "nauc_recall_at_20_diff1": 0.242244, - "nauc_recall_at_100_max": 0.380486, - "nauc_recall_at_100_std": 0.249646, - "nauc_recall_at_100_diff1": 0.240527, - "nauc_recall_at_1000_max": 0.487529, - "nauc_recall_at_1000_std": 0.466888, - "nauc_recall_at_1000_diff1": 0.234965, - "nauc_precision_at_1_max": 0.401716, - "nauc_precision_at_1_std": 0.065067, - "nauc_precision_at_1_diff1": 0.392721, - "nauc_precision_at_3_max": 0.403259, - "nauc_precision_at_3_std": 0.142162, - "nauc_precision_at_3_diff1": 0.269316, - "nauc_precision_at_5_max": 0.404385, - "nauc_precision_at_5_std": 0.178013, - "nauc_precision_at_5_diff1": 0.224482, - "nauc_precision_at_10_max": 0.367242, - "nauc_precision_at_10_std": 0.17353, - "nauc_precision_at_10_diff1": 0.183739, - "nauc_precision_at_20_max": 0.327196, - "nauc_precision_at_20_std": 0.202914, - "nauc_precision_at_20_diff1": 0.143319, - "nauc_precision_at_100_max": 0.234403, - "nauc_precision_at_100_std": 0.246467, - "nauc_precision_at_100_diff1": 0.036827, - "nauc_precision_at_1000_max": 0.077956, - "nauc_precision_at_1000_std": 0.148892, - "nauc_precision_at_1000_diff1": -0.060821, - "nauc_mrr_at_1_max": 0.401716, - "nauc_mrr_at_1_std": 0.065067, - "nauc_mrr_at_1_diff1": 0.392721, - "nauc_mrr_at_3_max": 0.401914, - "nauc_mrr_at_3_std": 0.080097, - "nauc_mrr_at_3_diff1": 0.351381, - "nauc_mrr_at_5_max": 0.401255, - "nauc_mrr_at_5_std": 0.088183, - "nauc_mrr_at_5_diff1": 0.342425, - "nauc_mrr_at_10_max": 0.395882, - "nauc_mrr_at_10_std": 0.085347, - "nauc_mrr_at_10_diff1": 0.338442, - "nauc_mrr_at_20_max": 0.394937, - "nauc_mrr_at_20_std": 0.086579, - "nauc_mrr_at_20_diff1": 0.337513, - "nauc_mrr_at_100_max": 0.397282, - "nauc_mrr_at_100_std": 0.088992, - "nauc_mrr_at_100_diff1": 0.339029, - "nauc_mrr_at_1000_max": 0.397251, - "nauc_mrr_at_1000_std": 0.088846, - "nauc_mrr_at_1000_diff1": 0.339196, - "main_score": 0.33489, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 57.37495994567871, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackStatsRetrieval.json b/beir/CQADupstackStatsRetrieval.json deleted file mode 100644 index 07b255ea0bcedccc6c44c4688d302f32670c6630..0000000000000000000000000000000000000000 --- a/beir/CQADupstackStatsRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a", - "task_name": "CQADupstackStatsRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.21166, - "ndcg_at_3": 0.24697, - "ndcg_at_5": 0.26206, - "ndcg_at_10": 0.28144, - "ndcg_at_20": 0.29877, - "ndcg_at_100": 0.32798, - "ndcg_at_1000": 0.35119, - "map_at_1": 0.18672, - "map_at_3": 0.2269, - "map_at_5": 0.23653, - "map_at_10": 0.24469, - "map_at_20": 0.2497, - "map_at_100": 0.25401, - "map_at_1000": 0.25481, - "recall_at_1": 0.18672, - "recall_at_3": 0.27173, - "recall_at_5": 0.30953, - "recall_at_10": 0.3688, - "recall_at_20": 0.43444, - "recall_at_100": 0.58217, - "recall_at_1000": 0.75725, - "precision_at_1": 0.21166, - "precision_at_3": 0.10685, - "precision_at_5": 0.07485, - "precision_at_10": 0.04479, - "precision_at_20": 0.02669, - "precision_at_100": 0.00729, - "precision_at_1000": 0.001, - "mrr_at_1": 0.211656, - "mrr_at_3": 0.255112, - "mrr_at_5": 0.263471, - "mrr_at_10": 0.272333, - "mrr_at_20": 0.276509, - "mrr_at_100": 0.280587, - "mrr_at_1000": 0.28122, - "nauc_ndcg_at_1_max": 0.265288, - "nauc_ndcg_at_1_std": 0.001032, - "nauc_ndcg_at_1_diff1": 0.50655, - "nauc_ndcg_at_3_max": 0.259172, - "nauc_ndcg_at_3_std": 0.0391, - "nauc_ndcg_at_3_diff1": 0.464349, - "nauc_ndcg_at_5_max": 0.253671, - "nauc_ndcg_at_5_std": 0.057907, - "nauc_ndcg_at_5_diff1": 0.453087, - "nauc_ndcg_at_10_max": 0.252675, - "nauc_ndcg_at_10_std": 0.068572, - "nauc_ndcg_at_10_diff1": 0.431656, - "nauc_ndcg_at_20_max": 0.252661, - "nauc_ndcg_at_20_std": 0.085497, - "nauc_ndcg_at_20_diff1": 0.417731, - "nauc_ndcg_at_100_max": 0.257578, - "nauc_ndcg_at_100_std": 0.09627, - "nauc_ndcg_at_100_diff1": 0.409369, - "nauc_ndcg_at_1000_max": 0.26693, - "nauc_ndcg_at_1000_std": 0.100071, - "nauc_ndcg_at_1000_diff1": 0.41458, - "nauc_map_at_1_max": 0.280534, - "nauc_map_at_1_std": -0.03001, - "nauc_map_at_1_diff1": 0.538926, - "nauc_map_at_3_max": 0.264728, - "nauc_map_at_3_std": 0.014403, - "nauc_map_at_3_diff1": 0.48676, - "nauc_map_at_5_max": 0.262414, - "nauc_map_at_5_std": 0.030925, - "nauc_map_at_5_diff1": 0.479445, - "nauc_map_at_10_max": 0.262277, - "nauc_map_at_10_std": 0.037764, - "nauc_map_at_10_diff1": 0.470099, - "nauc_map_at_20_max": 0.262636, - "nauc_map_at_20_std": 0.04315, - "nauc_map_at_20_diff1": 0.465854, - "nauc_map_at_100_max": 0.263191, - "nauc_map_at_100_std": 0.045135, - "nauc_map_at_100_diff1": 0.463709, - "nauc_map_at_1000_max": 0.263864, - "nauc_map_at_1000_std": 0.045534, - "nauc_map_at_1000_diff1": 0.463922, - "nauc_recall_at_1_max": 0.280534, - "nauc_recall_at_1_std": -0.03001, - "nauc_recall_at_1_diff1": 0.538926, - "nauc_recall_at_3_max": 0.236962, - "nauc_recall_at_3_std": 0.054231, - "nauc_recall_at_3_diff1": 0.425405, - "nauc_recall_at_5_max": 0.224468, - "nauc_recall_at_5_std": 0.098421, - "nauc_recall_at_5_diff1": 0.39783, - "nauc_recall_at_10_max": 0.218317, - "nauc_recall_at_10_std": 0.128712, - "nauc_recall_at_10_diff1": 0.337508, - "nauc_recall_at_20_max": 0.213863, - "nauc_recall_at_20_std": 0.183436, - "nauc_recall_at_20_diff1": 0.285907, - "nauc_recall_at_100_max": 0.227284, - "nauc_recall_at_100_std": 0.243125, - "nauc_recall_at_100_diff1": 0.233571, - "nauc_recall_at_1000_max": 0.277326, - "nauc_recall_at_1000_std": 0.328397, - "nauc_recall_at_1000_diff1": 0.20892, - "nauc_precision_at_1_max": 0.265288, - "nauc_precision_at_1_std": 0.001032, - "nauc_precision_at_1_diff1": 0.50655, - "nauc_precision_at_3_max": 0.253493, - "nauc_precision_at_3_std": 0.115181, - "nauc_precision_at_3_diff1": 0.392497, - "nauc_precision_at_5_max": 0.234647, - "nauc_precision_at_5_std": 0.187151, - "nauc_precision_at_5_diff1": 0.338881, - "nauc_precision_at_10_max": 0.237545, - "nauc_precision_at_10_std": 0.213893, - "nauc_precision_at_10_diff1": 0.285351, - "nauc_precision_at_20_max": 0.230892, - "nauc_precision_at_20_std": 0.262866, - "nauc_precision_at_20_diff1": 0.210742, - "nauc_precision_at_100_max": 0.232832, - "nauc_precision_at_100_std": 0.294046, - "nauc_precision_at_100_diff1": 0.140878, - "nauc_precision_at_1000_max": 0.239556, - "nauc_precision_at_1000_std": 0.263343, - "nauc_precision_at_1000_diff1": 0.042393, - "nauc_mrr_at_1_max": 0.265288, - "nauc_mrr_at_1_std": 0.001032, - "nauc_mrr_at_1_diff1": 0.50655, - "nauc_mrr_at_3_max": 0.257994, - "nauc_mrr_at_3_std": 0.04097, - "nauc_mrr_at_3_diff1": 0.456167, - "nauc_mrr_at_5_max": 0.255945, - "nauc_mrr_at_5_std": 0.049945, - "nauc_mrr_at_5_diff1": 0.450037, - "nauc_mrr_at_10_max": 0.256333, - "nauc_mrr_at_10_std": 0.054663, - "nauc_mrr_at_10_diff1": 0.440708, - "nauc_mrr_at_20_max": 0.255786, - "nauc_mrr_at_20_std": 0.058257, - "nauc_mrr_at_20_diff1": 0.437883, - "nauc_mrr_at_100_max": 0.256685, - "nauc_mrr_at_100_std": 0.05889, - "nauc_mrr_at_100_diff1": 0.436408, - "nauc_mrr_at_1000_max": 0.256891, - "nauc_mrr_at_1000_std": 0.059147, - "nauc_mrr_at_1000_diff1": 0.436692, - "main_score": 0.28144, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 86.29045724868774, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackTexRetrieval.json b/beir/CQADupstackTexRetrieval.json deleted file mode 100644 index 59bf496e2229318a687373e5818f9d9c1e58b31d..0000000000000000000000000000000000000000 --- a/beir/CQADupstackTexRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "46989137a86843e03a6195de44b09deda022eec7", - "task_name": "CQADupstackTexRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.14178, - "ndcg_at_3": 0.17057, - "ndcg_at_5": 0.1856, - "ndcg_at_10": 0.20471, - "ndcg_at_20": 0.22044, - "ndcg_at_100": 0.2503, - "ndcg_at_1000": 0.28231, - "map_at_1": 0.11553, - "map_at_3": 0.15025, - "map_at_5": 0.15972, - "map_at_10": 0.16841, - "map_at_20": 0.17291, - "map_at_100": 0.17711, - "map_at_1000": 0.17832, - "recall_at_1": 0.11553, - "recall_at_3": 0.18958, - "recall_at_5": 0.22824, - "recall_at_10": 0.28569, - "recall_at_20": 0.34504, - "recall_at_100": 0.49491, - "recall_at_1000": 0.72826, - "precision_at_1": 0.14178, - "precision_at_3": 0.08098, - "precision_at_5": 0.05953, - "precision_at_10": 0.03823, - "precision_at_20": 0.02326, - "precision_at_100": 0.0071, - "precision_at_1000": 0.00114, - "mrr_at_1": 0.141776, - "mrr_at_3": 0.179456, - "mrr_at_5": 0.189694, - "mrr_at_10": 0.198676, - "mrr_at_20": 0.203388, - "mrr_at_100": 0.207227, - "mrr_at_1000": 0.208095, - "nauc_ndcg_at_1_max": 0.273731, - "nauc_ndcg_at_1_std": -0.000848, - "nauc_ndcg_at_1_diff1": 0.402551, - "nauc_ndcg_at_3_max": 0.254625, - "nauc_ndcg_at_3_std": 0.005476, - "nauc_ndcg_at_3_diff1": 0.343983, - "nauc_ndcg_at_5_max": 0.251083, - "nauc_ndcg_at_5_std": 0.011607, - "nauc_ndcg_at_5_diff1": 0.328852, - "nauc_ndcg_at_10_max": 0.24845, - "nauc_ndcg_at_10_std": 0.024927, - "nauc_ndcg_at_10_diff1": 0.316519, - "nauc_ndcg_at_20_max": 0.251553, - "nauc_ndcg_at_20_std": 0.038963, - "nauc_ndcg_at_20_diff1": 0.306872, - "nauc_ndcg_at_100_max": 0.25391, - "nauc_ndcg_at_100_std": 0.049506, - "nauc_ndcg_at_100_diff1": 0.300219, - "nauc_ndcg_at_1000_max": 0.25447, - "nauc_ndcg_at_1000_std": 0.057198, - "nauc_ndcg_at_1000_diff1": 0.297107, - "nauc_map_at_1_max": 0.24317, - "nauc_map_at_1_std": -0.013325, - "nauc_map_at_1_diff1": 0.430924, - "nauc_map_at_3_max": 0.245786, - "nauc_map_at_3_std": -0.001987, - "nauc_map_at_3_diff1": 0.367577, - "nauc_map_at_5_max": 0.245709, - "nauc_map_at_5_std": 0.001682, - "nauc_map_at_5_diff1": 0.356493, - "nauc_map_at_10_max": 0.24637, - "nauc_map_at_10_std": 0.009096, - "nauc_map_at_10_diff1": 0.348743, - "nauc_map_at_20_max": 0.247864, - "nauc_map_at_20_std": 0.013651, - "nauc_map_at_20_diff1": 0.345685, - "nauc_map_at_100_max": 0.248458, - "nauc_map_at_100_std": 0.015959, - "nauc_map_at_100_diff1": 0.344828, - "nauc_map_at_1000_max": 0.248587, - "nauc_map_at_1000_std": 0.016378, - "nauc_map_at_1000_diff1": 0.344495, - "nauc_recall_at_1_max": 0.24317, - "nauc_recall_at_1_std": -0.013325, - "nauc_recall_at_1_diff1": 0.430924, - "nauc_recall_at_3_max": 0.239063, - "nauc_recall_at_3_std": 0.009005, - "nauc_recall_at_3_diff1": 0.313126, - "nauc_recall_at_5_max": 0.228969, - "nauc_recall_at_5_std": 0.018906, - "nauc_recall_at_5_diff1": 0.274421, - "nauc_recall_at_10_max": 0.224291, - "nauc_recall_at_10_std": 0.047312, - "nauc_recall_at_10_diff1": 0.244958, - "nauc_recall_at_20_max": 0.229385, - "nauc_recall_at_20_std": 0.086611, - "nauc_recall_at_20_diff1": 0.212233, - "nauc_recall_at_100_max": 0.237652, - "nauc_recall_at_100_std": 0.122032, - "nauc_recall_at_100_diff1": 0.180425, - "nauc_recall_at_1000_max": 0.232788, - "nauc_recall_at_1000_std": 0.208114, - "nauc_recall_at_1000_diff1": 0.120909, - "nauc_precision_at_1_max": 0.273731, - "nauc_precision_at_1_std": -0.000848, - "nauc_precision_at_1_diff1": 0.402551, - "nauc_precision_at_3_max": 0.275815, - "nauc_precision_at_3_std": 0.023241, - "nauc_precision_at_3_diff1": 0.281068, - "nauc_precision_at_5_max": 0.271773, - "nauc_precision_at_5_std": 0.037777, - "nauc_precision_at_5_diff1": 0.24271, - "nauc_precision_at_10_max": 0.264364, - "nauc_precision_at_10_std": 0.073815, - "nauc_precision_at_10_diff1": 0.210453, - "nauc_precision_at_20_max": 0.270537, - "nauc_precision_at_20_std": 0.109882, - "nauc_precision_at_20_diff1": 0.180007, - "nauc_precision_at_100_max": 0.247712, - "nauc_precision_at_100_std": 0.141652, - "nauc_precision_at_100_diff1": 0.110556, - "nauc_precision_at_1000_max": 0.195539, - "nauc_precision_at_1000_std": 0.152915, - "nauc_precision_at_1000_diff1": 0.008828, - "nauc_mrr_at_1_max": 0.273731, - "nauc_mrr_at_1_std": -0.000848, - "nauc_mrr_at_1_diff1": 0.402551, - "nauc_mrr_at_3_max": 0.264306, - "nauc_mrr_at_3_std": 0.01339, - "nauc_mrr_at_3_diff1": 0.344013, - "nauc_mrr_at_5_max": 0.262192, - "nauc_mrr_at_5_std": 0.017274, - "nauc_mrr_at_5_diff1": 0.333294, - "nauc_mrr_at_10_max": 0.261096, - "nauc_mrr_at_10_std": 0.022398, - "nauc_mrr_at_10_diff1": 0.32811, - "nauc_mrr_at_20_max": 0.262244, - "nauc_mrr_at_20_std": 0.026409, - "nauc_mrr_at_20_diff1": 0.32533, - "nauc_mrr_at_100_max": 0.262496, - "nauc_mrr_at_100_std": 0.027576, - "nauc_mrr_at_100_diff1": 0.324713, - "nauc_mrr_at_1000_max": 0.262422, - "nauc_mrr_at_1000_std": 0.027625, - "nauc_mrr_at_1000_diff1": 0.324694, - "main_score": 0.20471, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 198.2237286567688, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackUnixRetrieval.json b/beir/CQADupstackUnixRetrieval.json deleted file mode 100644 index b0f1a7343dccc53f3c5aee94185307d891a39fe1..0000000000000000000000000000000000000000 --- a/beir/CQADupstackUnixRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53", - "task_name": "CQADupstackUnixRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.24347, - "ndcg_at_3": 0.2739, - "ndcg_at_5": 0.29021, - "ndcg_at_10": 0.3119, - "ndcg_at_20": 0.33183, - "ndcg_at_100": 0.36465, - "ndcg_at_1000": 0.39555, - "map_at_1": 0.20973, - "map_at_3": 0.25159, - "map_at_5": 0.2616, - "map_at_10": 0.27125, - "map_at_20": 0.27681, - "map_at_100": 0.28175, - "map_at_1000": 0.28303, - "recall_at_1": 0.20973, - "recall_at_3": 0.29951, - "recall_at_5": 0.34009, - "recall_at_10": 0.40363, - "recall_at_20": 0.47728, - "recall_at_100": 0.63718, - "recall_at_1000": 0.8574, - "precision_at_1": 0.24347, - "precision_at_3": 0.11971, - "precision_at_5": 0.08321, - "precision_at_10": 0.05028, - "precision_at_20": 0.03032, - "precision_at_100": 0.00867, - "precision_at_1000": 0.00126, - "mrr_at_1": 0.24347, - "mrr_at_3": 0.285914, - "mrr_at_5": 0.297015, - "mrr_at_10": 0.306006, - "mrr_at_20": 0.311716, - "mrr_at_100": 0.315759, - "mrr_at_1000": 0.316592, - "nauc_ndcg_at_1_max": 0.366074, - "nauc_ndcg_at_1_std": -0.002938, - "nauc_ndcg_at_1_diff1": 0.511682, - "nauc_ndcg_at_3_max": 0.355856, - "nauc_ndcg_at_3_std": 0.033005, - "nauc_ndcg_at_3_diff1": 0.455243, - "nauc_ndcg_at_5_max": 0.342306, - "nauc_ndcg_at_5_std": 0.024515, - "nauc_ndcg_at_5_diff1": 0.434606, - "nauc_ndcg_at_10_max": 0.340939, - "nauc_ndcg_at_10_std": 0.036514, - "nauc_ndcg_at_10_diff1": 0.421622, - "nauc_ndcg_at_20_max": 0.339306, - "nauc_ndcg_at_20_std": 0.041006, - "nauc_ndcg_at_20_diff1": 0.418551, - "nauc_ndcg_at_100_max": 0.339351, - "nauc_ndcg_at_100_std": 0.058532, - "nauc_ndcg_at_100_diff1": 0.413425, - "nauc_ndcg_at_1000_max": 0.345893, - "nauc_ndcg_at_1000_std": 0.063455, - "nauc_ndcg_at_1000_diff1": 0.41893, - "nauc_map_at_1_max": 0.364348, - "nauc_map_at_1_std": -0.01631, - "nauc_map_at_1_diff1": 0.529856, - "nauc_map_at_3_max": 0.356602, - "nauc_map_at_3_std": 0.017935, - "nauc_map_at_3_diff1": 0.473642, - "nauc_map_at_5_max": 0.349516, - "nauc_map_at_5_std": 0.015001, - "nauc_map_at_5_diff1": 0.461318, - "nauc_map_at_10_max": 0.350392, - "nauc_map_at_10_std": 0.022047, - "nauc_map_at_10_diff1": 0.455432, - "nauc_map_at_20_max": 0.3504, - "nauc_map_at_20_std": 0.023646, - "nauc_map_at_20_diff1": 0.454156, - "nauc_map_at_100_max": 0.349922, - "nauc_map_at_100_std": 0.026192, - "nauc_map_at_100_diff1": 0.4536, - "nauc_map_at_1000_max": 0.350113, - "nauc_map_at_1000_std": 0.026625, - "nauc_map_at_1000_diff1": 0.453732, - "nauc_recall_at_1_max": 0.364348, - "nauc_recall_at_1_std": -0.01631, - "nauc_recall_at_1_diff1": 0.529856, - "nauc_recall_at_3_max": 0.343817, - "nauc_recall_at_3_std": 0.054702, - "nauc_recall_at_3_diff1": 0.414897, - "nauc_recall_at_5_max": 0.312585, - "nauc_recall_at_5_std": 0.040548, - "nauc_recall_at_5_diff1": 0.367072, - "nauc_recall_at_10_max": 0.302046, - "nauc_recall_at_10_std": 0.067438, - "nauc_recall_at_10_diff1": 0.328025, - "nauc_recall_at_20_max": 0.289, - "nauc_recall_at_20_std": 0.079, - "nauc_recall_at_20_diff1": 0.309696, - "nauc_recall_at_100_max": 0.279702, - "nauc_recall_at_100_std": 0.178914, - "nauc_recall_at_100_diff1": 0.259407, - "nauc_recall_at_1000_max": 0.358183, - "nauc_recall_at_1000_std": 0.36287, - "nauc_recall_at_1000_diff1": 0.232753, - "nauc_precision_at_1_max": 0.366074, - "nauc_precision_at_1_std": -0.002938, - "nauc_precision_at_1_diff1": 0.511682, - "nauc_precision_at_3_max": 0.342215, - "nauc_precision_at_3_std": 0.079457, - "nauc_precision_at_3_diff1": 0.379603, - "nauc_precision_at_5_max": 0.308996, - "nauc_precision_at_5_std": 0.065465, - "nauc_precision_at_5_diff1": 0.327939, - "nauc_precision_at_10_max": 0.293998, - "nauc_precision_at_10_std": 0.10973, - "nauc_precision_at_10_diff1": 0.268081, - "nauc_precision_at_20_max": 0.265752, - "nauc_precision_at_20_std": 0.123159, - "nauc_precision_at_20_diff1": 0.234389, - "nauc_precision_at_100_max": 0.178161, - "nauc_precision_at_100_std": 0.176852, - "nauc_precision_at_100_diff1": 0.089218, - "nauc_precision_at_1000_max": 0.055458, - "nauc_precision_at_1000_std": 0.142567, - "nauc_precision_at_1000_diff1": -0.047612, - "nauc_mrr_at_1_max": 0.366074, - "nauc_mrr_at_1_std": -0.002938, - "nauc_mrr_at_1_diff1": 0.511682, - "nauc_mrr_at_3_max": 0.354753, - "nauc_mrr_at_3_std": 0.024454, - "nauc_mrr_at_3_diff1": 0.456803, - "nauc_mrr_at_5_max": 0.349086, - "nauc_mrr_at_5_std": 0.021245, - "nauc_mrr_at_5_diff1": 0.444838, - "nauc_mrr_at_10_max": 0.346014, - "nauc_mrr_at_10_std": 0.024307, - "nauc_mrr_at_10_diff1": 0.440129, - "nauc_mrr_at_20_max": 0.345043, - "nauc_mrr_at_20_std": 0.026122, - "nauc_mrr_at_20_diff1": 0.439379, - "nauc_mrr_at_100_max": 0.345434, - "nauc_mrr_at_100_std": 0.028055, - "nauc_mrr_at_100_diff1": 0.439384, - "nauc_mrr_at_1000_max": 0.34558, - "nauc_mrr_at_1000_std": 0.028083, - "nauc_mrr_at_1000_diff1": 0.439607, - "main_score": 0.3119, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 103.79455494880676, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackWebmastersRetrieval.json b/beir/CQADupstackWebmastersRetrieval.json deleted file mode 100644 index 4cf29347b313cc0a23ccaf8cd16d186dc448b999..0000000000000000000000000000000000000000 --- a/beir/CQADupstackWebmastersRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "160c094312a0e1facb97e55eeddb698c0abe3571", - "task_name": "CQADupstackWebmastersRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.22925, - "ndcg_at_3": 0.2741, - "ndcg_at_5": 0.2929, - "ndcg_at_10": 0.31232, - "ndcg_at_20": 0.33154, - "ndcg_at_100": 0.36606, - "ndcg_at_1000": 0.39806, - "map_at_1": 0.19137, - "map_at_3": 0.23966, - "map_at_5": 0.25369, - "map_at_10": 0.26413, - "map_at_20": 0.27109, - "map_at_100": 0.27726, - "map_at_1000": 0.27925, - "recall_at_1": 0.19137, - "recall_at_3": 0.29464, - "recall_at_5": 0.34544, - "recall_at_10": 0.40694, - "recall_at_20": 0.4791, - "recall_at_100": 0.65313, - "recall_at_1000": 0.86383, - "precision_at_1": 0.22925, - "precision_at_3": 0.13109, - "precision_at_5": 0.09684, - "precision_at_10": 0.06047, - "precision_at_20": 0.03903, - "precision_at_100": 0.01235, - "precision_at_1000": 0.00207, - "mrr_at_1": 0.229249, - "mrr_at_3": 0.278656, - "mrr_at_5": 0.291601, - "mrr_at_10": 0.300793, - "mrr_at_20": 0.30649, - "mrr_at_100": 0.310727, - "mrr_at_1000": 0.311487, - "nauc_ndcg_at_1_max": 0.278397, - "nauc_ndcg_at_1_std": 0.074006, - "nauc_ndcg_at_1_diff1": 0.513375, - "nauc_ndcg_at_3_max": 0.277863, - "nauc_ndcg_at_3_std": 0.105389, - "nauc_ndcg_at_3_diff1": 0.48273, - "nauc_ndcg_at_5_max": 0.262458, - "nauc_ndcg_at_5_std": 0.109897, - "nauc_ndcg_at_5_diff1": 0.465795, - "nauc_ndcg_at_10_max": 0.259559, - "nauc_ndcg_at_10_std": 0.112909, - "nauc_ndcg_at_10_diff1": 0.458508, - "nauc_ndcg_at_20_max": 0.259624, - "nauc_ndcg_at_20_std": 0.115503, - "nauc_ndcg_at_20_diff1": 0.448082, - "nauc_ndcg_at_100_max": 0.257458, - "nauc_ndcg_at_100_std": 0.133477, - "nauc_ndcg_at_100_diff1": 0.437838, - "nauc_ndcg_at_1000_max": 0.264933, - "nauc_ndcg_at_1000_std": 0.139622, - "nauc_ndcg_at_1000_diff1": 0.448966, - "nauc_map_at_1_max": 0.264657, - "nauc_map_at_1_std": 0.035551, - "nauc_map_at_1_diff1": 0.536002, - "nauc_map_at_3_max": 0.271682, - "nauc_map_at_3_std": 0.077661, - "nauc_map_at_3_diff1": 0.495551, - "nauc_map_at_5_max": 0.26631, - "nauc_map_at_5_std": 0.080968, - "nauc_map_at_5_diff1": 0.481763, - "nauc_map_at_10_max": 0.263891, - "nauc_map_at_10_std": 0.084954, - "nauc_map_at_10_diff1": 0.477934, - "nauc_map_at_20_max": 0.263787, - "nauc_map_at_20_std": 0.088554, - "nauc_map_at_20_diff1": 0.475802, - "nauc_map_at_100_max": 0.263258, - "nauc_map_at_100_std": 0.095955, - "nauc_map_at_100_diff1": 0.475416, - "nauc_map_at_1000_max": 0.263256, - "nauc_map_at_1000_std": 0.09769, - "nauc_map_at_1000_diff1": 0.475749, - "nauc_recall_at_1_max": 0.264657, - "nauc_recall_at_1_std": 0.035551, - "nauc_recall_at_1_diff1": 0.536002, - "nauc_recall_at_3_max": 0.263286, - "nauc_recall_at_3_std": 0.110583, - "nauc_recall_at_3_diff1": 0.454661, - "nauc_recall_at_5_max": 0.241204, - "nauc_recall_at_5_std": 0.125151, - "nauc_recall_at_5_diff1": 0.411045, - "nauc_recall_at_10_max": 0.240086, - "nauc_recall_at_10_std": 0.137833, - "nauc_recall_at_10_diff1": 0.388961, - "nauc_recall_at_20_max": 0.231511, - "nauc_recall_at_20_std": 0.151779, - "nauc_recall_at_20_diff1": 0.35641, - "nauc_recall_at_100_max": 0.205584, - "nauc_recall_at_100_std": 0.242135, - "nauc_recall_at_100_diff1": 0.267027, - "nauc_recall_at_1000_max": 0.259238, - "nauc_recall_at_1000_std": 0.406363, - "nauc_recall_at_1000_diff1": 0.28158, - "nauc_precision_at_1_max": 0.278397, - "nauc_precision_at_1_std": 0.074006, - "nauc_precision_at_1_diff1": 0.513375, - "nauc_precision_at_3_max": 0.254205, - "nauc_precision_at_3_std": 0.150633, - "nauc_precision_at_3_diff1": 0.384094, - "nauc_precision_at_5_max": 0.20648, - "nauc_precision_at_5_std": 0.158803, - "nauc_precision_at_5_diff1": 0.310389, - "nauc_precision_at_10_max": 0.175023, - "nauc_precision_at_10_std": 0.188265, - "nauc_precision_at_10_diff1": 0.268628, - "nauc_precision_at_20_max": 0.160018, - "nauc_precision_at_20_std": 0.229377, - "nauc_precision_at_20_diff1": 0.216145, - "nauc_precision_at_100_max": 0.039446, - "nauc_precision_at_100_std": 0.309237, - "nauc_precision_at_100_diff1": 0.081114, - "nauc_precision_at_1000_max": -0.078246, - "nauc_precision_at_1000_std": 0.226591, - "nauc_precision_at_1000_diff1": -0.029668, - "nauc_mrr_at_1_max": 0.278397, - "nauc_mrr_at_1_std": 0.074006, - "nauc_mrr_at_1_diff1": 0.513375, - "nauc_mrr_at_3_max": 0.271664, - "nauc_mrr_at_3_std": 0.105992, - "nauc_mrr_at_3_diff1": 0.475073, - "nauc_mrr_at_5_max": 0.264996, - "nauc_mrr_at_5_std": 0.109831, - "nauc_mrr_at_5_diff1": 0.463956, - "nauc_mrr_at_10_max": 0.265579, - "nauc_mrr_at_10_std": 0.111244, - "nauc_mrr_at_10_diff1": 0.464022, - "nauc_mrr_at_20_max": 0.26659, - "nauc_mrr_at_20_std": 0.111591, - "nauc_mrr_at_20_diff1": 0.462309, - "nauc_mrr_at_100_max": 0.265909, - "nauc_mrr_at_100_std": 0.113546, - "nauc_mrr_at_100_diff1": 0.461344, - "nauc_mrr_at_1000_max": 0.266114, - "nauc_mrr_at_1000_std": 0.113636, - "nauc_mrr_at_1000_diff1": 0.461867, - "main_score": 0.31232, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 23.93776535987854, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/CQADupstackWordpressRetrieval.json b/beir/CQADupstackWordpressRetrieval.json deleted file mode 100644 index 88b8df8cbdec73f9c048776b357497213d2c8031..0000000000000000000000000000000000000000 --- a/beir/CQADupstackWordpressRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4", - "task_name": "CQADupstackWordpressRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.18299, - "ndcg_at_3": 0.22947, - "ndcg_at_5": 0.24289, - "ndcg_at_10": 0.26195, - "ndcg_at_20": 0.28138, - "ndcg_at_100": 0.31194, - "ndcg_at_1000": 0.34148, - "map_at_1": 0.16833, - "map_at_3": 0.211, - "map_at_5": 0.2197, - "map_at_10": 0.22788, - "map_at_20": 0.23321, - "map_at_100": 0.2373, - "map_at_1000": 0.23838, - "recall_at_1": 0.16833, - "recall_at_3": 0.26334, - "recall_at_5": 0.29575, - "recall_at_10": 0.3512, - "recall_at_20": 0.42532, - "recall_at_100": 0.58598, - "recall_at_1000": 0.80809, - "precision_at_1": 0.18299, - "precision_at_3": 0.0992, - "precision_at_5": 0.06765, - "precision_at_10": 0.04067, - "precision_at_20": 0.02486, - "precision_at_100": 0.00701, - "precision_at_1000": 0.00105, - "mrr_at_1": 0.182994, - "mrr_at_3": 0.227665, - "mrr_at_5": 0.235059, - "mrr_at_10": 0.243175, - "mrr_at_20": 0.248732, - "mrr_at_100": 0.252759, - "mrr_at_1000": 0.253625, - "nauc_ndcg_at_1_max": 0.290492, - "nauc_ndcg_at_1_std": -0.037429, - "nauc_ndcg_at_1_diff1": 0.427622, - "nauc_ndcg_at_3_max": 0.243326, - "nauc_ndcg_at_3_std": -0.02747, - "nauc_ndcg_at_3_diff1": 0.340168, - "nauc_ndcg_at_5_max": 0.238634, - "nauc_ndcg_at_5_std": -0.02659, - "nauc_ndcg_at_5_diff1": 0.330531, - "nauc_ndcg_at_10_max": 0.247575, - "nauc_ndcg_at_10_std": -0.017911, - "nauc_ndcg_at_10_diff1": 0.320757, - "nauc_ndcg_at_20_max": 0.245705, - "nauc_ndcg_at_20_std": 0.004907, - "nauc_ndcg_at_20_diff1": 0.320415, - "nauc_ndcg_at_100_max": 0.240886, - "nauc_ndcg_at_100_std": 0.023006, - "nauc_ndcg_at_100_diff1": 0.31633, - "nauc_ndcg_at_1000_max": 0.242627, - "nauc_ndcg_at_1000_std": 0.029307, - "nauc_ndcg_at_1000_diff1": 0.318627, - "nauc_map_at_1_max": 0.265847, - "nauc_map_at_1_std": -0.039069, - "nauc_map_at_1_diff1": 0.430649, - "nauc_map_at_3_max": 0.244435, - "nauc_map_at_3_std": -0.027471, - "nauc_map_at_3_diff1": 0.358874, - "nauc_map_at_5_max": 0.244507, - "nauc_map_at_5_std": -0.02695, - "nauc_map_at_5_diff1": 0.352913, - "nauc_map_at_10_max": 0.248881, - "nauc_map_at_10_std": -0.023212, - "nauc_map_at_10_diff1": 0.348928, - "nauc_map_at_20_max": 0.248784, - "nauc_map_at_20_std": -0.016656, - "nauc_map_at_20_diff1": 0.348707, - "nauc_map_at_100_max": 0.248009, - "nauc_map_at_100_std": -0.014712, - "nauc_map_at_100_diff1": 0.348352, - "nauc_map_at_1000_max": 0.248034, - "nauc_map_at_1000_std": -0.014321, - "nauc_map_at_1000_diff1": 0.348601, - "nauc_recall_at_1_max": 0.265847, - "nauc_recall_at_1_std": -0.039069, - "nauc_recall_at_1_diff1": 0.430649, - "nauc_recall_at_3_max": 0.213529, - "nauc_recall_at_3_std": -0.025467, - "nauc_recall_at_3_diff1": 0.286084, - "nauc_recall_at_5_max": 0.205815, - "nauc_recall_at_5_std": -0.025665, - "nauc_recall_at_5_diff1": 0.265638, - "nauc_recall_at_10_max": 0.223811, - "nauc_recall_at_10_std": -0.002912, - "nauc_recall_at_10_diff1": 0.236385, - "nauc_recall_at_20_max": 0.211582, - "nauc_recall_at_20_std": 0.067677, - "nauc_recall_at_20_diff1": 0.234745, - "nauc_recall_at_100_max": 0.182712, - "nauc_recall_at_100_std": 0.17472, - "nauc_recall_at_100_diff1": 0.201213, - "nauc_recall_at_1000_max": 0.174778, - "nauc_recall_at_1000_std": 0.340317, - "nauc_recall_at_1000_diff1": 0.125604, - "nauc_precision_at_1_max": 0.290492, - "nauc_precision_at_1_std": -0.037429, - "nauc_precision_at_1_diff1": 0.427622, - "nauc_precision_at_3_max": 0.227398, - "nauc_precision_at_3_std": -0.028379, - "nauc_precision_at_3_diff1": 0.266137, - "nauc_precision_at_5_max": 0.230675, - "nauc_precision_at_5_std": -0.017098, - "nauc_precision_at_5_diff1": 0.241378, - "nauc_precision_at_10_max": 0.243564, - "nauc_precision_at_10_std": 0.005694, - "nauc_precision_at_10_diff1": 0.209227, - "nauc_precision_at_20_max": 0.239849, - "nauc_precision_at_20_std": 0.089154, - "nauc_precision_at_20_diff1": 0.210395, - "nauc_precision_at_100_max": 0.187676, - "nauc_precision_at_100_std": 0.159012, - "nauc_precision_at_100_diff1": 0.145642, - "nauc_precision_at_1000_max": 0.040961, - "nauc_precision_at_1000_std": 0.149076, - "nauc_precision_at_1000_diff1": 0.029816, - "nauc_mrr_at_1_max": 0.290492, - "nauc_mrr_at_1_std": -0.037429, - "nauc_mrr_at_1_diff1": 0.427622, - "nauc_mrr_at_3_max": 0.261245, - "nauc_mrr_at_3_std": -0.029722, - "nauc_mrr_at_3_diff1": 0.363549, - "nauc_mrr_at_5_max": 0.259592, - "nauc_mrr_at_5_std": -0.028622, - "nauc_mrr_at_5_diff1": 0.357855, - "nauc_mrr_at_10_max": 0.264184, - "nauc_mrr_at_10_std": -0.024333, - "nauc_mrr_at_10_diff1": 0.353717, - "nauc_mrr_at_20_max": 0.264148, - "nauc_mrr_at_20_std": -0.017082, - "nauc_mrr_at_20_diff1": 0.353438, - "nauc_mrr_at_100_max": 0.263672, - "nauc_mrr_at_100_std": -0.015025, - "nauc_mrr_at_100_diff1": 0.353221, - "nauc_mrr_at_1000_max": 0.263556, - "nauc_mrr_at_1000_std": -0.014899, - "nauc_mrr_at_1000_diff1": 0.353607, - "main_score": 0.26195, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 105.96243500709534, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/ClimateFEVER.json b/beir/ClimateFEVER.json deleted file mode 100644 index b7cbe391fcebb4cca3f5cec350f3ab8f75947adc..0000000000000000000000000000000000000000 --- a/beir/ClimateFEVER.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380", - "task_name": "ClimateFEVER", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.21042, - "ndcg_at_3": 0.18007, - "ndcg_at_5": 0.19365, - "ndcg_at_10": 0.22698, - "ndcg_at_20": 0.25245, - "ndcg_at_100": 0.29216, - "ndcg_at_1000": 0.32348, - "map_at_1": 0.09732, - "map_at_3": 0.13098, - "map_at_5": 0.14246, - "map_at_10": 0.15738, - "map_at_20": 0.16557, - "map_at_100": 0.17281, - "map_at_1000": 0.17442, - "recall_at_1": 0.09732, - "recall_at_3": 0.16374, - "recall_at_5": 0.20321, - "recall_at_10": 0.27863, - "recall_at_20": 0.35101, - "recall_at_100": 0.50446, - "recall_at_1000": 0.68127, - "precision_at_1": 0.21042, - "precision_at_3": 0.12942, - "precision_at_5": 0.09967, - "precision_at_10": 0.07114, - "precision_at_20": 0.04635, - "precision_at_100": 0.014, - "precision_at_1000": 0.00197, - "mrr_at_1": 0.210423, - "mrr_at_3": 0.276439, - "mrr_at_5": 0.293377, - "mrr_at_10": 0.309231, - "mrr_at_20": 0.315901, - "mrr_at_100": 0.319553, - "mrr_at_1000": 0.320057, - "nauc_ndcg_at_1_max": 0.253668, - "nauc_ndcg_at_1_std": 0.041982, - "nauc_ndcg_at_1_diff1": 0.263596, - "nauc_ndcg_at_3_max": 0.303375, - "nauc_ndcg_at_3_std": 0.071602, - "nauc_ndcg_at_3_diff1": 0.231569, - "nauc_ndcg_at_5_max": 0.31039, - "nauc_ndcg_at_5_std": 0.08631, - "nauc_ndcg_at_5_diff1": 0.234282, - "nauc_ndcg_at_10_max": 0.321631, - "nauc_ndcg_at_10_std": 0.128647, - "nauc_ndcg_at_10_diff1": 0.224955, - "nauc_ndcg_at_20_max": 0.329699, - "nauc_ndcg_at_20_std": 0.159557, - "nauc_ndcg_at_20_diff1": 0.228594, - "nauc_ndcg_at_100_max": 0.345065, - "nauc_ndcg_at_100_std": 0.20812, - "nauc_ndcg_at_100_diff1": 0.230539, - "nauc_ndcg_at_1000_max": 0.351358, - "nauc_ndcg_at_1000_std": 0.224793, - "nauc_ndcg_at_1000_diff1": 0.228996, - "nauc_map_at_1_max": 0.281958, - "nauc_map_at_1_std": -0.021727, - "nauc_map_at_1_diff1": 0.313643, - "nauc_map_at_3_max": 0.298457, - "nauc_map_at_3_std": 0.022626, - "nauc_map_at_3_diff1": 0.257536, - "nauc_map_at_5_max": 0.304463, - "nauc_map_at_5_std": 0.038367, - "nauc_map_at_5_diff1": 0.257217, - "nauc_map_at_10_max": 0.31309, - "nauc_map_at_10_std": 0.066086, - "nauc_map_at_10_diff1": 0.251932, - "nauc_map_at_20_max": 0.31663, - "nauc_map_at_20_std": 0.07943, - "nauc_map_at_20_diff1": 0.253261, - "nauc_map_at_100_max": 0.320402, - "nauc_map_at_100_std": 0.092723, - "nauc_map_at_100_diff1": 0.252213, - "nauc_map_at_1000_max": 0.320828, - "nauc_map_at_1000_std": 0.094498, - "nauc_map_at_1000_diff1": 0.252002, - "nauc_recall_at_1_max": 0.281958, - "nauc_recall_at_1_std": -0.021727, - "nauc_recall_at_1_diff1": 0.313643, - "nauc_recall_at_3_max": 0.311157, - "nauc_recall_at_3_std": 0.060219, - "nauc_recall_at_3_diff1": 0.210486, - "nauc_recall_at_5_max": 0.297119, - "nauc_recall_at_5_std": 0.096385, - "nauc_recall_at_5_diff1": 0.194008, - "nauc_recall_at_10_max": 0.29758, - "nauc_recall_at_10_std": 0.167827, - "nauc_recall_at_10_diff1": 0.168048, - "nauc_recall_at_20_max": 0.302094, - "nauc_recall_at_20_std": 0.227934, - "nauc_recall_at_20_diff1": 0.167479, - "nauc_recall_at_100_max": 0.325903, - "nauc_recall_at_100_std": 0.356132, - "nauc_recall_at_100_diff1": 0.163043, - "nauc_recall_at_1000_max": 0.365571, - "nauc_recall_at_1000_std": 0.465573, - "nauc_recall_at_1000_diff1": 0.146935, - "nauc_precision_at_1_max": 0.253668, - "nauc_precision_at_1_std": 0.041982, - "nauc_precision_at_1_diff1": 0.263596, - "nauc_precision_at_3_max": 0.302638, - "nauc_precision_at_3_std": 0.164883, - "nauc_precision_at_3_diff1": 0.165101, - "nauc_precision_at_5_max": 0.299424, - "nauc_precision_at_5_std": 0.204229, - "nauc_precision_at_5_diff1": 0.152494, - "nauc_precision_at_10_max": 0.287718, - "nauc_precision_at_10_std": 0.276706, - "nauc_precision_at_10_diff1": 0.105829, - "nauc_precision_at_20_max": 0.276563, - "nauc_precision_at_20_std": 0.329983, - "nauc_precision_at_20_diff1": 0.10746, - "nauc_precision_at_100_max": 0.258573, - "nauc_precision_at_100_std": 0.432574, - "nauc_precision_at_100_diff1": 0.079839, - "nauc_precision_at_1000_max": 0.199198, - "nauc_precision_at_1000_std": 0.440233, - "nauc_precision_at_1000_diff1": 0.024789, - "nauc_mrr_at_1_max": 0.253668, - "nauc_mrr_at_1_std": 0.041982, - "nauc_mrr_at_1_diff1": 0.263596, - "nauc_mrr_at_3_max": 0.298863, - "nauc_mrr_at_3_std": 0.106849, - "nauc_mrr_at_3_diff1": 0.230179, - "nauc_mrr_at_5_max": 0.29585, - "nauc_mrr_at_5_std": 0.117695, - "nauc_mrr_at_5_diff1": 0.225112, - "nauc_mrr_at_10_max": 0.297507, - "nauc_mrr_at_10_std": 0.128053, - "nauc_mrr_at_10_diff1": 0.222167, - "nauc_mrr_at_20_max": 0.298771, - "nauc_mrr_at_20_std": 0.13317, - "nauc_mrr_at_20_diff1": 0.223668, - "nauc_mrr_at_100_max": 0.299334, - "nauc_mrr_at_100_std": 0.133076, - "nauc_mrr_at_100_diff1": 0.224883, - "nauc_mrr_at_1000_max": 0.299249, - "nauc_mrr_at_1000_std": 0.132749, - "nauc_mrr_at_1000_diff1": 0.224871, - "main_score": 0.22698, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 5528.608217954636, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/DBPedia.json b/beir/DBPedia.json deleted file mode 100644 index 08c9f0b6d910b304a805a02786003c86f8a05bdb..0000000000000000000000000000000000000000 --- a/beir/DBPedia.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659", - "task_name": "DBPedia", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.41375, - "ndcg_at_3": 0.32494, - "ndcg_at_5": 0.29511, - "ndcg_at_10": 0.27559, - "ndcg_at_20": 0.26821, - "ndcg_at_100": 0.29407, - "ndcg_at_1000": 0.34784, - "map_at_1": 0.07455, - "map_at_3": 0.10611, - "map_at_5": 0.1174, - "map_at_10": 0.13131, - "map_at_20": 0.14319, - "map_at_100": 0.1607, - "map_at_1000": 0.16731, - "recall_at_1": 0.07455, - "recall_at_3": 0.11476, - "recall_at_5": 0.13383, - "recall_at_10": 0.16742, - "recall_at_20": 0.2134, - "recall_at_100": 0.32125, - "recall_at_1000": 0.49776, - "precision_at_1": 0.51, - "precision_at_3": 0.3475, - "precision_at_5": 0.2705, - "precision_at_10": 0.19575, - "precision_at_20": 0.13988, - "precision_at_100": 0.0535, - "precision_at_1000": 0.01072, - "mrr_at_1": 0.51, - "mrr_at_3": 0.57875, - "mrr_at_5": 0.585375, - "mrr_at_10": 0.591851, - "mrr_at_20": 0.596647, - "mrr_at_100": 0.598812, - "mrr_at_1000": 0.599078, - "nauc_ndcg_at_1_max": 0.423916, - "nauc_ndcg_at_1_std": 0.155682, - "nauc_ndcg_at_1_diff1": 0.431251, - "nauc_ndcg_at_3_max": 0.438604, - "nauc_ndcg_at_3_std": 0.193398, - "nauc_ndcg_at_3_diff1": 0.308724, - "nauc_ndcg_at_5_max": 0.414941, - "nauc_ndcg_at_5_std": 0.182644, - "nauc_ndcg_at_5_diff1": 0.287275, - "nauc_ndcg_at_10_max": 0.390035, - "nauc_ndcg_at_10_std": 0.176473, - "nauc_ndcg_at_10_diff1": 0.279835, - "nauc_ndcg_at_20_max": 0.357069, - "nauc_ndcg_at_20_std": 0.165987, - "nauc_ndcg_at_20_diff1": 0.282564, - "nauc_ndcg_at_100_max": 0.343485, - "nauc_ndcg_at_100_std": 0.205251, - "nauc_ndcg_at_100_diff1": 0.258367, - "nauc_ndcg_at_1000_max": 0.380072, - "nauc_ndcg_at_1000_std": 0.265801, - "nauc_ndcg_at_1000_diff1": 0.257738, - "nauc_map_at_1_max": 0.118856, - "nauc_map_at_1_std": -0.093654, - "nauc_map_at_1_diff1": 0.383665, - "nauc_map_at_3_max": 0.163439, - "nauc_map_at_3_std": -0.050284, - "nauc_map_at_3_diff1": 0.323527, - "nauc_map_at_5_max": 0.17271, - "nauc_map_at_5_std": -0.034506, - "nauc_map_at_5_diff1": 0.305797, - "nauc_map_at_10_max": 0.187527, - "nauc_map_at_10_std": 0.003105, - "nauc_map_at_10_diff1": 0.281637, - "nauc_map_at_20_max": 0.206161, - "nauc_map_at_20_std": 0.043977, - "nauc_map_at_20_diff1": 0.26506, - "nauc_map_at_100_max": 0.236987, - "nauc_map_at_100_std": 0.115381, - "nauc_map_at_100_diff1": 0.233655, - "nauc_map_at_1000_max": 0.24553, - "nauc_map_at_1000_std": 0.132969, - "nauc_map_at_1000_diff1": 0.230466, - "nauc_recall_at_1_max": 0.118856, - "nauc_recall_at_1_std": -0.093654, - "nauc_recall_at_1_diff1": 0.383665, - "nauc_recall_at_3_max": 0.144271, - "nauc_recall_at_3_std": -0.053232, - "nauc_recall_at_3_diff1": 0.297432, - "nauc_recall_at_5_max": 0.150162, - "nauc_recall_at_5_std": -0.041646, - "nauc_recall_at_5_diff1": 0.283241, - "nauc_recall_at_10_max": 0.155396, - "nauc_recall_at_10_std": -0.00623, - "nauc_recall_at_10_diff1": 0.246545, - "nauc_recall_at_20_max": 0.155934, - "nauc_recall_at_20_std": 0.03434, - "nauc_recall_at_20_diff1": 0.217566, - "nauc_recall_at_100_max": 0.183441, - "nauc_recall_at_100_std": 0.179899, - "nauc_recall_at_100_diff1": 0.154327, - "nauc_recall_at_1000_max": 0.253257, - "nauc_recall_at_1000_std": 0.313636, - "nauc_recall_at_1000_diff1": 0.147442, - "nauc_precision_at_1_max": 0.451052, - "nauc_precision_at_1_std": 0.189451, - "nauc_precision_at_1_diff1": 0.462767, - "nauc_precision_at_3_max": 0.441379, - "nauc_precision_at_3_std": 0.265232, - "nauc_precision_at_3_diff1": 0.183643, - "nauc_precision_at_5_max": 0.412908, - "nauc_precision_at_5_std": 0.282232, - "nauc_precision_at_5_diff1": 0.105613, - "nauc_precision_at_10_max": 0.389052, - "nauc_precision_at_10_std": 0.366196, - "nauc_precision_at_10_diff1": 0.030044, - "nauc_precision_at_20_max": 0.372606, - "nauc_precision_at_20_std": 0.450747, - "nauc_precision_at_20_diff1": -0.014095, - "nauc_precision_at_100_max": 0.315374, - "nauc_precision_at_100_std": 0.498572, - "nauc_precision_at_100_diff1": -0.082395, - "nauc_precision_at_1000_max": 0.199796, - "nauc_precision_at_1000_std": 0.386212, - "nauc_precision_at_1000_diff1": -0.119969, - "nauc_mrr_at_1_max": 0.451052, - "nauc_mrr_at_1_std": 0.189451, - "nauc_mrr_at_1_diff1": 0.462767, - "nauc_mrr_at_3_max": 0.483251, - "nauc_mrr_at_3_std": 0.253881, - "nauc_mrr_at_3_diff1": 0.434542, - "nauc_mrr_at_5_max": 0.483313, - "nauc_mrr_at_5_std": 0.251794, - "nauc_mrr_at_5_diff1": 0.435443, - "nauc_mrr_at_10_max": 0.480391, - "nauc_mrr_at_10_std": 0.249537, - "nauc_mrr_at_10_diff1": 0.431642, - "nauc_mrr_at_20_max": 0.479453, - "nauc_mrr_at_20_std": 0.251964, - "nauc_mrr_at_20_diff1": 0.431792, - "nauc_mrr_at_100_max": 0.4801, - "nauc_mrr_at_100_std": 0.252416, - "nauc_mrr_at_100_diff1": 0.433313, - "nauc_mrr_at_1000_max": 0.479913, - "nauc_mrr_at_1000_std": 0.252061, - "nauc_mrr_at_1000_diff1": 0.433367, - "main_score": 0.27559, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 3010.4430978298187, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/EmotionClassification.json b/beir/EmotionClassification.json deleted file mode 100644 index b4b43d139cb2ea441602dc1ad2e90ce20a20b856..0000000000000000000000000000000000000000 --- a/beir/EmotionClassification.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "dataset_revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37", - "task_name": "EmotionClassification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.437, - "f1": 0.39358, - "f1_weighted": 0.45983, - "scores_per_experiment": [ - { - "accuracy": 0.4555, - "f1": 0.399212, - "f1_weighted": 0.484673 - }, - { - "accuracy": 0.432, - "f1": 0.394225, - "f1_weighted": 0.447477 - }, - { - "accuracy": 0.415, - "f1": 0.379312, - "f1_weighted": 0.440126 - }, - { - "accuracy": 0.413, - "f1": 0.374642, - "f1_weighted": 0.440663 - }, - { - "accuracy": 0.456, - "f1": 0.41365, - "f1_weighted": 0.47793 - }, - { - "accuracy": 0.4375, - "f1": 0.390637, - "f1_weighted": 0.45846 - }, - { - "accuracy": 0.447, - "f1": 0.402599, - "f1_weighted": 0.470503 - }, - { - "accuracy": 0.4135, - "f1": 0.376153, - "f1_weighted": 0.428264 - }, - { - "accuracy": 0.461, - "f1": 0.414495, - "f1_weighted": 0.481941 - }, - { - "accuracy": 0.4395, - "f1": 0.390877, - "f1_weighted": 0.468265 - } - ], - "main_score": 0.437, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 7.992861747741699, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/FEVER.json b/beir/FEVER.json deleted file mode 100644 index cd93df819b674325935aeab3cbbacaaae6fd7521..0000000000000000000000000000000000000000 --- a/beir/FEVER.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12", - "task_name": "FEVER", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.52625, - "ndcg_at_3": 0.61126, - "ndcg_at_5": 0.63731, - "ndcg_at_10": 0.65721, - "ndcg_at_20": 0.66718, - "ndcg_at_100": 0.67996, - "ndcg_at_1000": 0.68554, - "map_at_1": 0.48757, - "map_at_3": 0.57408, - "map_at_5": 0.58912, - "map_at_10": 0.59765, - "map_at_20": 0.60053, - "map_at_100": 0.60235, - "map_at_1000": 0.60258, - "recall_at_1": 0.48757, - "recall_at_3": 0.67712, - "recall_at_5": 0.74102, - "recall_at_10": 0.80181, - "recall_at_20": 0.83964, - "recall_at_100": 0.90629, - "recall_at_1000": 0.9479, - "precision_at_1": 0.52625, - "precision_at_3": 0.24482, - "precision_at_5": 0.16112, - "precision_at_10": 0.08738, - "precision_at_20": 0.0459, - "precision_at_100": 0.00997, - "precision_at_1000": 0.00105, - "mrr_at_1": 0.526253, - "mrr_at_3": 0.615912, - "mrr_at_5": 0.631183, - "mrr_at_10": 0.639444, - "mrr_at_20": 0.642131, - "mrr_at_100": 0.643808, - "mrr_at_1000": 0.643955, - "nauc_ndcg_at_1_max": 0.245061, - "nauc_ndcg_at_1_std": -0.137009, - "nauc_ndcg_at_1_diff1": 0.55709, - "nauc_ndcg_at_3_max": 0.248319, - "nauc_ndcg_at_3_std": -0.104366, - "nauc_ndcg_at_3_diff1": 0.457855, - "nauc_ndcg_at_5_max": 0.255093, - "nauc_ndcg_at_5_std": -0.091996, - "nauc_ndcg_at_5_diff1": 0.456859, - "nauc_ndcg_at_10_max": 0.250742, - "nauc_ndcg_at_10_std": -0.080252, - "nauc_ndcg_at_10_diff1": 0.461142, - "nauc_ndcg_at_20_max": 0.247866, - "nauc_ndcg_at_20_std": -0.075957, - "nauc_ndcg_at_20_diff1": 0.459826, - "nauc_ndcg_at_100_max": 0.24549, - "nauc_ndcg_at_100_std": -0.07457, - "nauc_ndcg_at_100_diff1": 0.460618, - "nauc_ndcg_at_1000_max": 0.245226, - "nauc_ndcg_at_1000_std": -0.077771, - "nauc_ndcg_at_1000_diff1": 0.462122, - "nauc_map_at_1_max": 0.223226, - "nauc_map_at_1_std": -0.111902, - "nauc_map_at_1_diff1": 0.506577, - "nauc_map_at_3_max": 0.236994, - "nauc_map_at_3_std": -0.1019, - "nauc_map_at_3_diff1": 0.460434, - "nauc_map_at_5_max": 0.241242, - "nauc_map_at_5_std": -0.096404, - "nauc_map_at_5_diff1": 0.461683, - "nauc_map_at_10_max": 0.239856, - "nauc_map_at_10_std": -0.092091, - "nauc_map_at_10_diff1": 0.463714, - "nauc_map_at_20_max": 0.239147, - "nauc_map_at_20_std": -0.091015, - "nauc_map_at_20_diff1": 0.463512, - "nauc_map_at_100_max": 0.238856, - "nauc_map_at_100_std": -0.090851, - "nauc_map_at_100_diff1": 0.463642, - "nauc_map_at_1000_max": 0.238833, - "nauc_map_at_1000_std": -0.090946, - "nauc_map_at_1000_diff1": 0.463676, - "nauc_recall_at_1_max": 0.223226, - "nauc_recall_at_1_std": -0.111902, - "nauc_recall_at_1_diff1": 0.506577, - "nauc_recall_at_3_max": 0.240556, - "nauc_recall_at_3_std": -0.078741, - "nauc_recall_at_3_diff1": 0.372825, - "nauc_recall_at_5_max": 0.257287, - "nauc_recall_at_5_std": -0.039755, - "nauc_recall_at_5_diff1": 0.352184, - "nauc_recall_at_10_max": 0.236078, - "nauc_recall_at_10_std": 0.021126, - "nauc_recall_at_10_diff1": 0.340759, - "nauc_recall_at_20_max": 0.211544, - "nauc_recall_at_20_std": 0.068803, - "nauc_recall_at_20_diff1": 0.302302, - "nauc_recall_at_100_max": 0.146115, - "nauc_recall_at_100_std": 0.18329, - "nauc_recall_at_100_diff1": 0.20034, - "nauc_recall_at_1000_max": 0.071731, - "nauc_recall_at_1000_std": 0.257758, - "nauc_recall_at_1000_diff1": 0.069671, - "nauc_precision_at_1_max": 0.245061, - "nauc_precision_at_1_std": -0.137009, - "nauc_precision_at_1_diff1": 0.55709, - "nauc_precision_at_3_max": 0.280319, - "nauc_precision_at_3_std": -0.106341, - "nauc_precision_at_3_diff1": 0.421819, - "nauc_precision_at_5_max": 0.306886, - "nauc_precision_at_5_std": -0.063768, - "nauc_precision_at_5_diff1": 0.400642, - "nauc_precision_at_10_max": 0.281175, - "nauc_precision_at_10_std": 0.010407, - "nauc_precision_at_10_diff1": 0.373056, - "nauc_precision_at_20_max": 0.246245, - "nauc_precision_at_20_std": 0.065341, - "nauc_precision_at_20_diff1": 0.313258, - "nauc_precision_at_100_max": 0.161083, - "nauc_precision_at_100_std": 0.165796, - "nauc_precision_at_100_diff1": 0.14073, - "nauc_precision_at_1000_max": 0.055871, - "nauc_precision_at_1000_std": 0.147183, - "nauc_precision_at_1000_diff1": -0.017647, - "nauc_mrr_at_1_max": 0.245061, - "nauc_mrr_at_1_std": -0.137009, - "nauc_mrr_at_1_diff1": 0.55709, - "nauc_mrr_at_3_max": 0.26592, - "nauc_mrr_at_3_std": -0.128604, - "nauc_mrr_at_3_diff1": 0.515274, - "nauc_mrr_at_5_max": 0.270103, - "nauc_mrr_at_5_std": -0.122957, - "nauc_mrr_at_5_diff1": 0.516955, - "nauc_mrr_at_10_max": 0.268861, - "nauc_mrr_at_10_std": -0.119639, - "nauc_mrr_at_10_diff1": 0.520291, - "nauc_mrr_at_20_max": 0.268109, - "nauc_mrr_at_20_std": -0.11923, - "nauc_mrr_at_20_diff1": 0.520273, - "nauc_mrr_at_100_max": 0.267881, - "nauc_mrr_at_100_std": -0.119438, - "nauc_mrr_at_100_diff1": 0.520616, - "nauc_mrr_at_1000_max": 0.267828, - "nauc_mrr_at_1000_std": -0.119632, - "nauc_mrr_at_1000_diff1": 0.520671, - "main_score": 0.65721, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 5951.847348451614, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/FiQA2018.json b/beir/FiQA2018.json deleted file mode 100644 index 3b26596f4c2c5174c6c952f8c6296a46e7df1038..0000000000000000000000000000000000000000 --- a/beir/FiQA2018.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06", - "task_name": "FiQA2018", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.29167, - "ndcg_at_3": 0.26738, - "ndcg_at_5": 0.27998, - "ndcg_at_10": 0.30568, - "ndcg_at_20": 0.32864, - "ndcg_at_100": 0.36517, - "ndcg_at_1000": 0.4022, - "map_at_1": 0.13794, - "map_at_3": 0.19969, - "map_at_5": 0.21926, - "map_at_10": 0.23496, - "map_at_20": 0.24335, - "map_at_100": 0.2499, - "map_at_1000": 0.25182, - "recall_at_1": 0.13794, - "recall_at_3": 0.24295, - "recall_at_5": 0.29735, - "recall_at_10": 0.38048, - "recall_at_20": 0.44955, - "recall_at_100": 0.60103, - "recall_at_1000": 0.82989, - "precision_at_1": 0.29167, - "precision_at_3": 0.18313, - "precision_at_5": 0.13827, - "precision_at_10": 0.08827, - "precision_at_20": 0.05386, - "precision_at_100": 0.01489, - "precision_at_1000": 0.00213, - "mrr_at_1": 0.291667, - "mrr_at_3": 0.344136, - "mrr_at_5": 0.357639, - "mrr_at_10": 0.370531, - "mrr_at_20": 0.375432, - "mrr_at_100": 0.379265, - "mrr_at_1000": 0.379918, - "nauc_ndcg_at_1_max": 0.382865, - "nauc_ndcg_at_1_std": -0.018401, - "nauc_ndcg_at_1_diff1": 0.497381, - "nauc_ndcg_at_3_max": 0.323251, - "nauc_ndcg_at_3_std": -0.015778, - "nauc_ndcg_at_3_diff1": 0.393907, - "nauc_ndcg_at_5_max": 0.321995, - "nauc_ndcg_at_5_std": 0.010656, - "nauc_ndcg_at_5_diff1": 0.390196, - "nauc_ndcg_at_10_max": 0.307513, - "nauc_ndcg_at_10_std": 0.025637, - "nauc_ndcg_at_10_diff1": 0.376314, - "nauc_ndcg_at_20_max": 0.308166, - "nauc_ndcg_at_20_std": 0.041018, - "nauc_ndcg_at_20_diff1": 0.366269, - "nauc_ndcg_at_100_max": 0.329718, - "nauc_ndcg_at_100_std": 0.06222, - "nauc_ndcg_at_100_diff1": 0.377411, - "nauc_ndcg_at_1000_max": 0.342405, - "nauc_ndcg_at_1000_std": 0.066923, - "nauc_ndcg_at_1000_diff1": 0.379008, - "nauc_map_at_1_max": 0.25955, - "nauc_map_at_1_std": -0.068494, - "nauc_map_at_1_diff1": 0.455645, - "nauc_map_at_3_max": 0.284944, - "nauc_map_at_3_std": -0.028556, - "nauc_map_at_3_diff1": 0.403951, - "nauc_map_at_5_max": 0.302179, - "nauc_map_at_5_std": -0.006055, - "nauc_map_at_5_diff1": 0.398122, - "nauc_map_at_10_max": 0.304114, - "nauc_map_at_10_std": 0.002683, - "nauc_map_at_10_diff1": 0.391394, - "nauc_map_at_20_max": 0.305538, - "nauc_map_at_20_std": 0.008069, - "nauc_map_at_20_diff1": 0.387448, - "nauc_map_at_100_max": 0.311454, - "nauc_map_at_100_std": 0.013278, - "nauc_map_at_100_diff1": 0.389779, - "nauc_map_at_1000_max": 0.312485, - "nauc_map_at_1000_std": 0.014026, - "nauc_map_at_1000_diff1": 0.389806, - "nauc_recall_at_1_max": 0.25955, - "nauc_recall_at_1_std": -0.068494, - "nauc_recall_at_1_diff1": 0.455645, - "nauc_recall_at_3_max": 0.240749, - "nauc_recall_at_3_std": -0.009688, - "nauc_recall_at_3_diff1": 0.338397, - "nauc_recall_at_5_max": 0.249765, - "nauc_recall_at_5_std": 0.041173, - "nauc_recall_at_5_diff1": 0.306595, - "nauc_recall_at_10_max": 0.194711, - "nauc_recall_at_10_std": 0.068708, - "nauc_recall_at_10_diff1": 0.235346, - "nauc_recall_at_20_max": 0.187834, - "nauc_recall_at_20_std": 0.115026, - "nauc_recall_at_20_diff1": 0.195265, - "nauc_recall_at_100_max": 0.248323, - "nauc_recall_at_100_std": 0.209214, - "nauc_recall_at_100_diff1": 0.222041, - "nauc_recall_at_1000_max": 0.30524, - "nauc_recall_at_1000_std": 0.365714, - "nauc_recall_at_1000_diff1": 0.148893, - "nauc_precision_at_1_max": 0.382865, - "nauc_precision_at_1_std": -0.018401, - "nauc_precision_at_1_diff1": 0.497381, - "nauc_precision_at_3_max": 0.367225, - "nauc_precision_at_3_std": 0.013713, - "nauc_precision_at_3_diff1": 0.337091, - "nauc_precision_at_5_max": 0.378168, - "nauc_precision_at_5_std": 0.063933, - "nauc_precision_at_5_diff1": 0.305218, - "nauc_precision_at_10_max": 0.365227, - "nauc_precision_at_10_std": 0.099963, - "nauc_precision_at_10_diff1": 0.252887, - "nauc_precision_at_20_max": 0.342121, - "nauc_precision_at_20_std": 0.122905, - "nauc_precision_at_20_diff1": 0.214998, - "nauc_precision_at_100_max": 0.342554, - "nauc_precision_at_100_std": 0.163248, - "nauc_precision_at_100_diff1": 0.1778, - "nauc_precision_at_1000_max": 0.28898, - "nauc_precision_at_1000_std": 0.146869, - "nauc_precision_at_1000_diff1": 0.081735, - "nauc_mrr_at_1_max": 0.382865, - "nauc_mrr_at_1_std": -0.018401, - "nauc_mrr_at_1_diff1": 0.497381, - "nauc_mrr_at_3_max": 0.361507, - "nauc_mrr_at_3_std": -0.010684, - "nauc_mrr_at_3_diff1": 0.448131, - "nauc_mrr_at_5_max": 0.364134, - "nauc_mrr_at_5_std": 0.0015, - "nauc_mrr_at_5_diff1": 0.448739, - "nauc_mrr_at_10_max": 0.358277, - "nauc_mrr_at_10_std": 0.003988, - "nauc_mrr_at_10_diff1": 0.439851, - "nauc_mrr_at_20_max": 0.358842, - "nauc_mrr_at_20_std": 0.008399, - "nauc_mrr_at_20_diff1": 0.438136, - "nauc_mrr_at_100_max": 0.359867, - "nauc_mrr_at_100_std": 0.009229, - "nauc_mrr_at_100_diff1": 0.439474, - "nauc_mrr_at_1000_max": 0.359973, - "nauc_mrr_at_1000_std": 0.008946, - "nauc_mrr_at_1000_diff1": 0.439639, - "main_score": 0.30568, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 80.82175087928772, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/HotpotQA.json b/beir/HotpotQA.json deleted file mode 100644 index e75a8b1449217a2386e500531fad17d6d56be112..0000000000000000000000000000000000000000 --- a/beir/HotpotQA.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "ab518f4d6fcca38d87c25209f94beba119d02014", - "task_name": "HotpotQA", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.60162, - "ndcg_at_3": 0.44502, - "ndcg_at_5": 0.46295, - "ndcg_at_10": 0.47809, - "ndcg_at_20": 0.48903, - "ndcg_at_100": 0.50535, - "ndcg_at_1000": 0.51923, - "map_at_1": 0.30081, - "map_at_3": 0.36953, - "map_at_5": 0.38167, - "map_at_10": 0.38938, - "map_at_20": 0.39314, - "map_at_100": 0.39603, - "map_at_1000": 0.39663, - "recall_at_1": 0.30081, - "recall_at_3": 0.40614, - "recall_at_5": 0.44159, - "recall_at_10": 0.47961, - "recall_at_20": 0.51499, - "recall_at_100": 0.58731, - "recall_at_1000": 0.68035, - "precision_at_1": 0.60162, - "precision_at_3": 0.27076, - "precision_at_5": 0.17664, - "precision_at_10": 0.09592, - "precision_at_20": 0.0515, - "precision_at_100": 0.01175, - "precision_at_1000": 0.00136, - "mrr_at_1": 0.601621, - "mrr_at_3": 0.652622, - "mrr_at_5": 0.660826, - "mrr_at_10": 0.665901, - "mrr_at_20": 0.668133, - "mrr_at_100": 0.669551, - "mrr_at_1000": 0.669784, - "nauc_ndcg_at_1_max": 0.5116, - "nauc_ndcg_at_1_std": 0.073935, - "nauc_ndcg_at_1_diff1": 0.715694, - "nauc_ndcg_at_3_max": 0.378233, - "nauc_ndcg_at_3_std": 0.067468, - "nauc_ndcg_at_3_diff1": 0.494853, - "nauc_ndcg_at_5_max": 0.359559, - "nauc_ndcg_at_5_std": 0.073257, - "nauc_ndcg_at_5_diff1": 0.465013, - "nauc_ndcg_at_10_max": 0.339948, - "nauc_ndcg_at_10_std": 0.076042, - "nauc_ndcg_at_10_diff1": 0.443659, - "nauc_ndcg_at_20_max": 0.328131, - "nauc_ndcg_at_20_std": 0.074644, - "nauc_ndcg_at_20_diff1": 0.432222, - "nauc_ndcg_at_100_max": 0.312544, - "nauc_ndcg_at_100_std": 0.078449, - "nauc_ndcg_at_100_diff1": 0.415341, - "nauc_ndcg_at_1000_max": 0.305183, - "nauc_ndcg_at_1000_std": 0.078385, - "nauc_ndcg_at_1000_diff1": 0.409027, - "nauc_map_at_1_max": 0.5116, - "nauc_map_at_1_std": 0.073935, - "nauc_map_at_1_diff1": 0.715694, - "nauc_map_at_3_max": 0.334791, - "nauc_map_at_3_std": 0.058541, - "nauc_map_at_3_diff1": 0.444158, - "nauc_map_at_5_max": 0.31872, - "nauc_map_at_5_std": 0.060961, - "nauc_map_at_5_diff1": 0.421822, - "nauc_map_at_10_max": 0.307675, - "nauc_map_at_10_std": 0.06241, - "nauc_map_at_10_diff1": 0.410375, - "nauc_map_at_20_max": 0.303193, - "nauc_map_at_20_std": 0.061659, - "nauc_map_at_20_diff1": 0.406205, - "nauc_map_at_100_max": 0.299934, - "nauc_map_at_100_std": 0.062056, - "nauc_map_at_100_diff1": 0.402882, - "nauc_map_at_1000_max": 0.299604, - "nauc_map_at_1000_std": 0.06212, - "nauc_map_at_1000_diff1": 0.402604, - "nauc_recall_at_1_max": 0.5116, - "nauc_recall_at_1_std": 0.073935, - "nauc_recall_at_1_diff1": 0.715694, - "nauc_recall_at_3_max": 0.295614, - "nauc_recall_at_3_std": 0.061532, - "nauc_recall_at_3_diff1": 0.365997, - "nauc_recall_at_5_max": 0.250882, - "nauc_recall_at_5_std": 0.072136, - "nauc_recall_at_5_diff1": 0.296869, - "nauc_recall_at_10_max": 0.187916, - "nauc_recall_at_10_std": 0.075065, - "nauc_recall_at_10_diff1": 0.227623, - "nauc_recall_at_20_max": 0.139061, - "nauc_recall_at_20_std": 0.067366, - "nauc_recall_at_20_diff1": 0.176912, - "nauc_recall_at_100_max": 0.053559, - "nauc_recall_at_100_std": 0.077563, - "nauc_recall_at_100_diff1": 0.080251, - "nauc_recall_at_1000_max": -0.03957, - "nauc_recall_at_1000_std": 0.069824, - "nauc_recall_at_1000_diff1": -0.020025, - "nauc_precision_at_1_max": 0.5116, - "nauc_precision_at_1_std": 0.073935, - "nauc_precision_at_1_diff1": 0.715694, - "nauc_precision_at_3_max": 0.295614, - "nauc_precision_at_3_std": 0.061532, - "nauc_precision_at_3_diff1": 0.365997, - "nauc_precision_at_5_max": 0.250882, - "nauc_precision_at_5_std": 0.072136, - "nauc_precision_at_5_diff1": 0.296869, - "nauc_precision_at_10_max": 0.187916, - "nauc_precision_at_10_std": 0.075065, - "nauc_precision_at_10_diff1": 0.227623, - "nauc_precision_at_20_max": 0.139061, - "nauc_precision_at_20_std": 0.067366, - "nauc_precision_at_20_diff1": 0.176912, - "nauc_precision_at_100_max": 0.053559, - "nauc_precision_at_100_std": 0.077563, - "nauc_precision_at_100_diff1": 0.080251, - "nauc_precision_at_1000_max": -0.03957, - "nauc_precision_at_1000_std": 0.069824, - "nauc_precision_at_1000_diff1": -0.020025, - "nauc_mrr_at_1_max": 0.5116, - "nauc_mrr_at_1_std": 0.073935, - "nauc_mrr_at_1_diff1": 0.715694, - "nauc_mrr_at_3_max": 0.516213, - "nauc_mrr_at_3_std": 0.084836, - "nauc_mrr_at_3_diff1": 0.68779, - "nauc_mrr_at_5_max": 0.518266, - "nauc_mrr_at_5_std": 0.090287, - "nauc_mrr_at_5_diff1": 0.684759, - "nauc_mrr_at_10_max": 0.516795, - "nauc_mrr_at_10_std": 0.091841, - "nauc_mrr_at_10_diff1": 0.682263, - "nauc_mrr_at_20_max": 0.516508, - "nauc_mrr_at_20_std": 0.092047, - "nauc_mrr_at_20_diff1": 0.682142, - "nauc_mrr_at_100_max": 0.516723, - "nauc_mrr_at_100_std": 0.093048, - "nauc_mrr_at_100_diff1": 0.682074, - "nauc_mrr_at_1000_max": 0.516691, - "nauc_mrr_at_1000_std": 0.093012, - "nauc_mrr_at_1000_diff1": 0.682124, - "main_score": 0.47809, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 3732.872656583786, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/ImdbClassification.json b/beir/ImdbClassification.json deleted file mode 100644 index a9c5c2fccc6e2164327cf06295ebce881dd5a19a..0000000000000000000000000000000000000000 --- a/beir/ImdbClassification.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "dataset_revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7", - "task_name": "ImdbClassification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.61454, - "f1": 0.611751, - "f1_weighted": 0.611751, - "ap": 0.572238, - "ap_weighted": 0.572238, - "scores_per_experiment": [ - { - "accuracy": 0.64184, - "f1": 0.641046, - "f1_weighted": 0.641046, - "ap": 0.593128, - "ap_weighted": 0.593128 - }, - { - "accuracy": 0.64316, - "f1": 0.643142, - "f1_weighted": 0.643142, - "ap": 0.591785, - "ap_weighted": 0.591785 - }, - { - "accuracy": 0.5602, - "f1": 0.556495, - "f1_weighted": 0.556495, - "ap": 0.534535, - "ap_weighted": 0.534535 - }, - { - "accuracy": 0.63604, - "f1": 0.635737, - "f1_weighted": 0.635737, - "ap": 0.58766, - "ap_weighted": 0.58766 - }, - { - "accuracy": 0.60992, - "f1": 0.604073, - "f1_weighted": 0.604073, - "ap": 0.56468, - "ap_weighted": 0.56468 - }, - { - "accuracy": 0.60764, - "f1": 0.606284, - "f1_weighted": 0.606284, - "ap": 0.566947, - "ap_weighted": 0.566947 - }, - { - "accuracy": 0.60716, - "f1": 0.592887, - "f1_weighted": 0.592887, - "ap": 0.571938, - "ap_weighted": 0.571938 - }, - { - "accuracy": 0.60104, - "f1": 0.599904, - "f1_weighted": 0.599904, - "ap": 0.561947, - "ap_weighted": 0.561947 - }, - { - "accuracy": 0.5806, - "f1": 0.580167, - "f1_weighted": 0.580167, - "ap": 0.546404, - "ap_weighted": 0.546404 - }, - { - "accuracy": 0.6578, - "f1": 0.657772, - "f1_weighted": 0.657772, - "ap": 0.603361, - "ap_weighted": 0.603361 - } - ], - "main_score": 0.61454, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 65.28035974502563, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/MSMARCO.json b/beir/MSMARCO.json deleted file mode 100644 index c6c59cd3e09c181b08253d82093028192f26b957..0000000000000000000000000000000000000000 --- a/beir/MSMARCO.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "c5a29a104738b98a9e76336939199e264163d4a0", - "task_name": "MSMARCO", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.76744, - "ndcg_at_3": 0.71949, - "ndcg_at_5": 0.68963, - "ndcg_at_10": 0.66652, - "ndcg_at_20": 0.63025, - "ndcg_at_100": 0.57169, - "ndcg_at_1000": 0.64349, - "map_at_1": 0.02314, - "map_at_3": 0.05259, - "map_at_5": 0.07976, - "map_at_10": 0.13288, - "map_at_20": 0.19885, - "map_at_100": 0.32109, - "map_at_1000": 0.38117, - "recall_at_1": 0.02314, - "recall_at_3": 0.05397, - "recall_at_5": 0.08814, - "recall_at_10": 0.15347, - "recall_at_20": 0.23955, - "recall_at_100": 0.44498, - "recall_at_1000": 0.69357, - "precision_at_1": 0.95349, - "precision_at_3": 0.86047, - "precision_at_5": 0.79535, - "precision_at_10": 0.72558, - "precision_at_20": 0.62907, - "precision_at_100": 0.31977, - "precision_at_1000": 0.06049, - "mrr_at_1": 0.953488, - "mrr_at_3": 0.965116, - "mrr_at_5": 0.969767, - "mrr_at_10": 0.969767, - "mrr_at_20": 0.969767, - "mrr_at_100": 0.969767, - "mrr_at_1000": 0.969767, - "nauc_ndcg_at_1_max": 0.335394, - "nauc_ndcg_at_1_std": 0.12295, - "nauc_ndcg_at_1_diff1": 0.160874, - "nauc_ndcg_at_3_max": 0.329494, - "nauc_ndcg_at_3_std": 0.212066, - "nauc_ndcg_at_3_diff1": 0.019303, - "nauc_ndcg_at_5_max": 0.268626, - "nauc_ndcg_at_5_std": 0.198698, - "nauc_ndcg_at_5_diff1": -0.130704, - "nauc_ndcg_at_10_max": 0.258121, - "nauc_ndcg_at_10_std": 0.314913, - "nauc_ndcg_at_10_diff1": -0.321204, - "nauc_ndcg_at_20_max": 0.308474, - "nauc_ndcg_at_20_std": 0.486443, - "nauc_ndcg_at_20_diff1": -0.412659, - "nauc_ndcg_at_100_max": 0.319625, - "nauc_ndcg_at_100_std": 0.561189, - "nauc_ndcg_at_100_diff1": -0.451756, - "nauc_ndcg_at_1000_max": 0.388789, - "nauc_ndcg_at_1000_std": 0.603502, - "nauc_ndcg_at_1000_diff1": -0.463901, - "nauc_map_at_1_max": -0.286523, - "nauc_map_at_1_std": -0.331061, - "nauc_map_at_1_diff1": -0.11994, - "nauc_map_at_3_max": -0.160122, - "nauc_map_at_3_std": -0.219302, - "nauc_map_at_3_diff1": -0.038522, - "nauc_map_at_5_max": -0.120035, - "nauc_map_at_5_std": -0.209837, - "nauc_map_at_5_diff1": -0.05801, - "nauc_map_at_10_max": -0.044091, - "nauc_map_at_10_std": -0.134428, - "nauc_map_at_10_diff1": -0.104398, - "nauc_map_at_20_max": 0.029722, - "nauc_map_at_20_std": -0.021077, - "nauc_map_at_20_diff1": -0.205479, - "nauc_map_at_100_max": 0.181352, - "nauc_map_at_100_std": 0.363231, - "nauc_map_at_100_diff1": -0.383395, - "nauc_map_at_1000_max": 0.28169, - "nauc_map_at_1000_std": 0.600619, - "nauc_map_at_1000_diff1": -0.438413, - "nauc_recall_at_1_max": -0.286523, - "nauc_recall_at_1_std": -0.331061, - "nauc_recall_at_1_diff1": -0.11994, - "nauc_recall_at_3_max": -0.162157, - "nauc_recall_at_3_std": -0.230294, - "nauc_recall_at_3_diff1": -0.042975, - "nauc_recall_at_5_max": -0.183205, - "nauc_recall_at_5_std": -0.268365, - "nauc_recall_at_5_diff1": -0.101943, - "nauc_recall_at_10_max": -0.119993, - "nauc_recall_at_10_std": -0.20104, - "nauc_recall_at_10_diff1": -0.124464, - "nauc_recall_at_20_max": -0.068623, - "nauc_recall_at_20_std": -0.114267, - "nauc_recall_at_20_diff1": -0.202857, - "nauc_recall_at_100_max": 0.055972, - "nauc_recall_at_100_std": 0.207599, - "nauc_recall_at_100_diff1": -0.354162, - "nauc_recall_at_1000_max": 0.260786, - "nauc_recall_at_1000_std": 0.526085, - "nauc_recall_at_1000_diff1": -0.411833, - "nauc_precision_at_1_max": 0.558702, - "nauc_precision_at_1_std": 0.67915, - "nauc_precision_at_1_diff1": 0.614293, - "nauc_precision_at_3_max": 0.373974, - "nauc_precision_at_3_std": 0.616163, - "nauc_precision_at_3_diff1": -0.125103, - "nauc_precision_at_5_max": 0.317393, - "nauc_precision_at_5_std": 0.453637, - "nauc_precision_at_5_diff1": -0.301655, - "nauc_precision_at_10_max": 0.346793, - "nauc_precision_at_10_std": 0.589706, - "nauc_precision_at_10_diff1": -0.421495, - "nauc_precision_at_20_max": 0.339533, - "nauc_precision_at_20_std": 0.666475, - "nauc_precision_at_20_diff1": -0.418728, - "nauc_precision_at_100_max": 0.292942, - "nauc_precision_at_100_std": 0.742772, - "nauc_precision_at_100_diff1": -0.334855, - "nauc_precision_at_1000_max": 0.2712, - "nauc_precision_at_1000_std": 0.677077, - "nauc_precision_at_1000_diff1": -0.222352, - "nauc_mrr_at_1_max": 0.558702, - "nauc_mrr_at_1_std": 0.67915, - "nauc_mrr_at_1_diff1": 0.614293, - "nauc_mrr_at_3_max": 0.705801, - "nauc_mrr_at_3_std": 0.7861, - "nauc_mrr_at_3_diff1": 0.699624, - "nauc_mrr_at_5_max": 0.66054, - "nauc_mrr_at_5_std": 0.753192, - "nauc_mrr_at_5_diff1": 0.673369, - "nauc_mrr_at_10_max": 0.66054, - "nauc_mrr_at_10_std": 0.753192, - "nauc_mrr_at_10_diff1": 0.673369, - "nauc_mrr_at_20_max": 0.66054, - "nauc_mrr_at_20_std": 0.753192, - "nauc_mrr_at_20_diff1": 0.673369, - "nauc_mrr_at_100_max": 0.66054, - "nauc_mrr_at_100_std": 0.753192, - "nauc_mrr_at_100_diff1": 0.673369, - "nauc_mrr_at_1000_max": 0.66054, - "nauc_mrr_at_1000_std": 0.753192, - "nauc_mrr_at_1000_diff1": 0.673369, - "main_score": 0.66652, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 5779.706394433975, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/MTOPDomainClassification.json b/beir/MTOPDomainClassification.json deleted file mode 100644 index a2d5ed3d0ed7706d9f7757550b6b18565fadba55..0000000000000000000000000000000000000000 --- a/beir/MTOPDomainClassification.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf", - "task_name": "MTOPDomainClassification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.889124, - "f1": 0.88758, - "f1_weighted": 0.889747, - "scores_per_experiment": [ - { - "accuracy": 0.854537, - "f1": 0.857721, - "f1_weighted": 0.855151 - }, - { - "accuracy": 0.900593, - "f1": 0.89766, - "f1_weighted": 0.901084 - }, - { - "accuracy": 0.901049, - "f1": 0.898648, - "f1_weighted": 0.900877 - }, - { - "accuracy": 0.897629, - "f1": 0.894282, - "f1_weighted": 0.898338 - }, - { - "accuracy": 0.891473, - "f1": 0.890253, - "f1_weighted": 0.892135 - }, - { - "accuracy": 0.882809, - "f1": 0.884257, - "f1_weighted": 0.885064 - }, - { - "accuracy": 0.887597, - "f1": 0.884408, - "f1_weighted": 0.887453 - }, - { - "accuracy": 0.891017, - "f1": 0.887588, - "f1_weighted": 0.890818 - }, - { - "accuracy": 0.891929, - "f1": 0.89255, - "f1_weighted": 0.892768 - }, - { - "accuracy": 0.892613, - "f1": 0.888437, - "f1_weighted": 0.893779 - } - ], - "main_score": 0.889124, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 10.538909912109375, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/MTOPIntentClassification.json b/beir/MTOPIntentClassification.json deleted file mode 100644 index dba6fdf6af688794239af47f19daf79db27cd0a6..0000000000000000000000000000000000000000 --- a/beir/MTOPIntentClassification.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "dataset_revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba", - "task_name": "MTOPIntentClassification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.656475, - "f1": 0.490948, - "f1_weighted": 0.693994, - "scores_per_experiment": [ - { - "accuracy": 0.641587, - "f1": 0.456483, - "f1_weighted": 0.683725 - }, - { - "accuracy": 0.671911, - "f1": 0.48881, - "f1_weighted": 0.709701 - }, - { - "accuracy": 0.673279, - "f1": 0.501567, - "f1_weighted": 0.711032 - }, - { - "accuracy": 0.656407, - "f1": 0.514271, - "f1_weighted": 0.692525 - }, - { - "accuracy": 0.639763, - "f1": 0.480901, - "f1_weighted": 0.671092 - }, - { - "accuracy": 0.645691, - "f1": 0.501634, - "f1_weighted": 0.685544 - }, - { - "accuracy": 0.663703, - "f1": 0.479191, - "f1_weighted": 0.704379 - }, - { - "accuracy": 0.674647, - "f1": 0.516005, - "f1_weighted": 0.709525 - }, - { - "accuracy": 0.650935, - "f1": 0.480141, - "f1_weighted": 0.689081 - }, - { - "accuracy": 0.646831, - "f1": 0.490481, - "f1_weighted": 0.683338 - } - ], - "main_score": 0.656475, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 13.826844215393066, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/MassiveIntentClassification.json b/beir/MassiveIntentClassification.json deleted file mode 100644 index 721571e6adc8ed76f47c4f3021351e1ed3a2de0f..0000000000000000000000000000000000000000 --- a/beir/MassiveIntentClassification.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", - "task_name": "MassiveIntentClassification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.652354, - "f1": 0.635945, - "f1_weighted": 0.650395, - "scores_per_experiment": [ - { - "accuracy": 0.644923, - "f1": 0.639513, - "f1_weighted": 0.640635 - }, - { - "accuracy": 0.677539, - "f1": 0.652575, - "f1_weighted": 0.67554 - }, - { - "accuracy": 0.649294, - "f1": 0.630992, - "f1_weighted": 0.645129 - }, - { - "accuracy": 0.680565, - "f1": 0.659978, - "f1_weighted": 0.675892 - }, - { - "accuracy": 0.653329, - "f1": 0.637184, - "f1_weighted": 0.649634 - }, - { - "accuracy": 0.641896, - "f1": 0.624381, - "f1_weighted": 0.640526 - }, - { - "accuracy": 0.642233, - "f1": 0.633383, - "f1_weighted": 0.643923 - }, - { - "accuracy": 0.638534, - "f1": 0.618431, - "f1_weighted": 0.633252 - }, - { - "accuracy": 0.64694, - "f1": 0.63405, - "f1_weighted": 0.649656 - }, - { - "accuracy": 0.648285, - "f1": 0.628963, - "f1_weighted": 0.649767 - } - ], - "main_score": 0.652354, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 11.81202483177185, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/MassiveScenarioClassification.json b/beir/MassiveScenarioClassification.json deleted file mode 100644 index babf7fc54d8db2f7b4be2f991d2dabb8f0bddf29..0000000000000000000000000000000000000000 --- a/beir/MassiveScenarioClassification.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", - "task_name": "MassiveScenarioClassification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.726126, - "f1": 0.724117, - "f1_weighted": 0.726333, - "scores_per_experiment": [ - { - "accuracy": 0.726295, - "f1": 0.724368, - "f1_weighted": 0.72364 - }, - { - "accuracy": 0.734701, - "f1": 0.737144, - "f1_weighted": 0.733128 - }, - { - "accuracy": 0.711163, - "f1": 0.711532, - "f1_weighted": 0.709362 - }, - { - "accuracy": 0.705783, - "f1": 0.699515, - "f1_weighted": 0.709819 - }, - { - "accuracy": 0.737727, - "f1": 0.733102, - "f1_weighted": 0.739757 - }, - { - "accuracy": 0.740081, - "f1": 0.729691, - "f1_weighted": 0.74056 - }, - { - "accuracy": 0.72226, - "f1": 0.724384, - "f1_weighted": 0.726585 - }, - { - "accuracy": 0.722596, - "f1": 0.725063, - "f1_weighted": 0.721339 - }, - { - "accuracy": 0.736382, - "f1": 0.733771, - "f1_weighted": 0.736615 - }, - { - "accuracy": 0.724277, - "f1": 0.722599, - "f1_weighted": 0.722527 - } - ], - "main_score": 0.726126, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 4.415860176086426, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/MedrxivClusteringP2P.json b/beir/MedrxivClusteringP2P.json deleted file mode 100644 index 4da03e3d05c1b17a76a89bb5d9255c6760bbd8f7..0000000000000000000000000000000000000000 --- a/beir/MedrxivClusteringP2P.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", - "task_name": "MedrxivClusteringP2P", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "v_measure": 0.2895, - "v_measure_std": 0.013911, - "v_measures": [ - 0.271819, - 0.271103, - 0.278713, - 0.272723, - 0.288926, - 0.304441, - 0.300211, - 0.303506, - 0.306792, - 0.296767 - ], - "main_score": 0.2895, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 114.98579716682434, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/MedrxivClusteringS2S.json b/beir/MedrxivClusteringS2S.json deleted file mode 100644 index 402d3cfeadeb178dfe54d58849b324f363ecfa06..0000000000000000000000000000000000000000 --- a/beir/MedrxivClusteringS2S.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", - "task_name": "MedrxivClusteringS2S", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "v_measure": 0.27076, - "v_measure_std": 0.017323, - "v_measures": [ - 0.256262, - 0.252559, - 0.253638, - 0.253727, - 0.25903, - 0.301547, - 0.277187, - 0.287709, - 0.292024, - 0.273922 - ], - "main_score": 0.27076, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 17.49012589454651, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/MindSmallReranking.json b/beir/MindSmallReranking.json deleted file mode 100644 index 8e10f2e398d07f63fe0c96596c107b10107ce02c..0000000000000000000000000000000000000000 --- a/beir/MindSmallReranking.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "59042f120c80e8afa9cdbb224f67076cec0fc9a7", - "task_name": "MindSmallReranking", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "map": 0.306437, - "mrr": 0.316768, - "nAUC_map_max": -0.182556, - "nAUC_map_std": -0.016765, - "nAUC_map_diff1": 0.134889, - "nAUC_mrr_max": -0.128279, - "nAUC_mrr_std": 0.003295, - "nAUC_mrr_diff1": 0.128104, - "main_score": 0.306437, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 120.96634793281555, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/NFCorpus.json b/beir/NFCorpus.json deleted file mode 100644 index d503c7504bd1e7e1dd1953b625928e5c2b680889..0000000000000000000000000000000000000000 --- a/beir/NFCorpus.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814", - "task_name": "NFCorpus", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.34675, - "ndcg_at_3": 0.31048, - "ndcg_at_5": 0.29451, - "ndcg_at_10": 0.2663, - "ndcg_at_20": 0.24402, - "ndcg_at_100": 0.24319, - "ndcg_at_1000": 0.33458, - "map_at_1": 0.04323, - "map_at_3": 0.06706, - "map_at_5": 0.08085, - "map_at_10": 0.09278, - "map_at_20": 0.1011, - "map_at_100": 0.11312, - "map_at_1000": 0.12461, - "recall_at_1": 0.04323, - "recall_at_3": 0.07573, - "recall_at_5": 0.10248, - "recall_at_10": 0.12779, - "recall_at_20": 0.15931, - "recall_at_100": 0.25619, - "recall_at_1000": 0.57667, - "precision_at_1": 0.36223, - "precision_at_3": 0.29102, - "precision_at_5": 0.25573, - "precision_at_10": 0.19505, - "precision_at_20": 0.13824, - "precision_at_100": 0.06037, - "precision_at_1000": 0.0188, - "mrr_at_1": 0.362229, - "mrr_at_3": 0.427245, - "mrr_at_5": 0.443189, - "mrr_at_10": 0.452435, - "mrr_at_20": 0.45666, - "mrr_at_100": 0.459342, - "mrr_at_1000": 0.459762, - "nauc_ndcg_at_1_max": 0.489247, - "nauc_ndcg_at_1_std": 0.207882, - "nauc_ndcg_at_1_diff1": 0.455278, - "nauc_ndcg_at_3_max": 0.484014, - "nauc_ndcg_at_3_std": 0.279633, - "nauc_ndcg_at_3_diff1": 0.373273, - "nauc_ndcg_at_5_max": 0.499214, - "nauc_ndcg_at_5_std": 0.285193, - "nauc_ndcg_at_5_diff1": 0.377699, - "nauc_ndcg_at_10_max": 0.471546, - "nauc_ndcg_at_10_std": 0.303156, - "nauc_ndcg_at_10_diff1": 0.326637, - "nauc_ndcg_at_20_max": 0.452904, - "nauc_ndcg_at_20_std": 0.286146, - "nauc_ndcg_at_20_diff1": 0.311691, - "nauc_ndcg_at_100_max": 0.471783, - "nauc_ndcg_at_100_std": 0.276204, - "nauc_ndcg_at_100_diff1": 0.330981, - "nauc_ndcg_at_1000_max": 0.534989, - "nauc_ndcg_at_1000_std": 0.320777, - "nauc_ndcg_at_1000_diff1": 0.346119, - "nauc_map_at_1_max": 0.328405, - "nauc_map_at_1_std": -0.055909, - "nauc_map_at_1_diff1": 0.464563, - "nauc_map_at_3_max": 0.317683, - "nauc_map_at_3_std": -0.002782, - "nauc_map_at_3_diff1": 0.38562, - "nauc_map_at_5_max": 0.374515, - "nauc_map_at_5_std": 0.023936, - "nauc_map_at_5_diff1": 0.41153, - "nauc_map_at_10_max": 0.397591, - "nauc_map_at_10_std": 0.069354, - "nauc_map_at_10_diff1": 0.375102, - "nauc_map_at_20_max": 0.412683, - "nauc_map_at_20_std": 0.109156, - "nauc_map_at_20_diff1": 0.360845, - "nauc_map_at_100_max": 0.424984, - "nauc_map_at_100_std": 0.159183, - "nauc_map_at_100_diff1": 0.342464, - "nauc_map_at_1000_max": 0.43113, - "nauc_map_at_1000_std": 0.18875, - "nauc_map_at_1000_diff1": 0.333251, - "nauc_recall_at_1_max": 0.328405, - "nauc_recall_at_1_std": -0.055909, - "nauc_recall_at_1_diff1": 0.464563, - "nauc_recall_at_3_max": 0.266527, - "nauc_recall_at_3_std": -0.012704, - "nauc_recall_at_3_diff1": 0.333511, - "nauc_recall_at_5_max": 0.317243, - "nauc_recall_at_5_std": -0.002978, - "nauc_recall_at_5_diff1": 0.38356, - "nauc_recall_at_10_max": 0.307398, - "nauc_recall_at_10_std": 0.039968, - "nauc_recall_at_10_diff1": 0.313532, - "nauc_recall_at_20_max": 0.304815, - "nauc_recall_at_20_std": 0.072316, - "nauc_recall_at_20_diff1": 0.271323, - "nauc_recall_at_100_max": 0.278974, - "nauc_recall_at_100_std": 0.152896, - "nauc_recall_at_100_diff1": 0.216284, - "nauc_recall_at_1000_max": 0.242439, - "nauc_recall_at_1000_std": 0.170621, - "nauc_recall_at_1000_diff1": 0.144589, - "nauc_precision_at_1_max": 0.493121, - "nauc_precision_at_1_std": 0.211866, - "nauc_precision_at_1_diff1": 0.439682, - "nauc_precision_at_3_max": 0.482029, - "nauc_precision_at_3_std": 0.326316, - "nauc_precision_at_3_diff1": 0.30629, - "nauc_precision_at_5_max": 0.501486, - "nauc_precision_at_5_std": 0.359998, - "nauc_precision_at_5_diff1": 0.271906, - "nauc_precision_at_10_max": 0.437322, - "nauc_precision_at_10_std": 0.424984, - "nauc_precision_at_10_diff1": 0.151228, - "nauc_precision_at_20_max": 0.363724, - "nauc_precision_at_20_std": 0.454707, - "nauc_precision_at_20_diff1": 0.083594, - "nauc_precision_at_100_max": 0.213989, - "nauc_precision_at_100_std": 0.452484, - "nauc_precision_at_100_diff1": -0.018369, - "nauc_precision_at_1000_max": 0.062987, - "nauc_precision_at_1000_std": 0.315722, - "nauc_precision_at_1000_diff1": -0.063268, - "nauc_mrr_at_1_max": 0.493121, - "nauc_mrr_at_1_std": 0.211866, - "nauc_mrr_at_1_diff1": 0.439682, - "nauc_mrr_at_3_max": 0.502532, - "nauc_mrr_at_3_std": 0.249172, - "nauc_mrr_at_3_diff1": 0.410622, - "nauc_mrr_at_5_max": 0.515072, - "nauc_mrr_at_5_std": 0.250577, - "nauc_mrr_at_5_diff1": 0.409294, - "nauc_mrr_at_10_max": 0.510025, - "nauc_mrr_at_10_std": 0.25547, - "nauc_mrr_at_10_diff1": 0.404908, - "nauc_mrr_at_20_max": 0.510314, - "nauc_mrr_at_20_std": 0.258227, - "nauc_mrr_at_20_diff1": 0.404249, - "nauc_mrr_at_100_max": 0.511729, - "nauc_mrr_at_100_std": 0.258183, - "nauc_mrr_at_100_diff1": 0.405071, - "nauc_mrr_at_1000_max": 0.511509, - "nauc_mrr_at_1000_std": 0.257945, - "nauc_mrr_at_1000_diff1": 0.405027, - "main_score": 0.2663, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 9.661004543304443, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/NQ.json b/beir/NQ.json deleted file mode 100644 index 129c5ebcfad2efa4dad322e6745864f8f9552936..0000000000000000000000000000000000000000 --- a/beir/NQ.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31", - "task_name": "NQ", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.28708, - "ndcg_at_3": 0.38322, - "ndcg_at_5": 0.42087, - "ndcg_at_10": 0.4532, - "ndcg_at_20": 0.4742, - "ndcg_at_100": 0.49995, - "ndcg_at_1000": 0.51263, - "map_at_1": 0.25702, - "map_at_3": 0.34911, - "map_at_5": 0.37137, - "map_at_10": 0.38587, - "map_at_20": 0.39226, - "map_at_100": 0.39638, - "map_at_1000": 0.39689, - "recall_at_1": 0.25702, - "recall_at_3": 0.45394, - "recall_at_5": 0.54089, - "recall_at_10": 0.63504, - "recall_at_20": 0.71323, - "recall_at_100": 0.84154, - "recall_at_1000": 0.93687, - "precision_at_1": 0.28708, - "precision_at_3": 0.17468, - "precision_at_5": 0.1263, - "precision_at_10": 0.07512, - "precision_at_20": 0.0425, - "precision_at_100": 0.01018, - "precision_at_1000": 0.00114, - "mrr_at_1": 0.28708, - "mrr_at_3": 0.377221, - "mrr_at_5": 0.396833, - "mrr_at_10": 0.409104, - "mrr_at_20": 0.41419, - "mrr_at_100": 0.417396, - "mrr_at_1000": 0.417789, - "nauc_ndcg_at_1_max": 0.250479, - "nauc_ndcg_at_1_std": 0.019234, - "nauc_ndcg_at_1_diff1": 0.335807, - "nauc_ndcg_at_3_max": 0.294583, - "nauc_ndcg_at_3_std": 0.050666, - "nauc_ndcg_at_3_diff1": 0.293529, - "nauc_ndcg_at_5_max": 0.307405, - "nauc_ndcg_at_5_std": 0.060516, - "nauc_ndcg_at_5_diff1": 0.287018, - "nauc_ndcg_at_10_max": 0.320593, - "nauc_ndcg_at_10_std": 0.084058, - "nauc_ndcg_at_10_diff1": 0.287482, - "nauc_ndcg_at_20_max": 0.328514, - "nauc_ndcg_at_20_std": 0.096083, - "nauc_ndcg_at_20_diff1": 0.286127, - "nauc_ndcg_at_100_max": 0.324329, - "nauc_ndcg_at_100_std": 0.105305, - "nauc_ndcg_at_100_diff1": 0.283013, - "nauc_ndcg_at_1000_max": 0.318549, - "nauc_ndcg_at_1000_std": 0.097215, - "nauc_ndcg_at_1000_diff1": 0.286643, - "nauc_map_at_1_max": 0.242088, - "nauc_map_at_1_std": 0.00276, - "nauc_map_at_1_diff1": 0.340467, - "nauc_map_at_3_max": 0.282923, - "nauc_map_at_3_std": 0.036139, - "nauc_map_at_3_diff1": 0.305555, - "nauc_map_at_5_max": 0.291093, - "nauc_map_at_5_std": 0.042332, - "nauc_map_at_5_diff1": 0.302144, - "nauc_map_at_10_max": 0.29688, - "nauc_map_at_10_std": 0.053188, - "nauc_map_at_10_diff1": 0.301168, - "nauc_map_at_20_max": 0.299107, - "nauc_map_at_20_std": 0.0566, - "nauc_map_at_20_diff1": 0.301049, - "nauc_map_at_100_max": 0.298118, - "nauc_map_at_100_std": 0.057722, - "nauc_map_at_100_diff1": 0.300482, - "nauc_map_at_1000_max": 0.297934, - "nauc_map_at_1000_std": 0.057475, - "nauc_map_at_1000_diff1": 0.300611, - "nauc_recall_at_1_max": 0.242088, - "nauc_recall_at_1_std": 0.00276, - "nauc_recall_at_1_diff1": 0.340467, - "nauc_recall_at_3_max": 0.314631, - "nauc_recall_at_3_std": 0.072352, - "nauc_recall_at_3_diff1": 0.260977, - "nauc_recall_at_5_max": 0.343255, - "nauc_recall_at_5_std": 0.094105, - "nauc_recall_at_5_diff1": 0.242641, - "nauc_recall_at_10_max": 0.389135, - "nauc_recall_at_10_std": 0.170459, - "nauc_recall_at_10_diff1": 0.240436, - "nauc_recall_at_20_max": 0.440327, - "nauc_recall_at_20_std": 0.243425, - "nauc_recall_at_20_diff1": 0.225185, - "nauc_recall_at_100_max": 0.488993, - "nauc_recall_at_100_std": 0.423469, - "nauc_recall_at_100_diff1": 0.170894, - "nauc_recall_at_1000_max": 0.535473, - "nauc_recall_at_1000_std": 0.585053, - "nauc_recall_at_1000_diff1": 0.161342, - "nauc_precision_at_1_max": 0.250479, - "nauc_precision_at_1_std": 0.019234, - "nauc_precision_at_1_diff1": 0.335807, - "nauc_precision_at_3_max": 0.312609, - "nauc_precision_at_3_std": 0.098992, - "nauc_precision_at_3_diff1": 0.232633, - "nauc_precision_at_5_max": 0.321341, - "nauc_precision_at_5_std": 0.121296, - "nauc_precision_at_5_diff1": 0.194865, - "nauc_precision_at_10_max": 0.316709, - "nauc_precision_at_10_std": 0.18397, - "nauc_precision_at_10_diff1": 0.157855, - "nauc_precision_at_20_max": 0.310475, - "nauc_precision_at_20_std": 0.225964, - "nauc_precision_at_20_diff1": 0.117234, - "nauc_precision_at_100_max": 0.217555, - "nauc_precision_at_100_std": 0.269504, - "nauc_precision_at_100_diff1": 0.029149, - "nauc_precision_at_1000_max": 0.094419, - "nauc_precision_at_1000_std": 0.203673, - "nauc_precision_at_1000_diff1": -0.029269, - "nauc_mrr_at_1_max": 0.250479, - "nauc_mrr_at_1_std": 0.019234, - "nauc_mrr_at_1_diff1": 0.335807, - "nauc_mrr_at_3_max": 0.285525, - "nauc_mrr_at_3_std": 0.053665, - "nauc_mrr_at_3_diff1": 0.296797, - "nauc_mrr_at_5_max": 0.290497, - "nauc_mrr_at_5_std": 0.058275, - "nauc_mrr_at_5_diff1": 0.293153, - "nauc_mrr_at_10_max": 0.29465, - "nauc_mrr_at_10_std": 0.065692, - "nauc_mrr_at_10_diff1": 0.295064, - "nauc_mrr_at_20_max": 0.295426, - "nauc_mrr_at_20_std": 0.066635, - "nauc_mrr_at_20_diff1": 0.295253, - "nauc_mrr_at_100_max": 0.294819, - "nauc_mrr_at_100_std": 0.067348, - "nauc_mrr_at_100_diff1": 0.295063, - "nauc_mrr_at_1000_max": 0.29466, - "nauc_mrr_at_1000_std": 0.067075, - "nauc_mrr_at_1000_diff1": 0.295171, - "main_score": 0.4532, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 2530.8694610595703, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/QuoraRetrieval.json b/beir/QuoraRetrieval.json deleted file mode 100644 index 5616819ca4cf7cb035fdd3266e71157250b2c502..0000000000000000000000000000000000000000 --- a/beir/QuoraRetrieval.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "e4e08e0b7dbe3c8700f0daef558ff32256715259", - "task_name": "QuoraRetrieval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.7981, - "ndcg_at_3": 0.83923, - "ndcg_at_5": 0.85516, - "ndcg_at_10": 0.86825, - "ndcg_at_20": 0.87562, - "ndcg_at_100": 0.88241, - "ndcg_at_1000": 0.88406, - "map_at_1": 0.69387, - "map_at_3": 0.80109, - "map_at_5": 0.81907, - "map_at_10": 0.82976, - "map_at_20": 0.83391, - "map_at_100": 0.83627, - "map_at_1000": 0.83648, - "recall_at_1": 0.69387, - "recall_at_3": 0.85762, - "recall_at_5": 0.90305, - "recall_at_10": 0.94156, - "recall_at_20": 0.9656, - "recall_at_100": 0.99099, - "recall_at_1000": 0.99905, - "precision_at_1": 0.7981, - "precision_at_3": 0.3661, - "precision_at_5": 0.24042, - "precision_at_10": 0.13102, - "precision_at_20": 0.06961, - "precision_at_100": 0.0151, - "precision_at_1000": 0.00156, - "mrr_at_1": 0.7979, - "mrr_at_3": 0.8504, - "mrr_at_5": 0.85789, - "mrr_at_10": 0.861217, - "mrr_at_20": 0.862272, - "mrr_at_100": 0.862594, - "mrr_at_1000": 0.862613, - "nauc_ndcg_at_1_max": 0.385323, - "nauc_ndcg_at_1_std": -0.352997, - "nauc_ndcg_at_1_diff1": 0.768904, - "nauc_ndcg_at_3_max": 0.358767, - "nauc_ndcg_at_3_std": -0.422362, - "nauc_ndcg_at_3_diff1": 0.750812, - "nauc_ndcg_at_5_max": 0.365901, - "nauc_ndcg_at_5_std": -0.437432, - "nauc_ndcg_at_5_diff1": 0.756063, - "nauc_ndcg_at_10_max": 0.371952, - "nauc_ndcg_at_10_std": -0.432721, - "nauc_ndcg_at_10_diff1": 0.758216, - "nauc_ndcg_at_20_max": 0.375014, - "nauc_ndcg_at_20_std": -0.419042, - "nauc_ndcg_at_20_diff1": 0.75704, - "nauc_ndcg_at_100_max": 0.380351, - "nauc_ndcg_at_100_std": -0.39988, - "nauc_ndcg_at_100_diff1": 0.756946, - "nauc_ndcg_at_1000_max": 0.380961, - "nauc_ndcg_at_1000_std": -0.395691, - "nauc_ndcg_at_1000_diff1": 0.756467, - "nauc_map_at_1_max": 0.276589, - "nauc_map_at_1_std": -0.386526, - "nauc_map_at_1_diff1": 0.7886, - "nauc_map_at_3_max": 0.332655, - "nauc_map_at_3_std": -0.448285, - "nauc_map_at_3_diff1": 0.762881, - "nauc_map_at_5_max": 0.348765, - "nauc_map_at_5_std": -0.448588, - "nauc_map_at_5_diff1": 0.761948, - "nauc_map_at_10_max": 0.357223, - "nauc_map_at_10_std": -0.437594, - "nauc_map_at_10_diff1": 0.760072, - "nauc_map_at_20_max": 0.360747, - "nauc_map_at_20_std": -0.427652, - "nauc_map_at_20_diff1": 0.75873, - "nauc_map_at_100_max": 0.363031, - "nauc_map_at_100_std": -0.420231, - "nauc_map_at_100_diff1": 0.758438, - "nauc_map_at_1000_max": 0.363384, - "nauc_map_at_1000_std": -0.41938, - "nauc_map_at_1000_diff1": 0.758393, - "nauc_recall_at_1_max": 0.276589, - "nauc_recall_at_1_std": -0.386526, - "nauc_recall_at_1_diff1": 0.7886, - "nauc_recall_at_3_max": 0.29953, - "nauc_recall_at_3_std": -0.507123, - "nauc_recall_at_3_diff1": 0.723936, - "nauc_recall_at_5_max": 0.318398, - "nauc_recall_at_5_std": -0.564493, - "nauc_recall_at_5_diff1": 0.717385, - "nauc_recall_at_10_max": 0.330489, - "nauc_recall_at_10_std": -0.596467, - "nauc_recall_at_10_diff1": 0.714574, - "nauc_recall_at_20_max": 0.34185, - "nauc_recall_at_20_std": -0.570183, - "nauc_recall_at_20_diff1": 0.707112, - "nauc_recall_at_100_max": 0.426654, - "nauc_recall_at_100_std": -0.408769, - "nauc_recall_at_100_diff1": 0.720717, - "nauc_recall_at_1000_max": 0.441165, - "nauc_recall_at_1000_std": 0.174422, - "nauc_recall_at_1000_diff1": 0.582066, - "nauc_precision_at_1_max": 0.385323, - "nauc_precision_at_1_std": -0.352997, - "nauc_precision_at_1_diff1": 0.768904, - "nauc_precision_at_3_max": 0.115283, - "nauc_precision_at_3_std": 0.034073, - "nauc_precision_at_3_diff1": -0.11805, - "nauc_precision_at_5_max": 0.067543, - "nauc_precision_at_5_std": 0.137509, - "nauc_precision_at_5_diff1": -0.254885, - "nauc_precision_at_10_max": 0.025493, - "nauc_precision_at_10_std": 0.236938, - "nauc_precision_at_10_diff1": -0.342161, - "nauc_precision_at_20_max": -0.001939, - "nauc_precision_at_20_std": 0.310031, - "nauc_precision_at_20_diff1": -0.388267, - "nauc_precision_at_100_max": -0.02205, - "nauc_precision_at_100_std": 0.38886, - "nauc_precision_at_100_diff1": -0.41661, - "nauc_precision_at_1000_max": -0.027452, - "nauc_precision_at_1000_std": 0.411492, - "nauc_precision_at_1000_diff1": -0.424162, - "nauc_mrr_at_1_max": 0.383192, - "nauc_mrr_at_1_std": -0.353824, - "nauc_mrr_at_1_diff1": 0.769376, - "nauc_mrr_at_3_max": 0.386992, - "nauc_mrr_at_3_std": -0.374313, - "nauc_mrr_at_3_diff1": 0.758136, - "nauc_mrr_at_5_max": 0.389249, - "nauc_mrr_at_5_std": -0.377732, - "nauc_mrr_at_5_diff1": 0.760258, - "nauc_mrr_at_10_max": 0.390148, - "nauc_mrr_at_10_std": -0.375749, - "nauc_mrr_at_10_diff1": 0.761165, - "nauc_mrr_at_20_max": 0.389778, - "nauc_mrr_at_20_std": -0.373585, - "nauc_mrr_at_20_diff1": 0.761091, - "nauc_mrr_at_100_max": 0.389507, - "nauc_mrr_at_100_std": -0.373448, - "nauc_mrr_at_100_diff1": 0.761166, - "nauc_mrr_at_1000_max": 0.389482, - "nauc_mrr_at_1000_std": -0.373441, - "nauc_mrr_at_1000_diff1": 0.76116, - "main_score": 0.86825, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 304.4700849056244, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/RedditClustering.json b/beir/RedditClustering.json deleted file mode 100644 index c68274e1dc54f8f9e6f90fd5c5ce154ffb840870..0000000000000000000000000000000000000000 --- a/beir/RedditClustering.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "dataset_revision": "24640382cdbf8abc73003fb0fa6d111a705499eb", - "task_name": "RedditClustering", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "v_measure": 0.411257, - "v_measure_std": 0.039637, - "v_measures": [ - 0.47826, - 0.463098, - 0.378164, - 0.408031, - 0.375972, - 0.408828, - 0.442864, - 0.366117, - 0.364104, - 0.38536, - 0.404472, - 0.412944, - 0.391103, - 0.42435, - 0.401238, - 0.36973, - 0.433922, - 0.482263, - 0.399745, - 0.386039, - 0.37345, - 0.392968, - 0.523073, - 0.426964, - 0.388366 - ], - "main_score": 0.411257, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 178.5294632911682, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/RedditClusteringP2P.json b/beir/RedditClusteringP2P.json deleted file mode 100644 index 11c619d8ad5d1eed5f3c3ee9d98164686c15eb43..0000000000000000000000000000000000000000 --- a/beir/RedditClusteringP2P.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "dataset_revision": "385e3cb46b4cfa89021f56c4380204149d0efe33", - "task_name": "RedditClusteringP2P", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "v_measure": 0.513872, - "v_measure_std": 0.116705, - "v_measures": [ - 0.563847, - 0.569759, - 0.580057, - 0.339426, - 0.569614, - 0.511353, - 0.24021, - 0.608951, - 0.56097, - 0.594533 - ], - "main_score": 0.513872, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 662.1253287792206, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/SCIDOCS.json b/beir/SCIDOCS.json deleted file mode 100644 index 1ceb7e42420b27ddc8acd0164972514cc1c66c8b..0000000000000000000000000000000000000000 --- a/beir/SCIDOCS.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88", - "task_name": "SCIDOCS", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.173, - "ndcg_at_3": 0.13497, - "ndcg_at_5": 0.11591, - "ndcg_at_10": 0.13679, - "ndcg_at_20": 0.15566, - "ndcg_at_100": 0.19218, - "ndcg_at_1000": 0.23768, - "map_at_1": 0.0349, - "map_at_3": 0.05844, - "map_at_5": 0.069, - "map_at_10": 0.07798, - "map_at_20": 0.08404, - "map_at_100": 0.09027, - "map_at_1000": 0.09238, - "recall_at_1": 0.0349, - "recall_at_3": 0.0747, - "recall_at_5": 0.10045, - "recall_at_10": 0.1389, - "recall_at_20": 0.18338, - "recall_at_100": 0.3019, - "recall_at_1000": 0.52682, - "precision_at_1": 0.173, - "precision_at_3": 0.12333, - "precision_at_5": 0.0992, - "precision_at_10": 0.0685, - "precision_at_20": 0.0452, - "precision_at_100": 0.01486, - "precision_at_1000": 0.00259, - "mrr_at_1": 0.173, - "mrr_at_3": 0.231667, - "mrr_at_5": 0.244817, - "mrr_at_10": 0.256625, - "mrr_at_20": 0.262569, - "mrr_at_100": 0.267332, - "mrr_at_1000": 0.268125, - "nauc_ndcg_at_1_max": 0.242005, - "nauc_ndcg_at_1_std": 0.119834, - "nauc_ndcg_at_1_diff1": 0.24468, - "nauc_ndcg_at_3_max": 0.29079, - "nauc_ndcg_at_3_std": 0.16193, - "nauc_ndcg_at_3_diff1": 0.212417, - "nauc_ndcg_at_5_max": 0.302191, - "nauc_ndcg_at_5_std": 0.183763, - "nauc_ndcg_at_5_diff1": 0.194039, - "nauc_ndcg_at_10_max": 0.307147, - "nauc_ndcg_at_10_std": 0.215882, - "nauc_ndcg_at_10_diff1": 0.177705, - "nauc_ndcg_at_20_max": 0.311477, - "nauc_ndcg_at_20_std": 0.2469, - "nauc_ndcg_at_20_diff1": 0.169089, - "nauc_ndcg_at_100_max": 0.321562, - "nauc_ndcg_at_100_std": 0.30425, - "nauc_ndcg_at_100_diff1": 0.171303, - "nauc_ndcg_at_1000_max": 0.324754, - "nauc_ndcg_at_1000_std": 0.325746, - "nauc_ndcg_at_1000_diff1": 0.174281, - "nauc_map_at_1_max": 0.244036, - "nauc_map_at_1_std": 0.122252, - "nauc_map_at_1_diff1": 0.245346, - "nauc_map_at_3_max": 0.30043, - "nauc_map_at_3_std": 0.162703, - "nauc_map_at_3_diff1": 0.218286, - "nauc_map_at_5_max": 0.315919, - "nauc_map_at_5_std": 0.186765, - "nauc_map_at_5_diff1": 0.191174, - "nauc_map_at_10_max": 0.321404, - "nauc_map_at_10_std": 0.210341, - "nauc_map_at_10_diff1": 0.183068, - "nauc_map_at_20_max": 0.325777, - "nauc_map_at_20_std": 0.230543, - "nauc_map_at_20_diff1": 0.175999, - "nauc_map_at_100_max": 0.32757, - "nauc_map_at_100_std": 0.252143, - "nauc_map_at_100_diff1": 0.175648, - "nauc_map_at_1000_max": 0.32836, - "nauc_map_at_1000_std": 0.256243, - "nauc_map_at_1000_diff1": 0.176171, - "nauc_recall_at_1_max": 0.244036, - "nauc_recall_at_1_std": 0.122252, - "nauc_recall_at_1_diff1": 0.245346, - "nauc_recall_at_3_max": 0.304984, - "nauc_recall_at_3_std": 0.17664, - "nauc_recall_at_3_diff1": 0.193246, - "nauc_recall_at_5_max": 0.308366, - "nauc_recall_at_5_std": 0.203564, - "nauc_recall_at_5_diff1": 0.156459, - "nauc_recall_at_10_max": 0.303941, - "nauc_recall_at_10_std": 0.251872, - "nauc_recall_at_10_diff1": 0.122576, - "nauc_recall_at_20_max": 0.294931, - "nauc_recall_at_20_std": 0.298445, - "nauc_recall_at_20_diff1": 0.102911, - "nauc_recall_at_100_max": 0.295687, - "nauc_recall_at_100_std": 0.404675, - "nauc_recall_at_100_diff1": 0.108211, - "nauc_recall_at_1000_max": 0.261128, - "nauc_recall_at_1000_std": 0.42622, - "nauc_recall_at_1000_diff1": 0.095162, - "nauc_precision_at_1_max": 0.242005, - "nauc_precision_at_1_std": 0.119834, - "nauc_precision_at_1_diff1": 0.24468, - "nauc_precision_at_3_max": 0.305687, - "nauc_precision_at_3_std": 0.175571, - "nauc_precision_at_3_diff1": 0.193951, - "nauc_precision_at_5_max": 0.310911, - "nauc_precision_at_5_std": 0.203866, - "nauc_precision_at_5_diff1": 0.160506, - "nauc_precision_at_10_max": 0.310221, - "nauc_precision_at_10_std": 0.253927, - "nauc_precision_at_10_diff1": 0.128608, - "nauc_precision_at_20_max": 0.300773, - "nauc_precision_at_20_std": 0.299155, - "nauc_precision_at_20_diff1": 0.108257, - "nauc_precision_at_100_max": 0.299943, - "nauc_precision_at_100_std": 0.4021, - "nauc_precision_at_100_diff1": 0.112121, - "nauc_precision_at_1000_max": 0.263822, - "nauc_precision_at_1000_std": 0.417393, - "nauc_precision_at_1000_diff1": 0.099425, - "nauc_mrr_at_1_max": 0.242005, - "nauc_mrr_at_1_std": 0.119834, - "nauc_mrr_at_1_diff1": 0.24468, - "nauc_mrr_at_3_max": 0.265063, - "nauc_mrr_at_3_std": 0.146784, - "nauc_mrr_at_3_diff1": 0.224465, - "nauc_mrr_at_5_max": 0.270678, - "nauc_mrr_at_5_std": 0.151935, - "nauc_mrr_at_5_diff1": 0.226012, - "nauc_mrr_at_10_max": 0.269741, - "nauc_mrr_at_10_std": 0.159622, - "nauc_mrr_at_10_diff1": 0.217963, - "nauc_mrr_at_20_max": 0.270563, - "nauc_mrr_at_20_std": 0.162419, - "nauc_mrr_at_20_diff1": 0.217493, - "nauc_mrr_at_100_max": 0.270795, - "nauc_mrr_at_100_std": 0.164246, - "nauc_mrr_at_100_diff1": 0.218387, - "nauc_mrr_at_1000_max": 0.270832, - "nauc_mrr_at_1000_std": 0.164069, - "nauc_mrr_at_1000_diff1": 0.218429, - "main_score": 0.13679, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 46.585543632507324, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/SICK-R.json b/beir/SICK-R.json deleted file mode 100644 index 6b14ef521d1d758d49b92b644cfe26e822691b46..0000000000000000000000000000000000000000 --- a/beir/SICK-R.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "20a6d6f312dd54037fe07a32d58e5e168867909d", - "task_name": "SICK-R", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "pearson": 0.782689, - "spearman": 0.709342, - "cosine_pearson": 0.782689, - "cosine_spearman": 0.709342, - "manhattan_pearson": 0.747536, - "manhattan_spearman": 0.708905, - "euclidean_pearson": 0.747187, - "euclidean_spearman": 0.70888, - "main_score": 0.709342, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 8.33829402923584, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/STS12.json b/beir/STS12.json deleted file mode 100644 index cea56abd39aaed17352987088969da791b9c73cc..0000000000000000000000000000000000000000 --- a/beir/STS12.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", - "task_name": "STS12", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "pearson": 0.733449, - "spearman": 0.672036, - "cosine_pearson": 0.733449, - "cosine_spearman": 0.672036, - "manhattan_pearson": 0.685626, - "manhattan_spearman": 0.668326, - "euclidean_pearson": 0.684606, - "euclidean_spearman": 0.668238, - "main_score": 0.672036, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 2.7828357219696045, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/STS13.json b/beir/STS13.json deleted file mode 100644 index e42c26bb58eb0de9bf0694cb31b1371460fde040..0000000000000000000000000000000000000000 --- a/beir/STS13.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca", - "task_name": "STS13", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "pearson": 0.773137, - "spearman": 0.774032, - "cosine_pearson": 0.773137, - "cosine_spearman": 0.774032, - "manhattan_pearson": 0.762551, - "manhattan_spearman": 0.766366, - "euclidean_pearson": 0.761261, - "euclidean_spearman": 0.765129, - "main_score": 0.774032, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 1.3985404968261719, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/STS14.json b/beir/STS14.json deleted file mode 100644 index 356c236d3b1b8229f87f9e0c26d9939be100ab89..0000000000000000000000000000000000000000 --- a/beir/STS14.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", - "task_name": "STS14", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "pearson": 0.771255, - "spearman": 0.723476, - "cosine_pearson": 0.771255, - "cosine_spearman": 0.723476, - "manhattan_pearson": 0.741639, - "manhattan_spearman": 0.721268, - "euclidean_pearson": 0.74118, - "euclidean_spearman": 0.721061, - "main_score": 0.723476, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 3.2774949073791504, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/STS15.json b/beir/STS15.json deleted file mode 100644 index 17dc37d112b4608be03528a3d757be358e78dc4c..0000000000000000000000000000000000000000 --- a/beir/STS15.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3", - "task_name": "STS15", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "pearson": 0.81992, - "spearman": 0.824958, - "cosine_pearson": 0.81992, - "cosine_spearman": 0.824958, - "manhattan_pearson": 0.813437, - "manhattan_spearman": 0.819727, - "euclidean_pearson": 0.813543, - "euclidean_spearman": 0.819557, - "main_score": 0.824958, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 2.9901087284088135, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/STS16.json b/beir/STS16.json deleted file mode 100644 index 63e490907e97f95ceb7c88d6b7630b3e47842f5c..0000000000000000000000000000000000000000 --- a/beir/STS16.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "4d8694f8f0e0100860b497b999b3dbed754a0513", - "task_name": "STS16", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "pearson": 0.78755, - "spearman": 0.79408, - "cosine_pearson": 0.78755, - "cosine_spearman": 0.79408, - "manhattan_pearson": 0.800336, - "manhattan_spearman": 0.805912, - "euclidean_pearson": 0.800315, - "euclidean_spearman": 0.805413, - "main_score": 0.79408, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 1.2775132656097412, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/STS17.json b/beir/STS17.json deleted file mode 100644 index 82ff0cb0af8949320896fd4a1ea3cdaf1cd58f93..0000000000000000000000000000000000000000 --- a/beir/STS17.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "dataset_revision": "faeb762787bd10488a50c8b5be4a3b82e411949c", - "task_name": "STS17", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "pearson": 0.412807, - "spearman": 0.399908, - "cosine_pearson": 0.412807, - "cosine_spearman": 0.399908, - "manhattan_pearson": 0.41515, - "manhattan_spearman": 0.396907, - "euclidean_pearson": 0.415204, - "euclidean_spearman": 0.396877, - "main_score": 0.399908, - "hf_subset": "fr-en", - "languages": [ - "fra-Latn", - "eng-Latn" - ] - }, - { - "pearson": 0.843554, - "spearman": 0.847607, - "cosine_pearson": 0.843554, - "cosine_spearman": 0.847607, - "manhattan_pearson": 0.841083, - "manhattan_spearman": 0.843598, - "euclidean_pearson": 0.841249, - "euclidean_spearman": 0.843307, - "main_score": 0.847607, - "hf_subset": "en-en", - "languages": [ - "eng-Latn" - ] - }, - { - "pearson": 0.360455, - "spearman": 0.369017, - "cosine_pearson": 0.360455, - "cosine_spearman": 0.369017, - "manhattan_pearson": 0.331281, - "manhattan_spearman": 0.339834, - "euclidean_pearson": 0.334346, - "euclidean_spearman": 0.346078, - "main_score": 0.369017, - "hf_subset": "es-en", - "languages": [ - "spa-Latn", - "eng-Latn" - ] - }, - { - "pearson": 0.383329, - "spearman": 0.368844, - "cosine_pearson": 0.38333, - "cosine_spearman": 0.368844, - "manhattan_pearson": 0.347257, - "manhattan_spearman": 0.340722, - "euclidean_pearson": 0.350828, - "euclidean_spearman": 0.341548, - "main_score": 0.368844, - "hf_subset": "en-de", - "languages": [ - "eng-Latn", - "deu-Latn" - ] - }, - { - "pearson": 0.085032, - "spearman": 0.056236, - "cosine_pearson": 0.085032, - "cosine_spearman": 0.056236, - "manhattan_pearson": 0.117706, - "manhattan_spearman": 0.085126, - "euclidean_pearson": 0.116449, - "euclidean_spearman": 0.077363, - "main_score": 0.056236, - "hf_subset": "en-tr", - "languages": [ - "eng-Latn", - "tur-Latn" - ] - }, - { - "pearson": 0.307729, - "spearman": 0.278259, - "cosine_pearson": 0.307729, - "cosine_spearman": 0.278259, - "manhattan_pearson": 0.341556, - "manhattan_spearman": 0.328526, - "euclidean_pearson": 0.338333, - "euclidean_spearman": 0.328262, - "main_score": 0.278259, - "hf_subset": "it-en", - "languages": [ - "ita-Latn", - "eng-Latn" - ] - }, - { - "pearson": 0.374576, - "spearman": 0.33533, - "cosine_pearson": 0.374575, - "cosine_spearman": 0.335406, - "manhattan_pearson": 0.387747, - "manhattan_spearman": 0.368791, - "euclidean_pearson": 0.394756, - "euclidean_spearman": 0.383307, - "main_score": 0.335406, - "hf_subset": "nl-en", - "languages": [ - "nld-Latn", - "eng-Latn" - ] - }, - { - "pearson": 0.128259, - "spearman": 0.132374, - "cosine_pearson": 0.128259, - "cosine_spearman": 0.132374, - "manhattan_pearson": 0.160609, - "manhattan_spearman": 0.148943, - "euclidean_pearson": 0.141566, - "euclidean_spearman": 0.126913, - "main_score": 0.132374, - "hf_subset": "en-ar", - "languages": [ - "eng-Latn", - "ara-Arab" - ] - } - ] - }, - "evaluation_time": 2.167579174041748, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/STS22.json b/beir/STS22.json deleted file mode 100644 index 2a2a04f9de05f7ce75015338fcd3dafba6b7e659..0000000000000000000000000000000000000000 --- a/beir/STS22.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", - "task_name": "STS22", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "pearson": 0.661988, - "spearman": 0.666842, - "cosine_pearson": 0.661988, - "cosine_spearman": 0.666842, - "manhattan_pearson": 0.667457, - "manhattan_spearman": 0.66002, - "euclidean_pearson": 0.669588, - "euclidean_spearman": 0.661658, - "main_score": 0.666842, - "hf_subset": "en", - "languages": [ - "eng-Latn" - ] - }, - { - "pearson": 0.55526, - "spearman": 0.587906, - "cosine_pearson": 0.55526, - "cosine_spearman": 0.587906, - "manhattan_pearson": 0.579185, - "manhattan_spearman": 0.604479, - "euclidean_pearson": 0.580086, - "euclidean_spearman": 0.605008, - "main_score": 0.587906, - "hf_subset": "es-en", - "languages": [ - "spa-Latn", - "eng-Latn" - ] - }, - { - "pearson": 0.363507, - "spearman": 0.399716, - "cosine_pearson": 0.363506, - "cosine_spearman": 0.399716, - "manhattan_pearson": 0.253105, - "manhattan_spearman": 0.281907, - "euclidean_pearson": 0.278303, - "euclidean_spearman": 0.298408, - "main_score": 0.399716, - "hf_subset": "zh-en", - "languages": [ - "cmn-Hans", - "eng-Latn" - ] - }, - { - "pearson": 0.477598, - "spearman": 0.490005, - "cosine_pearson": 0.477598, - "cosine_spearman": 0.490005, - "manhattan_pearson": 0.462315, - "manhattan_spearman": 0.499363, - "euclidean_pearson": 0.468186, - "euclidean_spearman": 0.507778, - "main_score": 0.490005, - "hf_subset": "de-en", - "languages": [ - "deu-Latn", - "eng-Latn" - ] - }, - { - "pearson": 0.492874, - "spearman": 0.435968, - "cosine_pearson": 0.492874, - "cosine_spearman": 0.435968, - "manhattan_pearson": 0.560733, - "manhattan_spearman": 0.510045, - "euclidean_pearson": 0.563564, - "euclidean_spearman": 0.499678, - "main_score": 0.435968, - "hf_subset": "pl-en", - "languages": [ - "pol-Latn", - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 11.842099905014038, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/STSBenchmark.json b/beir/STSBenchmark.json deleted file mode 100644 index 89b7cce0a9d9ad0a1b1191945348e862ed544022..0000000000000000000000000000000000000000 --- a/beir/STSBenchmark.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831", - "task_name": "STSBenchmark", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "pearson": 0.786387, - "spearman": 0.774847, - "cosine_pearson": 0.786387, - "cosine_spearman": 0.774847, - "manhattan_pearson": 0.775199, - "manhattan_spearman": 0.771479, - "euclidean_pearson": 0.775843, - "euclidean_spearman": 0.771979, - "main_score": 0.774847, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 1.3312280178070068, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/SciDocsRR.json b/beir/SciDocsRR.json deleted file mode 100644 index 6910cf649ffd2fb87ae333c98083faaba739fe25..0000000000000000000000000000000000000000 --- a/beir/SciDocsRR.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab", - "task_name": "SciDocsRR", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "map": 0.724633, - "mrr": 0.907647, - "nAUC_map_max": 0.51911, - "nAUC_map_std": 0.610636, - "nAUC_map_diff1": 0.108873, - "nAUC_mrr_max": 0.740887, - "nAUC_mrr_std": 0.663654, - "nAUC_mrr_diff1": 0.481591, - "main_score": 0.724633, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 38.36741638183594, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/SciFact.json b/beir/SciFact.json deleted file mode 100644 index 10f5145c972b57415fcc765ae2108fa54d846393..0000000000000000000000000000000000000000 --- a/beir/SciFact.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "0228b52cf27578f30900b9e5271d331663a030d7", - "task_name": "SciFact", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.49, - "ndcg_at_3": 0.55969, - "ndcg_at_5": 0.58111, - "ndcg_at_10": 0.60427, - "ndcg_at_20": 0.61527, - "ndcg_at_100": 0.63535, - "ndcg_at_1000": 0.64773, - "map_at_1": 0.46344, - "map_at_3": 0.53188, - "map_at_5": 0.54567, - "map_at_10": 0.55704, - "map_at_20": 0.56055, - "map_at_100": 0.5635, - "map_at_1000": 0.56397, - "recall_at_1": 0.46344, - "recall_at_3": 0.60911, - "recall_at_5": 0.66294, - "recall_at_10": 0.72872, - "recall_at_20": 0.77033, - "recall_at_100": 0.87656, - "recall_at_1000": 0.97333, - "precision_at_1": 0.49, - "precision_at_3": 0.22111, - "precision_at_5": 0.146, - "precision_at_10": 0.082, - "precision_at_20": 0.04383, - "precision_at_100": 0.00997, - "precision_at_1000": 0.0011, - "mrr_at_1": 0.49, - "mrr_at_3": 0.555556, - "mrr_at_5": 0.568056, - "mrr_at_10": 0.575951, - "mrr_at_20": 0.578088, - "mrr_at_100": 0.580653, - "mrr_at_1000": 0.581053, - "nauc_ndcg_at_1_max": 0.494156, - "nauc_ndcg_at_1_std": 0.179823, - "nauc_ndcg_at_1_diff1": 0.701234, - "nauc_ndcg_at_3_max": 0.543891, - "nauc_ndcg_at_3_std": 0.143104, - "nauc_ndcg_at_3_diff1": 0.673382, - "nauc_ndcg_at_5_max": 0.529631, - "nauc_ndcg_at_5_std": 0.117775, - "nauc_ndcg_at_5_diff1": 0.662906, - "nauc_ndcg_at_10_max": 0.549531, - "nauc_ndcg_at_10_std": 0.133055, - "nauc_ndcg_at_10_diff1": 0.664653, - "nauc_ndcg_at_20_max": 0.547342, - "nauc_ndcg_at_20_std": 0.145716, - "nauc_ndcg_at_20_diff1": 0.663893, - "nauc_ndcg_at_100_max": 0.542887, - "nauc_ndcg_at_100_std": 0.163373, - "nauc_ndcg_at_100_diff1": 0.662149, - "nauc_ndcg_at_1000_max": 0.544544, - "nauc_ndcg_at_1000_std": 0.159653, - "nauc_ndcg_at_1000_diff1": 0.667584, - "nauc_map_at_1_max": 0.484528, - "nauc_map_at_1_std": 0.112364, - "nauc_map_at_1_diff1": 0.713798, - "nauc_map_at_3_max": 0.52629, - "nauc_map_at_3_std": 0.127539, - "nauc_map_at_3_diff1": 0.684559, - "nauc_map_at_5_max": 0.521172, - "nauc_map_at_5_std": 0.11887, - "nauc_map_at_5_diff1": 0.678594, - "nauc_map_at_10_max": 0.5326, - "nauc_map_at_10_std": 0.128623, - "nauc_map_at_10_diff1": 0.679285, - "nauc_map_at_20_max": 0.531401, - "nauc_map_at_20_std": 0.133123, - "nauc_map_at_20_diff1": 0.678706, - "nauc_map_at_100_max": 0.530781, - "nauc_map_at_100_std": 0.135701, - "nauc_map_at_100_diff1": 0.678354, - "nauc_map_at_1000_max": 0.530964, - "nauc_map_at_1000_std": 0.135718, - "nauc_map_at_1000_diff1": 0.678608, - "nauc_recall_at_1_max": 0.484528, - "nauc_recall_at_1_std": 0.112364, - "nauc_recall_at_1_diff1": 0.713798, - "nauc_recall_at_3_max": 0.552991, - "nauc_recall_at_3_std": 0.104741, - "nauc_recall_at_3_diff1": 0.639389, - "nauc_recall_at_5_max": 0.523593, - "nauc_recall_at_5_std": 0.052456, - "nauc_recall_at_5_diff1": 0.599453, - "nauc_recall_at_10_max": 0.580827, - "nauc_recall_at_10_std": 0.080142, - "nauc_recall_at_10_diff1": 0.596446, - "nauc_recall_at_20_max": 0.589062, - "nauc_recall_at_20_std": 0.139035, - "nauc_recall_at_20_diff1": 0.59781, - "nauc_recall_at_100_max": 0.58319, - "nauc_recall_at_100_std": 0.347022, - "nauc_recall_at_100_diff1": 0.559343, - "nauc_recall_at_1000_max": 0.791958, - "nauc_recall_at_1000_std": 0.637138, - "nauc_recall_at_1000_diff1": 0.628326, - "nauc_precision_at_1_max": 0.494156, - "nauc_precision_at_1_std": 0.179823, - "nauc_precision_at_1_diff1": 0.701234, - "nauc_precision_at_3_max": 0.572273, - "nauc_precision_at_3_std": 0.270677, - "nauc_precision_at_3_diff1": 0.528038, - "nauc_precision_at_5_max": 0.514554, - "nauc_precision_at_5_std": 0.233615, - "nauc_precision_at_5_diff1": 0.441645, - "nauc_precision_at_10_max": 0.504284, - "nauc_precision_at_10_std": 0.284772, - "nauc_precision_at_10_diff1": 0.314772, - "nauc_precision_at_20_max": 0.425568, - "nauc_precision_at_20_std": 0.326138, - "nauc_precision_at_20_diff1": 0.234987, - "nauc_precision_at_100_max": 0.337983, - "nauc_precision_at_100_std": 0.448859, - "nauc_precision_at_100_diff1": 0.079106, - "nauc_precision_at_1000_max": 0.230914, - "nauc_precision_at_1000_std": 0.460357, - "nauc_precision_at_1000_diff1": -0.125412, - "nauc_mrr_at_1_max": 0.494156, - "nauc_mrr_at_1_std": 0.179823, - "nauc_mrr_at_1_diff1": 0.701234, - "nauc_mrr_at_3_max": 0.535852, - "nauc_mrr_at_3_std": 0.169294, - "nauc_mrr_at_3_diff1": 0.679143, - "nauc_mrr_at_5_max": 0.528663, - "nauc_mrr_at_5_std": 0.160492, - "nauc_mrr_at_5_diff1": 0.671386, - "nauc_mrr_at_10_max": 0.532132, - "nauc_mrr_at_10_std": 0.165224, - "nauc_mrr_at_10_diff1": 0.671661, - "nauc_mrr_at_20_max": 0.531545, - "nauc_mrr_at_20_std": 0.166536, - "nauc_mrr_at_20_diff1": 0.671882, - "nauc_mrr_at_100_max": 0.530415, - "nauc_mrr_at_100_std": 0.167896, - "nauc_mrr_at_100_diff1": 0.671524, - "nauc_mrr_at_1000_max": 0.53043, - "nauc_mrr_at_1000_std": 0.167823, - "nauc_mrr_at_1000_diff1": 0.671834, - "main_score": 0.60427, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 12.572911739349365, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/SprintDuplicateQuestions.json b/beir/SprintDuplicateQuestions.json deleted file mode 100644 index d1adf6bb72551aa13f59d4d8ab56855f3b39b8f6..0000000000000000000000000000000000000000 --- a/beir/SprintDuplicateQuestions.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", - "task_name": "SprintDuplicateQuestions", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "similarity_accuracy": 0.99802, - "similarity_accuracy_threshold": 0.689054, - "similarity_f1": 0.898944, - "similarity_f1_threshold": 0.681838, - "similarity_precision": 0.903943, - "similarity_recall": 0.894, - "similarity_ap": 0.951114, - "cosine_accuracy": 0.99802, - "cosine_accuracy_threshold": 0.689053, - "cosine_f1": 0.898944, - "cosine_f1_threshold": 0.681838, - "cosine_precision": 0.903943, - "cosine_recall": 0.894, - "cosine_ap": 0.951114, - "manhattan_accuracy": 0.998, - "manhattan_accuracy_threshold": 565.695862, - "manhattan_f1": 0.898899, - "manhattan_f1_threshold": 565.695862, - "manhattan_precision": 0.8998, - "manhattan_recall": 0.898, - "manhattan_ap": 0.949934, - "euclidean_accuracy": 0.99799, - "euclidean_accuracy_threshold": 25.397804, - "euclidean_f1": 0.898348, - "euclidean_f1_threshold": 25.688805, - "euclidean_precision": 0.899699, - "euclidean_recall": 0.897, - "euclidean_ap": 0.9506, - "dot_accuracy": 0.997723, - "dot_accuracy_threshold": 723.987732, - "dot_f1": 0.880829, - "dot_f1_threshold": 723.987732, - "dot_precision": 0.913978, - "dot_recall": 0.85, - "dot_ap": 0.937404, - "max_accuracy": 0.99802, - "max_f1": 0.898944, - "max_precision": 0.913978, - "max_recall": 0.898, - "max_ap": 0.951114, - "main_score": 0.951114, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 9.09257197380066, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/StackExchangeClustering.json b/beir/StackExchangeClustering.json deleted file mode 100644 index 66bf1d8814c375618129e336ead096090b1fc4a7..0000000000000000000000000000000000000000 --- a/beir/StackExchangeClustering.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", - "task_name": "StackExchangeClustering", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "v_measure": 0.56641, - "v_measure_std": 0.046061, - "v_measures": [ - 0.618282, - 0.600575, - 0.475684, - 0.602598, - 0.543083, - 0.507604, - 0.522691, - 0.621222, - 0.5887, - 0.553584, - 0.662264, - 0.637221, - 0.652978, - 0.565985, - 0.523508, - 0.560681, - 0.529304, - 0.577785, - 0.542509, - 0.533142, - 0.550816, - 0.520423, - 0.540373, - 0.57613, - 0.553114 - ], - "main_score": 0.56641, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 157.4830355644226, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/StackExchangeClusteringP2P.json b/beir/StackExchangeClusteringP2P.json deleted file mode 100644 index 797e16ad7d0ab9fc4445e8ffff2387e77538fc9c..0000000000000000000000000000000000000000 --- a/beir/StackExchangeClusteringP2P.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", - "task_name": "StackExchangeClusteringP2P", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "v_measure": 0.318501, - "v_measure_std": 0.013043, - "v_measures": [ - 0.307064, - 0.308754, - 0.305133, - 0.305295, - 0.303372, - 0.338656, - 0.326189, - 0.327465, - 0.33349, - 0.329596 - ], - "main_score": 0.318501, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 164.03653264045715, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/StackOverflowDupQuestions.json b/beir/StackOverflowDupQuestions.json deleted file mode 100644 index 019946a722009be76eb01552632565799c67e442..0000000000000000000000000000000000000000 --- a/beir/StackOverflowDupQuestions.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dataset_revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69", - "task_name": "StackOverflowDupQuestions", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "map": 0.481493, - "mrr": 0.488603, - "nAUC_map_max": 0.143434, - "nAUC_map_std": 0.081154, - "nAUC_map_diff1": 0.350177, - "nAUC_mrr_max": 0.150698, - "nAUC_mrr_std": 0.089674, - "nAUC_mrr_diff1": 0.350292, - "main_score": 0.481493, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 35.406832695007324, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/SummEval.json b/beir/SummEval.json deleted file mode 100644 index 12e90b42b6ef0463abeb568040edaac1618f6e05..0000000000000000000000000000000000000000 --- a/beir/SummEval.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "dataset_revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c", - "task_name": "SummEval", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "pearson": 0.299904, - "spearman": 0.318365, - "cosine_spearman": 0.318365, - "cosine_pearson": 0.299904, - "dot_spearman": 0.297131, - "dot_pearson": 0.283169, - "main_score": 0.318365, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 2.6189992427825928, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/TRECCOVID.json b/beir/TRECCOVID.json deleted file mode 100644 index 36a27582588eab88d28a61dedb96b6a4a10c0af7..0000000000000000000000000000000000000000 --- a/beir/TRECCOVID.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "bb9466bac8153a0349341eb1b22e06409e78ef4e", - "task_name": "TRECCOVID", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.81, - "ndcg_at_3": 0.75989, - "ndcg_at_5": 0.75205, - "ndcg_at_10": 0.71532, - "ndcg_at_20": 0.68284, - "ndcg_at_100": 0.52276, - "ndcg_at_1000": 0.45097, - "map_at_1": 0.00231, - "map_at_3": 0.00583, - "map_at_5": 0.00937, - "map_at_10": 0.01718, - "map_at_20": 0.03005, - "map_at_100": 0.09005, - "map_at_1000": 0.20935, - "recall_at_1": 0.00231, - "recall_at_3": 0.00619, - "recall_at_5": 0.01024, - "recall_at_10": 0.01961, - "recall_at_20": 0.03532, - "recall_at_100": 0.12046, - "recall_at_1000": 0.41831, - "precision_at_1": 0.88, - "precision_at_3": 0.8, - "precision_at_5": 0.8, - "precision_at_10": 0.758, - "precision_at_20": 0.715, - "precision_at_100": 0.5306, - "precision_at_1000": 0.20356, - "mrr_at_1": 0.88, - "mrr_at_3": 0.926667, - "mrr_at_5": 0.931667, - "mrr_at_10": 0.931667, - "mrr_at_20": 0.931667, - "mrr_at_100": 0.931667, - "mrr_at_1000": 0.931667, - "nauc_ndcg_at_1_max": 0.235033, - "nauc_ndcg_at_1_std": 0.425107, - "nauc_ndcg_at_1_diff1": -0.166331, - "nauc_ndcg_at_3_max": 0.457534, - "nauc_ndcg_at_3_std": 0.49449, - "nauc_ndcg_at_3_diff1": -0.218177, - "nauc_ndcg_at_5_max": 0.523915, - "nauc_ndcg_at_5_std": 0.540234, - "nauc_ndcg_at_5_diff1": -0.224601, - "nauc_ndcg_at_10_max": 0.489955, - "nauc_ndcg_at_10_std": 0.579181, - "nauc_ndcg_at_10_diff1": -0.318485, - "nauc_ndcg_at_20_max": 0.536688, - "nauc_ndcg_at_20_std": 0.690301, - "nauc_ndcg_at_20_diff1": -0.354435, - "nauc_ndcg_at_100_max": 0.518196, - "nauc_ndcg_at_100_std": 0.822754, - "nauc_ndcg_at_100_diff1": -0.397011, - "nauc_ndcg_at_1000_max": 0.654034, - "nauc_ndcg_at_1000_std": 0.817756, - "nauc_ndcg_at_1000_diff1": -0.301325, - "nauc_map_at_1_max": 0.030979, - "nauc_map_at_1_std": -0.112343, - "nauc_map_at_1_diff1": 0.207618, - "nauc_map_at_3_max": 0.286885, - "nauc_map_at_3_std": 0.029605, - "nauc_map_at_3_diff1": 0.113494, - "nauc_map_at_5_max": 0.317008, - "nauc_map_at_5_std": 0.056763, - "nauc_map_at_5_diff1": 0.091052, - "nauc_map_at_10_max": 0.299906, - "nauc_map_at_10_std": 0.065652, - "nauc_map_at_10_diff1": 0.054684, - "nauc_map_at_20_max": 0.386532, - "nauc_map_at_20_std": 0.228509, - "nauc_map_at_20_diff1": -0.026004, - "nauc_map_at_100_max": 0.589551, - "nauc_map_at_100_std": 0.673813, - "nauc_map_at_100_diff1": -0.237286, - "nauc_map_at_1000_max": 0.665071, - "nauc_map_at_1000_std": 0.876026, - "nauc_map_at_1000_diff1": -0.342005, - "nauc_recall_at_1_max": 0.030979, - "nauc_recall_at_1_std": -0.112343, - "nauc_recall_at_1_diff1": 0.207618, - "nauc_recall_at_3_max": 0.225952, - "nauc_recall_at_3_std": -0.050699, - "nauc_recall_at_3_diff1": 0.121543, - "nauc_recall_at_5_max": 0.231872, - "nauc_recall_at_5_std": -0.041483, - "nauc_recall_at_5_diff1": 0.119388, - "nauc_recall_at_10_max": 0.185173, - "nauc_recall_at_10_std": -0.063048, - "nauc_recall_at_10_diff1": 0.087213, - "nauc_recall_at_20_max": 0.271838, - "nauc_recall_at_20_std": 0.096419, - "nauc_recall_at_20_diff1": 0.036646, - "nauc_recall_at_100_max": 0.511496, - "nauc_recall_at_100_std": 0.527999, - "nauc_recall_at_100_diff1": -0.177583, - "nauc_recall_at_1000_max": 0.646146, - "nauc_recall_at_1000_std": 0.733405, - "nauc_recall_at_1000_diff1": -0.223217, - "nauc_precision_at_1_max": 0.4895, - "nauc_precision_at_1_std": 0.758988, - "nauc_precision_at_1_diff1": -0.034839, - "nauc_precision_at_3_max": 0.709443, - "nauc_precision_at_3_std": 0.6753, - "nauc_precision_at_3_diff1": -0.198922, - "nauc_precision_at_5_max": 0.730315, - "nauc_precision_at_5_std": 0.708924, - "nauc_precision_at_5_diff1": -0.243446, - "nauc_precision_at_10_max": 0.55359, - "nauc_precision_at_10_std": 0.627347, - "nauc_precision_at_10_diff1": -0.33091, - "nauc_precision_at_20_max": 0.591229, - "nauc_precision_at_20_std": 0.760406, - "nauc_precision_at_20_diff1": -0.35061, - "nauc_precision_at_100_max": 0.531281, - "nauc_precision_at_100_std": 0.858325, - "nauc_precision_at_100_diff1": -0.406393, - "nauc_precision_at_1000_max": 0.413845, - "nauc_precision_at_1000_std": 0.676367, - "nauc_precision_at_1000_diff1": -0.332047, - "nauc_mrr_at_1_max": 0.4895, - "nauc_mrr_at_1_std": 0.758988, - "nauc_mrr_at_1_diff1": -0.034839, - "nauc_mrr_at_3_max": 0.545437, - "nauc_mrr_at_3_std": 0.71448, - "nauc_mrr_at_3_diff1": -0.100942, - "nauc_mrr_at_5_max": 0.532249, - "nauc_mrr_at_5_std": 0.741067, - "nauc_mrr_at_5_diff1": -0.066437, - "nauc_mrr_at_10_max": 0.532249, - "nauc_mrr_at_10_std": 0.741067, - "nauc_mrr_at_10_diff1": -0.066437, - "nauc_mrr_at_20_max": 0.532249, - "nauc_mrr_at_20_std": 0.741067, - "nauc_mrr_at_20_diff1": -0.066437, - "nauc_mrr_at_100_max": 0.532249, - "nauc_mrr_at_100_std": 0.741067, - "nauc_mrr_at_100_diff1": -0.066437, - "nauc_mrr_at_1000_max": 0.532249, - "nauc_mrr_at_1000_std": 0.741067, - "nauc_mrr_at_1000_diff1": -0.066437, - "main_score": 0.71532, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 301.5285987854004, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/Touche2020.json b/beir/Touche2020.json deleted file mode 100644 index 3d24d410649b09898fbf58184cd09aac69cc686c..0000000000000000000000000000000000000000 --- a/beir/Touche2020.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "dataset_revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f", - "task_name": "Touche2020", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "ndcg_at_1": 0.2449, - "ndcg_at_3": 0.24729, - "ndcg_at_5": 0.23791, - "ndcg_at_10": 0.22788, - "ndcg_at_20": 0.25138, - "ndcg_at_100": 0.35506, - "ndcg_at_1000": 0.47535, - "map_at_1": 0.02094, - "map_at_3": 0.05037, - "map_at_5": 0.06667, - "map_at_10": 0.09567, - "map_at_20": 0.12238, - "map_at_100": 0.15456, - "map_at_1000": 0.17134, - "recall_at_1": 0.02094, - "recall_at_3": 0.06356, - "recall_at_5": 0.09517, - "recall_at_10": 0.15589, - "recall_at_20": 0.24322, - "recall_at_100": 0.46668, - "recall_at_1000": 0.83852, - "precision_at_1": 0.26531, - "precision_at_3": 0.26531, - "precision_at_5": 0.2449, - "precision_at_10": 0.20408, - "precision_at_20": 0.17143, - "precision_at_100": 0.0749, - "precision_at_1000": 0.01551, - "mrr_at_1": 0.265306, - "mrr_at_3": 0.380952, - "mrr_at_5": 0.409524, - "mrr_at_10": 0.42151, - "mrr_at_20": 0.431482, - "mrr_at_100": 0.433322, - "mrr_at_1000": 0.433603, - "nauc_ndcg_at_1_max": -0.21184, - "nauc_ndcg_at_1_std": 0.045078, - "nauc_ndcg_at_1_diff1": -0.127358, - "nauc_ndcg_at_3_max": -0.194252, - "nauc_ndcg_at_3_std": -0.062552, - "nauc_ndcg_at_3_diff1": 0.013456, - "nauc_ndcg_at_5_max": -0.21515, - "nauc_ndcg_at_5_std": -0.078813, - "nauc_ndcg_at_5_diff1": -0.012183, - "nauc_ndcg_at_10_max": -0.1812, - "nauc_ndcg_at_10_std": -0.055853, - "nauc_ndcg_at_10_diff1": 0.012866, - "nauc_ndcg_at_20_max": -0.279821, - "nauc_ndcg_at_20_std": -0.058553, - "nauc_ndcg_at_20_diff1": 0.031812, - "nauc_ndcg_at_100_max": -0.230923, - "nauc_ndcg_at_100_std": 0.126236, - "nauc_ndcg_at_100_diff1": 0.08099, - "nauc_ndcg_at_1000_max": -0.201611, - "nauc_ndcg_at_1000_std": 0.222888, - "nauc_ndcg_at_1000_diff1": 0.038947, - "nauc_map_at_1_max": -0.151336, - "nauc_map_at_1_std": -0.05686, - "nauc_map_at_1_diff1": -0.110982, - "nauc_map_at_3_max": -0.17957, - "nauc_map_at_3_std": -0.143613, - "nauc_map_at_3_diff1": -0.040538, - "nauc_map_at_5_max": -0.197211, - "nauc_map_at_5_std": -0.154328, - "nauc_map_at_5_diff1": -0.05491, - "nauc_map_at_10_max": -0.146394, - "nauc_map_at_10_std": -0.151459, - "nauc_map_at_10_diff1": -0.020209, - "nauc_map_at_20_max": -0.194593, - "nauc_map_at_20_std": -0.150695, - "nauc_map_at_20_diff1": 0.027577, - "nauc_map_at_100_max": -0.172374, - "nauc_map_at_100_std": -0.062214, - "nauc_map_at_100_diff1": 0.077665, - "nauc_map_at_1000_max": -0.164723, - "nauc_map_at_1000_std": -0.024473, - "nauc_map_at_1000_diff1": 0.071406, - "nauc_recall_at_1_max": -0.151336, - "nauc_recall_at_1_std": -0.05686, - "nauc_recall_at_1_diff1": -0.110982, - "nauc_recall_at_3_max": -0.216947, - "nauc_recall_at_3_std": -0.208214, - "nauc_recall_at_3_diff1": -0.021295, - "nauc_recall_at_5_max": -0.24176, - "nauc_recall_at_5_std": -0.213149, - "nauc_recall_at_5_diff1": -0.06896, - "nauc_recall_at_10_max": -0.179446, - "nauc_recall_at_10_std": -0.142058, - "nauc_recall_at_10_diff1": -0.022227, - "nauc_recall_at_20_max": -0.301632, - "nauc_recall_at_20_std": -0.119398, - "nauc_recall_at_20_diff1": 0.042804, - "nauc_recall_at_100_max": -0.21802, - "nauc_recall_at_100_std": 0.203004, - "nauc_recall_at_100_diff1": 0.081152, - "nauc_recall_at_1000_max": -0.154843, - "nauc_recall_at_1000_std": 0.610456, - "nauc_recall_at_1000_diff1": -0.223118, - "nauc_precision_at_1_max": -0.163555, - "nauc_precision_at_1_std": 0.014887, - "nauc_precision_at_1_diff1": -0.072894, - "nauc_precision_at_3_max": -0.185059, - "nauc_precision_at_3_std": -0.110688, - "nauc_precision_at_3_diff1": 0.082707, - "nauc_precision_at_5_max": -0.20949, - "nauc_precision_at_5_std": -0.135561, - "nauc_precision_at_5_diff1": 0.038719, - "nauc_precision_at_10_max": -0.115299, - "nauc_precision_at_10_std": -0.016832, - "nauc_precision_at_10_diff1": 0.116033, - "nauc_precision_at_20_max": -0.271497, - "nauc_precision_at_20_std": 0.057508, - "nauc_precision_at_20_diff1": 0.200541, - "nauc_precision_at_100_max": -0.02628, - "nauc_precision_at_100_std": 0.470634, - "nauc_precision_at_100_diff1": 0.298468, - "nauc_precision_at_1000_max": 0.308665, - "nauc_precision_at_1000_std": 0.338682, - "nauc_precision_at_1000_diff1": 0.113792, - "nauc_mrr_at_1_max": -0.163555, - "nauc_mrr_at_1_std": 0.014887, - "nauc_mrr_at_1_diff1": -0.072894, - "nauc_mrr_at_3_max": -0.201747, - "nauc_mrr_at_3_std": -0.098292, - "nauc_mrr_at_3_diff1": 0.023817, - "nauc_mrr_at_5_max": -0.202904, - "nauc_mrr_at_5_std": -0.081643, - "nauc_mrr_at_5_diff1": -0.001708, - "nauc_mrr_at_10_max": -0.19431, - "nauc_mrr_at_10_std": -0.046234, - "nauc_mrr_at_10_diff1": -0.009555, - "nauc_mrr_at_20_max": -0.197291, - "nauc_mrr_at_20_std": -0.044182, - "nauc_mrr_at_20_diff1": -0.007163, - "nauc_mrr_at_100_max": -0.19662, - "nauc_mrr_at_100_std": -0.044295, - "nauc_mrr_at_100_diff1": -0.009366, - "nauc_mrr_at_1000_max": -0.196591, - "nauc_mrr_at_1000_std": -0.045101, - "nauc_mrr_at_1000_diff1": -0.009622, - "main_score": 0.22788, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 1095.016489982605, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/ToxicConversationsClassification.json b/beir/ToxicConversationsClassification.json deleted file mode 100644 index 9331703d269813de1fce90c1bad0a0a3ba5699a5..0000000000000000000000000000000000000000 --- a/beir/ToxicConversationsClassification.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "dataset_revision": "edfaf9da55d3dd50d43143d90c1ac476895ae6de", - "task_name": "ToxicConversationsClassification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.644141, - "f1": 0.485129, - "f1_weighted": 0.722769, - "ap": 0.104382, - "ap_weighted": 0.104382, - "scores_per_experiment": [ - { - "accuracy": 0.506836, - "f1": 0.416453, - "f1_weighted": 0.609778, - "ap": 0.09937, - "ap_weighted": 0.09937 - }, - { - "accuracy": 0.742676, - "f1": 0.543386, - "f1_weighted": 0.797322, - "ap": 0.119804, - "ap_weighted": 0.119804 - }, - { - "accuracy": 0.687012, - "f1": 0.509065, - "f1_weighted": 0.757873, - "ap": 0.108171, - "ap_weighted": 0.108171 - }, - { - "accuracy": 0.687988, - "f1": 0.513283, - "f1_weighted": 0.758753, - "ap": 0.112236, - "ap_weighted": 0.112236 - }, - { - "accuracy": 0.605957, - "f1": 0.459904, - "f1_weighted": 0.696332, - "ap": 0.094358, - "ap_weighted": 0.094358 - }, - { - "accuracy": 0.541016, - "f1": 0.427484, - "f1_weighted": 0.642099, - "ap": 0.092116, - "ap_weighted": 0.092116 - }, - { - "accuracy": 0.708496, - "f1": 0.505065, - "f1_weighted": 0.772175, - "ap": 0.0967, - "ap_weighted": 0.0967 - }, - { - "accuracy": 0.69043, - "f1": 0.513077, - "f1_weighted": 0.760452, - "ap": 0.110868, - "ap_weighted": 0.110868 - }, - { - "accuracy": 0.569336, - "f1": 0.455629, - "f1_weighted": 0.665064, - "ap": 0.108882, - "ap_weighted": 0.108882 - }, - { - "accuracy": 0.70166, - "f1": 0.507948, - "f1_weighted": 0.767839, - "ap": 0.101317, - "ap_weighted": 0.101317 - } - ], - "main_score": 0.644141, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 10.779565811157227, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/TweetSentimentExtractionClassification.json b/beir/TweetSentimentExtractionClassification.json deleted file mode 100644 index 81e9d11278404be5704f69ee9ddb0f46a3236d11..0000000000000000000000000000000000000000 --- a/beir/TweetSentimentExtractionClassification.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "dataset_revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a", - "task_name": "TweetSentimentExtractionClassification", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "accuracy": 0.526316, - "f1": 0.529212, - "f1_weighted": 0.523632, - "scores_per_experiment": [ - { - "accuracy": 0.532258, - "f1": 0.534863, - "f1_weighted": 0.527884 - }, - { - "accuracy": 0.533956, - "f1": 0.537524, - "f1_weighted": 0.529935 - }, - { - "accuracy": 0.522071, - "f1": 0.525822, - "f1_weighted": 0.521472 - }, - { - "accuracy": 0.523769, - "f1": 0.525527, - "f1_weighted": 0.524488 - }, - { - "accuracy": 0.54867, - "f1": 0.550189, - "f1_weighted": 0.54339 - }, - { - "accuracy": 0.55348, - "f1": 0.556835, - "f1_weighted": 0.549865 - }, - { - "accuracy": 0.517261, - "f1": 0.520434, - "f1_weighted": 0.515424 - }, - { - "accuracy": 0.526316, - "f1": 0.529428, - "f1_weighted": 0.521649 - }, - { - "accuracy": 0.50283, - "f1": 0.506101, - "f1_weighted": 0.501108 - }, - { - "accuracy": 0.502547, - "f1": 0.505401, - "f1_weighted": 0.501102 - } - ], - "main_score": 0.526316, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 7.326297998428345, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/TwentyNewsgroupsClustering.json b/beir/TwentyNewsgroupsClustering.json deleted file mode 100644 index 886bbc77f53a22f23ef3ce876f483c69ec43bd2b..0000000000000000000000000000000000000000 --- a/beir/TwentyNewsgroupsClustering.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", - "task_name": "TwentyNewsgroupsClustering", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "v_measure": 0.343169, - "v_measure_std": 0.01669, - "v_measures": [ - 0.371176, - 0.329113, - 0.358309, - 0.359354, - 0.321954, - 0.33045, - 0.34941, - 0.3566, - 0.327921, - 0.327403 - ], - "main_score": 0.343169, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 26.601304531097412, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/TwitterSemEval2015.json b/beir/TwitterSemEval2015.json deleted file mode 100644 index 04474fa810177ac1cfe769129fb11e05bee04116..0000000000000000000000000000000000000000 --- a/beir/TwitterSemEval2015.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", - "task_name": "TwitterSemEval2015", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "similarity_accuracy": 0.834595, - "similarity_accuracy_threshold": 0.724461, - "similarity_f1": 0.617155, - "similarity_f1_threshold": 0.625381, - "similarity_precision": 0.560681, - "similarity_recall": 0.68628, - "similarity_ap": 0.650861, - "cosine_accuracy": 0.834595, - "cosine_accuracy_threshold": 0.724461, - "cosine_f1": 0.617155, - "cosine_f1_threshold": 0.625381, - "cosine_precision": 0.560681, - "cosine_recall": 0.68628, - "cosine_ap": 0.650861, - "manhattan_accuracy": 0.833999, - "manhattan_accuracy_threshold": 534.933594, - "manhattan_f1": 0.610456, - "manhattan_f1_threshold": 607.510315, - "manhattan_precision": 0.576589, - "manhattan_recall": 0.648549, - "manhattan_ap": 0.645486, - "euclidean_accuracy": 0.834059, - "euclidean_accuracy_threshold": 24.222414, - "euclidean_f1": 0.609938, - "euclidean_f1_threshold": 27.656849, - "euclidean_precision": 0.576291, - "euclidean_recall": 0.647757, - "euclidean_ap": 0.646071, - "dot_accuracy": 0.826012, - "dot_accuracy_threshold": 771.538452, - "dot_f1": 0.599331, - "dot_f1_threshold": 671.008545, - "dot_precision": 0.547858, - "dot_recall": 0.661478, - "dot_ap": 0.621271, - "max_accuracy": 0.834595, - "max_f1": 0.617155, - "max_precision": 0.576589, - "max_recall": 0.68628, - "max_ap": 0.650861, - "main_score": 0.650861, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 8.805907487869263, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/TwitterURLCorpus.json b/beir/TwitterURLCorpus.json deleted file mode 100644 index a2859d18c25419a49bbc24b5c7f55179d2bef2dd..0000000000000000000000000000000000000000 --- a/beir/TwitterURLCorpus.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", - "task_name": "TwitterURLCorpus", - "mteb_version": "1.25.1", - "scores": { - "test": [ - { - "similarity_accuracy": 0.881612, - "similarity_accuracy_threshold": 0.638545, - "similarity_f1": 0.763737, - "similarity_f1_threshold": 0.604226, - "similarity_precision": 0.741711, - "similarity_recall": 0.787111, - "similarity_ap": 0.835651, - "cosine_accuracy": 0.881612, - "cosine_accuracy_threshold": 0.638545, - "cosine_f1": 0.763737, - "cosine_f1_threshold": 0.604226, - "cosine_precision": 0.741711, - "cosine_recall": 0.787111, - "cosine_ap": 0.835651, - "manhattan_accuracy": 0.879652, - "manhattan_accuracy_threshold": 578.377014, - "manhattan_f1": 0.761352, - "manhattan_f1_threshold": 617.466187, - "manhattan_precision": 0.723335, - "manhattan_recall": 0.803588, - "manhattan_ap": 0.833006, - "euclidean_accuracy": 0.880002, - "euclidean_accuracy_threshold": 26.073254, - "euclidean_f1": 0.762589, - "euclidean_f1_threshold": 27.932058, - "euclidean_precision": 0.731054, - "euclidean_recall": 0.796966, - "euclidean_ap": 0.833352, - "dot_accuracy": 0.876994, - "dot_accuracy_threshold": 621.233459, - "dot_f1": 0.75509, - "dot_f1_threshold": 561.458374, - "dot_precision": 0.709651, - "dot_recall": 0.806745, - "dot_ap": 0.817367, - "max_accuracy": 0.881612, - "max_f1": 0.763737, - "max_precision": 0.741711, - "max_recall": 0.806745, - "max_ap": 0.835651, - "main_score": 0.835651, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ] - } - ] - }, - "evaluation_time": 22.606329441070557, - "kg_co2_emissions": null -} \ No newline at end of file diff --git a/beir/model_meta.json b/beir/model_meta.json deleted file mode 100644 index 94698b49c9810ffa2fda32895b06b63d8dea58c9..0000000000000000000000000000000000000000 --- a/beir/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "joe32140/ModernBERT-base-msmarco", "revision": "c681d6fd230415ac36cb6468d0d98571fef81d58", "release_date": null, "languages": [], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "loader": null} \ No newline at end of file