Hugging Face's logo

joe32140
/
ModernBERT-base-msmarco

Sentence Similarity
sentence-transformers
Safetensors
English
modernbert
feature-extraction
Generated from Trainer
dataset_size:11662655
loss:CachedMultipleNegativesRankingLoss
Eval Results
Inference Endpoints
Model card Files Community
joe32140 commited on
Commit
81d107a
•
1 Parent(s): 4d3c7ce

Upload 68 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. mteb/AmazonCounterfactualClassification.json +179 -0
  2. mteb/AmazonPolarityClassification.json +95 -0
  3. mteb/AmazonReviewsClassification.json +73 -0
  4. mteb/ArguAna.json +158 -0
  5. mteb/ArxivClusteringP2P.json +53 -0
  6. mteb/ArxivClusteringS2S.json +53 -0
  7. mteb/AskUbuntuDupQuestions.json +26 -0
  8. mteb/BIOSSES.json +26 -0
  9. mteb/Banking77Classification.json +73 -0
  10. mteb/BiorxivClusteringP2P.json +32 -0
  11. mteb/BiorxivClusteringS2S.json +32 -0
  12. mteb/CQADupstackAndroidRetrieval.json +158 -0
  13. mteb/CQADupstackEnglishRetrieval.json +158 -0
  14. mteb/CQADupstackGamingRetrieval.json +158 -0
  15. mteb/CQADupstackGisRetrieval.json +158 -0
  16. mteb/CQADupstackMathematicaRetrieval.json +158 -0
  17. mteb/CQADupstackPhysicsRetrieval.json +158 -0
  18. mteb/CQADupstackProgrammersRetrieval.json +158 -0
  19. mteb/CQADupstackStatsRetrieval.json +158 -0
  20. mteb/CQADupstackTexRetrieval.json +158 -0
  21. mteb/CQADupstackUnixRetrieval.json +158 -0
  22. mteb/CQADupstackWebmastersRetrieval.json +158 -0
  23. mteb/CQADupstackWordpressRetrieval.json +158 -0
  24. mteb/ClimateFEVER.json +158 -0
  25. mteb/DBPedia.json +158 -0
  26. mteb/EmotionClassification.json +73 -0
  27. mteb/FEVER.json +158 -0
  28. mteb/FiQA2018.json +158 -0
  29. mteb/HotpotQA.json +158 -0
  30. mteb/ImdbClassification.json +95 -0
  31. mteb/MSMARCO.json +158 -0
  32. mteb/MTOPDomainClassification.json +73 -0
  33. mteb/MTOPIntentClassification.json +73 -0
  34. mteb/MassiveIntentClassification.json +73 -0
  35. mteb/MassiveScenarioClassification.json +73 -0
  36. mteb/MedrxivClusteringP2P.json +32 -0
  37. mteb/MedrxivClusteringS2S.json +32 -0
  38. mteb/MindSmallReranking.json +26 -0
  39. mteb/NFCorpus.json +158 -0
  40. mteb/NQ.json +158 -0
  41. mteb/QuoraRetrieval.json +158 -0
  42. mteb/RedditClustering.json +47 -0
  43. mteb/RedditClusteringP2P.json +32 -0
  44. mteb/SCIDOCS.json +158 -0
  45. mteb/SICK-R.json +26 -0
  46. mteb/STS12.json +26 -0
  47. mteb/STS13.json +26 -0
  48. mteb/STS14.json +26 -0
  49. mteb/STS15.json +26 -0
  50. mteb/STS16.json +26 -0
mteb/AmazonCounterfactualClassification.json ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205",
3
+ "task_name": "AmazonCounterfactualClassification",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "accuracy": 0.65997,
9
+ "f1": 0.535951,
10
+ "f1_weighted": 0.725304,
11
+ "ap": 0.163093,
12
+ "ap_weighted": 0.163093,
13
+ "scores_per_experiment": [
14
+ {
15
+ "accuracy": 0.644678,
16
+ "f1": 0.531347,
17
+ "f1_weighted": 0.713782,
18
+ "ap": 0.166602,
19
+ "ap_weighted": 0.166602
20
+ },
21
+ {
22
+ "accuracy": 0.608696,
23
+ "f1": 0.518256,
24
+ "f1_weighted": 0.683489,
25
+ "ap": 0.176106,
26
+ "ap_weighted": 0.176106
27
+ },
28
+ {
29
+ "accuracy": 0.643928,
30
+ "f1": 0.529901,
31
+ "f1_weighted": 0.713177,
32
+ "ap": 0.164896,
33
+ "ap_weighted": 0.164896
34
+ },
35
+ {
36
+ "accuracy": 0.641679,
37
+ "f1": 0.51983,
38
+ "f1_weighted": 0.711307,
39
+ "ap": 0.151661,
40
+ "ap_weighted": 0.151661
41
+ },
42
+ {
43
+ "accuracy": 0.668666,
44
+ "f1": 0.544738,
45
+ "f1_weighted": 0.732766,
46
+ "ap": 0.169199,
47
+ "ap_weighted": 0.169199
48
+ },
49
+ {
50
+ "accuracy": 0.707646,
51
+ "f1": 0.550963,
52
+ "f1_weighted": 0.760934,
53
+ "ap": 0.15327,
54
+ "ap_weighted": 0.15327
55
+ },
56
+ {
57
+ "accuracy": 0.654423,
58
+ "f1": 0.525677,
59
+ "f1_weighted": 0.721296,
60
+ "ap": 0.151452,
61
+ "ap_weighted": 0.151452
62
+ },
63
+ {
64
+ "accuracy": 0.664918,
65
+ "f1": 0.539092,
66
+ "f1_weighted": 0.729726,
67
+ "ap": 0.163361,
68
+ "ap_weighted": 0.163361
69
+ },
70
+ {
71
+ "accuracy": 0.696402,
72
+ "f1": 0.563025,
73
+ "f1_weighted": 0.754132,
74
+ "ap": 0.176998,
75
+ "ap_weighted": 0.176998
76
+ },
77
+ {
78
+ "accuracy": 0.668666,
79
+ "f1": 0.536679,
80
+ "f1_weighted": 0.732435,
81
+ "ap": 0.15738,
82
+ "ap_weighted": 0.15738
83
+ }
84
+ ],
85
+ "main_score": 0.65997,
86
+ "hf_subset": "en-ext",
87
+ "languages": [
88
+ "eng-Latn"
89
+ ]
90
+ },
91
+ {
92
+ "accuracy": 0.64194,
93
+ "f1": 0.585969,
94
+ "f1_weighted": 0.677482,
95
+ "ap": 0.280748,
96
+ "ap_weighted": 0.280748,
97
+ "scores_per_experiment": [
98
+ {
99
+ "accuracy": 0.659701,
100
+ "f1": 0.606678,
101
+ "f1_weighted": 0.694619,
102
+ "ap": 0.299469,
103
+ "ap_weighted": 0.299469
104
+ },
105
+ {
106
+ "accuracy": 0.61194,
107
+ "f1": 0.564082,
108
+ "f1_weighted": 0.652038,
109
+ "ap": 0.268001,
110
+ "ap_weighted": 0.268001
111
+ },
112
+ {
113
+ "accuracy": 0.592537,
114
+ "f1": 0.553977,
115
+ "f1_weighted": 0.633838,
116
+ "ap": 0.269473,
117
+ "ap_weighted": 0.269473
118
+ },
119
+ {
120
+ "accuracy": 0.61194,
121
+ "f1": 0.565044,
122
+ "f1_weighted": 0.652016,
123
+ "ap": 0.269697,
124
+ "ap_weighted": 0.269697
125
+ },
126
+ {
127
+ "accuracy": 0.668657,
128
+ "f1": 0.603271,
129
+ "f1_weighted": 0.70135,
130
+ "ap": 0.285582,
131
+ "ap_weighted": 0.285582
132
+ },
133
+ {
134
+ "accuracy": 0.625373,
135
+ "f1": 0.56863,
136
+ "f1_weighted": 0.663902,
137
+ "ap": 0.26389,
138
+ "ap_weighted": 0.26389
139
+ },
140
+ {
141
+ "accuracy": 0.744776,
142
+ "f1": 0.648233,
143
+ "f1_weighted": 0.760454,
144
+ "ap": 0.307423,
145
+ "ap_weighted": 0.307423
146
+ },
147
+ {
148
+ "accuracy": 0.667164,
149
+ "f1": 0.613827,
150
+ "f1_weighted": 0.701223,
151
+ "ap": 0.30575,
152
+ "ap_weighted": 0.30575
153
+ },
154
+ {
155
+ "accuracy": 0.591045,
156
+ "f1": 0.547427,
157
+ "f1_weighted": 0.632985,
158
+ "ap": 0.259001,
159
+ "ap_weighted": 0.259001
160
+ },
161
+ {
162
+ "accuracy": 0.646269,
163
+ "f1": 0.588519,
164
+ "f1_weighted": 0.682391,
165
+ "ap": 0.279194,
166
+ "ap_weighted": 0.279194
167
+ }
168
+ ],
169
+ "main_score": 0.64194,
170
+ "hf_subset": "en",
171
+ "languages": [
172
+ "eng-Latn"
173
+ ]
174
+ }
175
+ ]
176
+ },
177
+ "evaluation_time": 18.05203342437744,
178
+ "kg_co2_emissions": null
179
+ }
mteb/AmazonPolarityClassification.json ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "e2d317d38cd51312af73b3d32a06d1a08b442046",
3
+ "task_name": "AmazonPolarityClassification",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "accuracy": 0.671713,
9
+ "f1": 0.669443,
10
+ "f1_weighted": 0.669443,
11
+ "ap": 0.617296,
12
+ "ap_weighted": 0.617296,
13
+ "scores_per_experiment": [
14
+ {
15
+ "accuracy": 0.730195,
16
+ "f1": 0.729644,
17
+ "f1_weighted": 0.729644,
18
+ "ap": 0.673348,
19
+ "ap_weighted": 0.673348
20
+ },
21
+ {
22
+ "accuracy": 0.648605,
23
+ "f1": 0.647126,
24
+ "f1_weighted": 0.647126,
25
+ "ap": 0.593854,
26
+ "ap_weighted": 0.593854
27
+ },
28
+ {
29
+ "accuracy": 0.720182,
30
+ "f1": 0.716345,
31
+ "f1_weighted": 0.716345,
32
+ "ap": 0.649422,
33
+ "ap_weighted": 0.649422
34
+ },
35
+ {
36
+ "accuracy": 0.717512,
37
+ "f1": 0.715561,
38
+ "f1_weighted": 0.715561,
39
+ "ap": 0.665462,
40
+ "ap_weighted": 0.665462
41
+ },
42
+ {
43
+ "accuracy": 0.660528,
44
+ "f1": 0.659856,
45
+ "f1_weighted": 0.659856,
46
+ "ap": 0.60393,
47
+ "ap_weighted": 0.60393
48
+ },
49
+ {
50
+ "accuracy": 0.66763,
51
+ "f1": 0.66655,
52
+ "f1_weighted": 0.66655,
53
+ "ap": 0.609044,
54
+ "ap_weighted": 0.609044
55
+ },
56
+ {
57
+ "accuracy": 0.67263,
58
+ "f1": 0.670559,
59
+ "f1_weighted": 0.670559,
60
+ "ap": 0.612038,
61
+ "ap_weighted": 0.612038
62
+ },
63
+ {
64
+ "accuracy": 0.677455,
65
+ "f1": 0.674938,
66
+ "f1_weighted": 0.674938,
67
+ "ap": 0.626943,
68
+ "ap_weighted": 0.626943
69
+ },
70
+ {
71
+ "accuracy": 0.582143,
72
+ "f1": 0.574911,
73
+ "f1_weighted": 0.574911,
74
+ "ap": 0.546423,
75
+ "ap_weighted": 0.546423
76
+ },
77
+ {
78
+ "accuracy": 0.64025,
79
+ "f1": 0.638938,
80
+ "f1_weighted": 0.638938,
81
+ "ap": 0.592492,
82
+ "ap_weighted": 0.592492
83
+ }
84
+ ],
85
+ "main_score": 0.671713,
86
+ "hf_subset": "default",
87
+ "languages": [
88
+ "eng-Latn"
89
+ ]
90
+ }
91
+ ]
92
+ },
93
+ "evaluation_time": 439.54625058174133,
94
+ "kg_co2_emissions": null
95
+ }
mteb/AmazonReviewsClassification.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d",
3
+ "task_name": "AmazonReviewsClassification",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "accuracy": 0.32318,
9
+ "f1": 0.319973,
10
+ "f1_weighted": 0.319973,
11
+ "scores_per_experiment": [
12
+ {
13
+ "accuracy": 0.341,
14
+ "f1": 0.333855,
15
+ "f1_weighted": 0.333855
16
+ },
17
+ {
18
+ "accuracy": 0.3368,
19
+ "f1": 0.338041,
20
+ "f1_weighted": 0.338041
21
+ },
22
+ {
23
+ "accuracy": 0.3336,
24
+ "f1": 0.330524,
25
+ "f1_weighted": 0.330524
26
+ },
27
+ {
28
+ "accuracy": 0.3086,
29
+ "f1": 0.309647,
30
+ "f1_weighted": 0.309647
31
+ },
32
+ {
33
+ "accuracy": 0.3114,
34
+ "f1": 0.305097,
35
+ "f1_weighted": 0.305097
36
+ },
37
+ {
38
+ "accuracy": 0.3334,
39
+ "f1": 0.330257,
40
+ "f1_weighted": 0.330257
41
+ },
42
+ {
43
+ "accuracy": 0.299,
44
+ "f1": 0.292306,
45
+ "f1_weighted": 0.292306
46
+ },
47
+ {
48
+ "accuracy": 0.3384,
49
+ "f1": 0.336788,
50
+ "f1_weighted": 0.336788
51
+ },
52
+ {
53
+ "accuracy": 0.3242,
54
+ "f1": 0.31879,
55
+ "f1_weighted": 0.31879
56
+ },
57
+ {
58
+ "accuracy": 0.3054,
59
+ "f1": 0.304421,
60
+ "f1_weighted": 0.304421
61
+ }
62
+ ],
63
+ "main_score": 0.32318,
64
+ "hf_subset": "en",
65
+ "languages": [
66
+ "eng-Latn"
67
+ ]
68
+ }
69
+ ]
70
+ },
71
+ "evaluation_time": 9.712511777877808,
72
+ "kg_co2_emissions": null
73
+ }
mteb/ArguAna.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a",
3
+ "task_name": "ArguAna",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.24609,
9
+ "ndcg_at_3": 0.36278,
10
+ "ndcg_at_5": 0.40903,
11
+ "ndcg_at_10": 0.46381,
12
+ "ndcg_at_20": 0.49571,
13
+ "ndcg_at_100": 0.51666,
14
+ "ndcg_at_1000": 0.52098,
15
+ "map_at_1": 0.24609,
16
+ "map_at_3": 0.33404,
17
+ "map_at_5": 0.35968,
18
+ "map_at_10": 0.38204,
19
+ "map_at_20": 0.39113,
20
+ "map_at_100": 0.39432,
21
+ "map_at_1000": 0.3945,
22
+ "recall_at_1": 0.24609,
23
+ "recall_at_3": 0.44595,
24
+ "recall_at_5": 0.55832,
25
+ "recall_at_10": 0.72902,
26
+ "recall_at_20": 0.85277,
27
+ "recall_at_100": 0.96159,
28
+ "recall_at_1000": 0.99431,
29
+ "precision_at_1": 0.24609,
30
+ "precision_at_3": 0.14865,
31
+ "precision_at_5": 0.11166,
32
+ "precision_at_10": 0.0729,
33
+ "precision_at_20": 0.04264,
34
+ "precision_at_100": 0.00962,
35
+ "precision_at_1000": 0.00099,
36
+ "mrr_at_1": 0.251067,
37
+ "mrr_at_3": 0.335941,
38
+ "mrr_at_5": 0.361546,
39
+ "mrr_at_10": 0.383849,
40
+ "mrr_at_20": 0.393004,
41
+ "mrr_at_100": 0.396189,
42
+ "mrr_at_1000": 0.396374,
43
+ "nauc_ndcg_at_1_max": -0.056006,
44
+ "nauc_ndcg_at_1_std": -0.048049,
45
+ "nauc_ndcg_at_1_diff1": 0.14903,
46
+ "nauc_ndcg_at_3_max": -0.00408,
47
+ "nauc_ndcg_at_3_std": -0.019375,
48
+ "nauc_ndcg_at_3_diff1": 0.121454,
49
+ "nauc_ndcg_at_5_max": -0.004685,
50
+ "nauc_ndcg_at_5_std": -0.020393,
51
+ "nauc_ndcg_at_5_diff1": 0.107318,
52
+ "nauc_ndcg_at_10_max": 0.030675,
53
+ "nauc_ndcg_at_10_std": -0.003638,
54
+ "nauc_ndcg_at_10_diff1": 0.113071,
55
+ "nauc_ndcg_at_20_max": 0.025101,
56
+ "nauc_ndcg_at_20_std": -0.003348,
57
+ "nauc_ndcg_at_20_diff1": 0.115383,
58
+ "nauc_ndcg_at_100_max": 0.010754,
59
+ "nauc_ndcg_at_100_std": 0.001077,
60
+ "nauc_ndcg_at_100_diff1": 0.118117,
61
+ "nauc_ndcg_at_1000_max": 0.005791,
62
+ "nauc_ndcg_at_1000_std": -0.006433,
63
+ "nauc_ndcg_at_1000_diff1": 0.119586,
64
+ "nauc_map_at_1_max": -0.056006,
65
+ "nauc_map_at_1_std": -0.048049,
66
+ "nauc_map_at_1_diff1": 0.14903,
67
+ "nauc_map_at_3_max": -0.015511,
68
+ "nauc_map_at_3_std": -0.024433,
69
+ "nauc_map_at_3_diff1": 0.126974,
70
+ "nauc_map_at_5_max": -0.015512,
71
+ "nauc_map_at_5_std": -0.024537,
72
+ "nauc_map_at_5_diff1": 0.11982,
73
+ "nauc_map_at_10_max": -0.00244,
74
+ "nauc_map_at_10_std": -0.017698,
75
+ "nauc_map_at_10_diff1": 0.122534,
76
+ "nauc_map_at_20_max": -0.00477,
77
+ "nauc_map_at_20_std": -0.018189,
78
+ "nauc_map_at_20_diff1": 0.123218,
79
+ "nauc_map_at_100_max": -0.006815,
80
+ "nauc_map_at_100_std": -0.017488,
81
+ "nauc_map_at_100_diff1": 0.123542,
82
+ "nauc_map_at_1000_max": -0.006971,
83
+ "nauc_map_at_1000_std": -0.017692,
84
+ "nauc_map_at_1000_diff1": 0.123594,
85
+ "nauc_recall_at_1_max": -0.056006,
86
+ "nauc_recall_at_1_std": -0.048049,
87
+ "nauc_recall_at_1_diff1": 0.14903,
88
+ "nauc_recall_at_3_max": 0.026873,
89
+ "nauc_recall_at_3_std": -0.006182,
90
+ "nauc_recall_at_3_diff1": 0.106725,
91
+ "nauc_recall_at_5_max": 0.025734,
92
+ "nauc_recall_at_5_std": -0.009853,
93
+ "nauc_recall_at_5_diff1": 0.068572,
94
+ "nauc_recall_at_10_max": 0.178111,
95
+ "nauc_recall_at_10_std": 0.056516,
96
+ "nauc_recall_at_10_diff1": 0.07601,
97
+ "nauc_recall_at_20_max": 0.239859,
98
+ "nauc_recall_at_20_std": 0.104235,
99
+ "nauc_recall_at_20_diff1": 0.070172,
100
+ "nauc_recall_at_100_max": 0.307669,
101
+ "nauc_recall_at_100_std": 0.509552,
102
+ "nauc_recall_at_100_diff1": 0.051657,
103
+ "nauc_recall_at_1000_max": 0.254176,
104
+ "nauc_recall_at_1000_std": 0.730444,
105
+ "nauc_recall_at_1000_diff1": 0.166349,
106
+ "nauc_precision_at_1_max": -0.056006,
107
+ "nauc_precision_at_1_std": -0.048049,
108
+ "nauc_precision_at_1_diff1": 0.14903,
109
+ "nauc_precision_at_3_max": 0.026873,
110
+ "nauc_precision_at_3_std": -0.006182,
111
+ "nauc_precision_at_3_diff1": 0.106725,
112
+ "nauc_precision_at_5_max": 0.025734,
113
+ "nauc_precision_at_5_std": -0.009853,
114
+ "nauc_precision_at_5_diff1": 0.068572,
115
+ "nauc_precision_at_10_max": 0.178111,
116
+ "nauc_precision_at_10_std": 0.056516,
117
+ "nauc_precision_at_10_diff1": 0.07601,
118
+ "nauc_precision_at_20_max": 0.239859,
119
+ "nauc_precision_at_20_std": 0.104235,
120
+ "nauc_precision_at_20_diff1": 0.070172,
121
+ "nauc_precision_at_100_max": 0.307669,
122
+ "nauc_precision_at_100_std": 0.509552,
123
+ "nauc_precision_at_100_diff1": 0.051657,
124
+ "nauc_precision_at_1000_max": 0.254176,
125
+ "nauc_precision_at_1000_std": 0.730444,
126
+ "nauc_precision_at_1000_diff1": 0.166349,
127
+ "nauc_mrr_at_1_max": -0.051381,
128
+ "nauc_mrr_at_1_std": -0.041855,
129
+ "nauc_mrr_at_1_diff1": 0.132056,
130
+ "nauc_mrr_at_3_max": -0.017796,
131
+ "nauc_mrr_at_3_std": -0.023432,
132
+ "nauc_mrr_at_3_diff1": 0.114369,
133
+ "nauc_mrr_at_5_max": -0.017394,
134
+ "nauc_mrr_at_5_std": -0.023168,
135
+ "nauc_mrr_at_5_diff1": 0.107454,
136
+ "nauc_mrr_at_10_max": -0.005076,
137
+ "nauc_mrr_at_10_std": -0.016223,
138
+ "nauc_mrr_at_10_diff1": 0.109067,
139
+ "nauc_mrr_at_20_max": -0.00715,
140
+ "nauc_mrr_at_20_std": -0.016808,
141
+ "nauc_mrr_at_20_diff1": 0.109766,
142
+ "nauc_mrr_at_100_max": -0.009223,
143
+ "nauc_mrr_at_100_std": -0.016097,
144
+ "nauc_mrr_at_100_diff1": 0.109945,
145
+ "nauc_mrr_at_1000_max": -0.00938,
146
+ "nauc_mrr_at_1000_std": -0.0163,
147
+ "nauc_mrr_at_1000_diff1": 0.109988,
148
+ "main_score": 0.46381,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 18.032260179519653,
157
+ "kg_co2_emissions": null
158
+ }
mteb/ArxivClusteringP2P.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d",
3
+ "task_name": "ArxivClusteringP2P",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "v_measure": 0.389781,
9
+ "v_measure_std": 0.140064,
10
+ "v_measures": [
11
+ 0.397318,
12
+ 0.395931,
13
+ 0.398734,
14
+ 0.421181,
15
+ 0.402112,
16
+ 0.39659,
17
+ 0.40027,
18
+ 0.4156,
19
+ 0.418672,
20
+ 0.408705,
21
+ 0.442656,
22
+ 0.441182,
23
+ 0.438416,
24
+ 0.447467,
25
+ 0.440814,
26
+ 0.442237,
27
+ 0.445092,
28
+ 0.440632,
29
+ 0.445347,
30
+ 0.437578,
31
+ 0.391066,
32
+ 0.265296,
33
+ 0.30278,
34
+ 0.296997,
35
+ 0.291818,
36
+ 0.212322,
37
+ 0.256389,
38
+ 0.159543,
39
+ 0.200181,
40
+ 1.0,
41
+ 0.230291
42
+ ],
43
+ "main_score": 0.389781,
44
+ "hf_subset": "default",
45
+ "languages": [
46
+ "eng-Latn"
47
+ ]
48
+ }
49
+ ]
50
+ },
51
+ "evaluation_time": 1234.510992050171,
52
+ "kg_co2_emissions": null
53
+ }
mteb/ArxivClusteringS2S.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53",
3
+ "task_name": "ArxivClusteringS2S",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "v_measure": 0.289645,
9
+ "v_measure_std": 0.142769,
10
+ "v_measures": [
11
+ 0.284647,
12
+ 0.288473,
13
+ 0.27734,
14
+ 0.269279,
15
+ 0.263681,
16
+ 0.266674,
17
+ 0.272582,
18
+ 0.251594,
19
+ 0.287376,
20
+ 0.275816,
21
+ 0.326475,
22
+ 0.330363,
23
+ 0.320361,
24
+ 0.330296,
25
+ 0.328716,
26
+ 0.325683,
27
+ 0.321654,
28
+ 0.324298,
29
+ 0.326664,
30
+ 0.316507,
31
+ 0.304434,
32
+ 0.188541,
33
+ 0.204597,
34
+ 0.259977,
35
+ 0.237603,
36
+ 0.164765,
37
+ 0.199714,
38
+ 0.108873,
39
+ 0.155307,
40
+ 1.0,
41
+ 0.166707
42
+ ],
43
+ "main_score": 0.289645,
44
+ "hf_subset": "default",
45
+ "languages": [
46
+ "eng-Latn"
47
+ ]
48
+ }
49
+ ]
50
+ },
51
+ "evaluation_time": 300.5607361793518,
52
+ "kg_co2_emissions": null
53
+ }
mteb/AskUbuntuDupQuestions.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "2000358ca161889fa9c082cb41daa8dcfb161a54",
3
+ "task_name": "AskUbuntuDupQuestions",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "map": 0.579553,
9
+ "mrr": 0.708293,
10
+ "nAUC_map_max": 0.232041,
11
+ "nAUC_map_std": 0.210013,
12
+ "nAUC_map_diff1": 0.067763,
13
+ "nAUC_mrr_max": 0.331435,
14
+ "nAUC_mrr_std": 0.258722,
15
+ "nAUC_mrr_diff1": 0.102834,
16
+ "main_score": 0.579553,
17
+ "hf_subset": "default",
18
+ "languages": [
19
+ "eng-Latn"
20
+ ]
21
+ }
22
+ ]
23
+ },
24
+ "evaluation_time": 2.8852455615997314,
25
+ "kg_co2_emissions": null
26
+ }
mteb/BIOSSES.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a",
3
+ "task_name": "BIOSSES",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "pearson": 0.850909,
9
+ "spearman": 0.807386,
10
+ "cosine_pearson": 0.850909,
11
+ "cosine_spearman": 0.807386,
12
+ "manhattan_pearson": 0.820327,
13
+ "manhattan_spearman": 0.78562,
14
+ "euclidean_pearson": 0.826159,
15
+ "euclidean_spearman": 0.793542,
16
+ "main_score": 0.807386,
17
+ "hf_subset": "default",
18
+ "languages": [
19
+ "eng-Latn"
20
+ ]
21
+ }
22
+ ]
23
+ },
24
+ "evaluation_time": 0.15388250350952148,
25
+ "kg_co2_emissions": null
26
+ }
mteb/Banking77Classification.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "0fd18e25b25c072e09e0d92ab615fda904d66300",
3
+ "task_name": "Banking77Classification",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "accuracy": 0.82026,
9
+ "f1": 0.819738,
10
+ "f1_weighted": 0.819738,
11
+ "scores_per_experiment": [
12
+ {
13
+ "accuracy": 0.820779,
14
+ "f1": 0.820659,
15
+ "f1_weighted": 0.820659
16
+ },
17
+ {
18
+ "accuracy": 0.827273,
19
+ "f1": 0.827807,
20
+ "f1_weighted": 0.827807
21
+ },
22
+ {
23
+ "accuracy": 0.837013,
24
+ "f1": 0.836189,
25
+ "f1_weighted": 0.836189
26
+ },
27
+ {
28
+ "accuracy": 0.827273,
29
+ "f1": 0.827111,
30
+ "f1_weighted": 0.827111
31
+ },
32
+ {
33
+ "accuracy": 0.816558,
34
+ "f1": 0.816573,
35
+ "f1_weighted": 0.816573
36
+ },
37
+ {
38
+ "accuracy": 0.800649,
39
+ "f1": 0.79907,
40
+ "f1_weighted": 0.79907
41
+ },
42
+ {
43
+ "accuracy": 0.816558,
44
+ "f1": 0.815777,
45
+ "f1_weighted": 0.815777
46
+ },
47
+ {
48
+ "accuracy": 0.812338,
49
+ "f1": 0.811554,
50
+ "f1_weighted": 0.811554
51
+ },
52
+ {
53
+ "accuracy": 0.828247,
54
+ "f1": 0.828126,
55
+ "f1_weighted": 0.828126
56
+ },
57
+ {
58
+ "accuracy": 0.815909,
59
+ "f1": 0.814514,
60
+ "f1_weighted": 0.814514
61
+ }
62
+ ],
63
+ "main_score": 0.82026,
64
+ "hf_subset": "default",
65
+ "languages": [
66
+ "eng-Latn"
67
+ ]
68
+ }
69
+ ]
70
+ },
71
+ "evaluation_time": 11.350855588912964,
72
+ "kg_co2_emissions": null
73
+ }
mteb/BiorxivClusteringP2P.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40",
3
+ "task_name": "BiorxivClusteringP2P",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "v_measure": 0.30787,
9
+ "v_measure_std": 0.009286,
10
+ "v_measures": [
11
+ 0.294563,
12
+ 0.300806,
13
+ 0.300434,
14
+ 0.298455,
15
+ 0.313446,
16
+ 0.313647,
17
+ 0.310572,
18
+ 0.303024,
19
+ 0.3205,
20
+ 0.323252
21
+ ],
22
+ "main_score": 0.30787,
23
+ "hf_subset": "default",
24
+ "languages": [
25
+ "eng-Latn"
26
+ ]
27
+ }
28
+ ]
29
+ },
30
+ "evaluation_time": 183.2675678730011,
31
+ "kg_co2_emissions": null
32
+ }
mteb/BiorxivClusteringS2S.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908",
3
+ "task_name": "BiorxivClusteringS2S",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "v_measure": 0.270998,
9
+ "v_measure_std": 0.007908,
10
+ "v_measures": [
11
+ 0.269737,
12
+ 0.264231,
13
+ 0.263528,
14
+ 0.267448,
15
+ 0.268571,
16
+ 0.273179,
17
+ 0.261491,
18
+ 0.27174,
19
+ 0.282342,
20
+ 0.287709
21
+ ],
22
+ "main_score": 0.270998,
23
+ "hf_subset": "default",
24
+ "languages": [
25
+ "eng-Latn"
26
+ ]
27
+ }
28
+ ]
29
+ },
30
+ "evaluation_time": 32.66898798942566,
31
+ "kg_co2_emissions": null
32
+ }
mteb/CQADupstackAndroidRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "f46a197baaae43b4f621051089b82a364682dfeb",
3
+ "task_name": "CQADupstackAndroidRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.29757,
9
+ "ndcg_at_3": 0.34736,
10
+ "ndcg_at_5": 0.36529,
11
+ "ndcg_at_10": 0.39114,
12
+ "ndcg_at_20": 0.4105,
13
+ "ndcg_at_100": 0.44359,
14
+ "ndcg_at_1000": 0.47015,
15
+ "map_at_1": 0.24501,
16
+ "map_at_3": 0.30857,
17
+ "map_at_5": 0.32307,
18
+ "map_at_10": 0.33582,
19
+ "map_at_20": 0.34229,
20
+ "map_at_100": 0.34801,
21
+ "map_at_1000": 0.34936,
22
+ "recall_at_1": 0.24501,
23
+ "recall_at_3": 0.37393,
24
+ "recall_at_5": 0.42701,
25
+ "recall_at_10": 0.5047,
26
+ "recall_at_20": 0.57451,
27
+ "recall_at_100": 0.72912,
28
+ "recall_at_1000": 0.90643,
29
+ "precision_at_1": 0.29757,
30
+ "precision_at_3": 0.16738,
31
+ "precision_at_5": 0.11788,
32
+ "precision_at_10": 0.07282,
33
+ "precision_at_20": 0.04349,
34
+ "precision_at_100": 0.01209,
35
+ "precision_at_1000": 0.00172,
36
+ "mrr_at_1": 0.297568,
37
+ "mrr_at_3": 0.360515,
38
+ "mrr_at_5": 0.37432,
39
+ "mrr_at_10": 0.384942,
40
+ "mrr_at_20": 0.389932,
41
+ "mrr_at_100": 0.393632,
42
+ "mrr_at_1000": 0.394271,
43
+ "nauc_ndcg_at_1_max": 0.35163,
44
+ "nauc_ndcg_at_1_std": -0.04949,
45
+ "nauc_ndcg_at_1_diff1": 0.506784,
46
+ "nauc_ndcg_at_3_max": 0.360879,
47
+ "nauc_ndcg_at_3_std": -0.032672,
48
+ "nauc_ndcg_at_3_diff1": 0.472847,
49
+ "nauc_ndcg_at_5_max": 0.364638,
50
+ "nauc_ndcg_at_5_std": -0.016767,
51
+ "nauc_ndcg_at_5_diff1": 0.466553,
52
+ "nauc_ndcg_at_10_max": 0.373779,
53
+ "nauc_ndcg_at_10_std": -0.007015,
54
+ "nauc_ndcg_at_10_diff1": 0.468114,
55
+ "nauc_ndcg_at_20_max": 0.374599,
56
+ "nauc_ndcg_at_20_std": -0.000285,
57
+ "nauc_ndcg_at_20_diff1": 0.460127,
58
+ "nauc_ndcg_at_100_max": 0.376178,
59
+ "nauc_ndcg_at_100_std": 0.003917,
60
+ "nauc_ndcg_at_100_diff1": 0.459498,
61
+ "nauc_ndcg_at_1000_max": 0.378896,
62
+ "nauc_ndcg_at_1000_std": 0.011141,
63
+ "nauc_ndcg_at_1000_diff1": 0.45977,
64
+ "nauc_map_at_1_max": 0.329735,
65
+ "nauc_map_at_1_std": -0.06633,
66
+ "nauc_map_at_1_diff1": 0.52641,
67
+ "nauc_map_at_3_max": 0.352305,
68
+ "nauc_map_at_3_std": -0.050847,
69
+ "nauc_map_at_3_diff1": 0.496157,
70
+ "nauc_map_at_5_max": 0.356133,
71
+ "nauc_map_at_5_std": -0.040399,
72
+ "nauc_map_at_5_diff1": 0.488895,
73
+ "nauc_map_at_10_max": 0.361679,
74
+ "nauc_map_at_10_std": -0.03574,
75
+ "nauc_map_at_10_diff1": 0.487396,
76
+ "nauc_map_at_20_max": 0.362918,
77
+ "nauc_map_at_20_std": -0.032796,
78
+ "nauc_map_at_20_diff1": 0.484663,
79
+ "nauc_map_at_100_max": 0.364498,
80
+ "nauc_map_at_100_std": -0.030688,
81
+ "nauc_map_at_100_diff1": 0.484524,
82
+ "nauc_map_at_1000_max": 0.364623,
83
+ "nauc_map_at_1000_std": -0.030062,
84
+ "nauc_map_at_1000_diff1": 0.484259,
85
+ "nauc_recall_at_1_max": 0.329735,
86
+ "nauc_recall_at_1_std": -0.06633,
87
+ "nauc_recall_at_1_diff1": 0.52641,
88
+ "nauc_recall_at_3_max": 0.34925,
89
+ "nauc_recall_at_3_std": -0.020609,
90
+ "nauc_recall_at_3_diff1": 0.441485,
91
+ "nauc_recall_at_5_max": 0.347606,
92
+ "nauc_recall_at_5_std": 0.016887,
93
+ "nauc_recall_at_5_diff1": 0.414227,
94
+ "nauc_recall_at_10_max": 0.3657,
95
+ "nauc_recall_at_10_std": 0.055427,
96
+ "nauc_recall_at_10_diff1": 0.409419,
97
+ "nauc_recall_at_20_max": 0.366151,
98
+ "nauc_recall_at_20_std": 0.091305,
99
+ "nauc_recall_at_20_diff1": 0.372562,
100
+ "nauc_recall_at_100_max": 0.375285,
101
+ "nauc_recall_at_100_std": 0.138761,
102
+ "nauc_recall_at_100_diff1": 0.355754,
103
+ "nauc_recall_at_1000_max": 0.485408,
104
+ "nauc_recall_at_1000_std": 0.470295,
105
+ "nauc_recall_at_1000_diff1": 0.311568,
106
+ "nauc_precision_at_1_max": 0.35163,
107
+ "nauc_precision_at_1_std": -0.04949,
108
+ "nauc_precision_at_1_diff1": 0.506784,
109
+ "nauc_precision_at_3_max": 0.332905,
110
+ "nauc_precision_at_3_std": -0.016148,
111
+ "nauc_precision_at_3_diff1": 0.331776,
112
+ "nauc_precision_at_5_max": 0.333257,
113
+ "nauc_precision_at_5_std": 0.04402,
114
+ "nauc_precision_at_5_diff1": 0.272371,
115
+ "nauc_precision_at_10_max": 0.307887,
116
+ "nauc_precision_at_10_std": 0.070138,
117
+ "nauc_precision_at_10_diff1": 0.20856,
118
+ "nauc_precision_at_20_max": 0.27621,
119
+ "nauc_precision_at_20_std": 0.099861,
120
+ "nauc_precision_at_20_diff1": 0.131662,
121
+ "nauc_precision_at_100_max": 0.176654,
122
+ "nauc_precision_at_100_std": 0.126523,
123
+ "nauc_precision_at_100_diff1": 0.007577,
124
+ "nauc_precision_at_1000_max": -0.000854,
125
+ "nauc_precision_at_1000_std": 0.095769,
126
+ "nauc_precision_at_1000_diff1": -0.16308,
127
+ "nauc_mrr_at_1_max": 0.35163,
128
+ "nauc_mrr_at_1_std": -0.04949,
129
+ "nauc_mrr_at_1_diff1": 0.506784,
130
+ "nauc_mrr_at_3_max": 0.363793,
131
+ "nauc_mrr_at_3_std": -0.03288,
132
+ "nauc_mrr_at_3_diff1": 0.469367,
133
+ "nauc_mrr_at_5_max": 0.364904,
134
+ "nauc_mrr_at_5_std": -0.021184,
135
+ "nauc_mrr_at_5_diff1": 0.463741,
136
+ "nauc_mrr_at_10_max": 0.368009,
137
+ "nauc_mrr_at_10_std": -0.016722,
138
+ "nauc_mrr_at_10_diff1": 0.464536,
139
+ "nauc_mrr_at_20_max": 0.368492,
140
+ "nauc_mrr_at_20_std": -0.015649,
141
+ "nauc_mrr_at_20_diff1": 0.462973,
142
+ "nauc_mrr_at_100_max": 0.367136,
143
+ "nauc_mrr_at_100_std": -0.016903,
144
+ "nauc_mrr_at_100_diff1": 0.463234,
145
+ "nauc_mrr_at_1000_max": 0.367098,
146
+ "nauc_mrr_at_1000_std": -0.016532,
147
+ "nauc_mrr_at_1000_diff1": 0.463207,
148
+ "main_score": 0.39114,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 27.28526473045349,
157
+ "kg_co2_emissions": null
158
+ }
mteb/CQADupstackEnglishRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0",
3
+ "task_name": "CQADupstackEnglishRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.32229,
9
+ "ndcg_at_3": 0.35352,
10
+ "ndcg_at_5": 0.36907,
11
+ "ndcg_at_10": 0.38867,
12
+ "ndcg_at_20": 0.40624,
13
+ "ndcg_at_100": 0.43169,
14
+ "ndcg_at_1000": 0.45411,
15
+ "map_at_1": 0.25286,
16
+ "map_at_3": 0.31391,
17
+ "map_at_5": 0.3276,
18
+ "map_at_10": 0.33835,
19
+ "map_at_20": 0.34453,
20
+ "map_at_100": 0.34926,
21
+ "map_at_1000": 0.35039,
22
+ "recall_at_1": 0.25286,
23
+ "recall_at_3": 0.36961,
24
+ "recall_at_5": 0.4154,
25
+ "recall_at_10": 0.47543,
26
+ "recall_at_20": 0.53979,
27
+ "recall_at_100": 0.65744,
28
+ "recall_at_1000": 0.80437,
29
+ "precision_at_1": 0.32229,
30
+ "precision_at_3": 0.17113,
31
+ "precision_at_5": 0.11962,
32
+ "precision_at_10": 0.0721,
33
+ "precision_at_20": 0.04287,
34
+ "precision_at_100": 0.01177,
35
+ "precision_at_1000": 0.00165,
36
+ "mrr_at_1": 0.322293,
37
+ "mrr_at_3": 0.377707,
38
+ "mrr_at_5": 0.388917,
39
+ "mrr_at_10": 0.398009,
40
+ "mrr_at_20": 0.402384,
41
+ "mrr_at_100": 0.40498,
42
+ "mrr_at_1000": 0.40545,
43
+ "nauc_ndcg_at_1_max": 0.386829,
44
+ "nauc_ndcg_at_1_std": 0.074424,
45
+ "nauc_ndcg_at_1_diff1": 0.528493,
46
+ "nauc_ndcg_at_3_max": 0.370654,
47
+ "nauc_ndcg_at_3_std": 0.054984,
48
+ "nauc_ndcg_at_3_diff1": 0.466131,
49
+ "nauc_ndcg_at_5_max": 0.36923,
50
+ "nauc_ndcg_at_5_std": 0.061408,
51
+ "nauc_ndcg_at_5_diff1": 0.457153,
52
+ "nauc_ndcg_at_10_max": 0.365462,
53
+ "nauc_ndcg_at_10_std": 0.062711,
54
+ "nauc_ndcg_at_10_diff1": 0.455885,
55
+ "nauc_ndcg_at_20_max": 0.364087,
56
+ "nauc_ndcg_at_20_std": 0.064921,
57
+ "nauc_ndcg_at_20_diff1": 0.453433,
58
+ "nauc_ndcg_at_100_max": 0.366238,
59
+ "nauc_ndcg_at_100_std": 0.083159,
60
+ "nauc_ndcg_at_100_diff1": 0.447424,
61
+ "nauc_ndcg_at_1000_max": 0.37042,
62
+ "nauc_ndcg_at_1000_std": 0.089336,
63
+ "nauc_ndcg_at_1000_diff1": 0.448673,
64
+ "nauc_map_at_1_max": 0.332074,
65
+ "nauc_map_at_1_std": 0.021052,
66
+ "nauc_map_at_1_diff1": 0.546382,
67
+ "nauc_map_at_3_max": 0.352336,
68
+ "nauc_map_at_3_std": 0.025225,
69
+ "nauc_map_at_3_diff1": 0.49159,
70
+ "nauc_map_at_5_max": 0.355997,
71
+ "nauc_map_at_5_std": 0.037594,
72
+ "nauc_map_at_5_diff1": 0.484115,
73
+ "nauc_map_at_10_max": 0.358431,
74
+ "nauc_map_at_10_std": 0.042814,
75
+ "nauc_map_at_10_diff1": 0.482818,
76
+ "nauc_map_at_20_max": 0.359632,
77
+ "nauc_map_at_20_std": 0.046509,
78
+ "nauc_map_at_20_diff1": 0.481984,
79
+ "nauc_map_at_100_max": 0.361803,
80
+ "nauc_map_at_100_std": 0.051838,
81
+ "nauc_map_at_100_diff1": 0.480556,
82
+ "nauc_map_at_1000_max": 0.362442,
83
+ "nauc_map_at_1000_std": 0.052821,
84
+ "nauc_map_at_1000_diff1": 0.480396,
85
+ "nauc_recall_at_1_max": 0.332074,
86
+ "nauc_recall_at_1_std": 0.021052,
87
+ "nauc_recall_at_1_diff1": 0.546382,
88
+ "nauc_recall_at_3_max": 0.33786,
89
+ "nauc_recall_at_3_std": 0.022159,
90
+ "nauc_recall_at_3_diff1": 0.421871,
91
+ "nauc_recall_at_5_max": 0.336662,
92
+ "nauc_recall_at_5_std": 0.046278,
93
+ "nauc_recall_at_5_diff1": 0.393118,
94
+ "nauc_recall_at_10_max": 0.322503,
95
+ "nauc_recall_at_10_std": 0.05947,
96
+ "nauc_recall_at_10_diff1": 0.37952,
97
+ "nauc_recall_at_20_max": 0.312259,
98
+ "nauc_recall_at_20_std": 0.068895,
99
+ "nauc_recall_at_20_diff1": 0.355049,
100
+ "nauc_recall_at_100_max": 0.303406,
101
+ "nauc_recall_at_100_std": 0.156142,
102
+ "nauc_recall_at_100_diff1": 0.295623,
103
+ "nauc_recall_at_1000_max": 0.305454,
104
+ "nauc_recall_at_1000_std": 0.214645,
105
+ "nauc_recall_at_1000_diff1": 0.271848,
106
+ "nauc_precision_at_1_max": 0.386829,
107
+ "nauc_precision_at_1_std": 0.074424,
108
+ "nauc_precision_at_1_diff1": 0.528493,
109
+ "nauc_precision_at_3_max": 0.370629,
110
+ "nauc_precision_at_3_std": 0.114377,
111
+ "nauc_precision_at_3_diff1": 0.322216,
112
+ "nauc_precision_at_5_max": 0.356068,
113
+ "nauc_precision_at_5_std": 0.161786,
114
+ "nauc_precision_at_5_diff1": 0.256865,
115
+ "nauc_precision_at_10_max": 0.336078,
116
+ "nauc_precision_at_10_std": 0.188689,
117
+ "nauc_precision_at_10_diff1": 0.195934,
118
+ "nauc_precision_at_20_max": 0.294667,
119
+ "nauc_precision_at_20_std": 0.210742,
120
+ "nauc_precision_at_20_diff1": 0.13482,
121
+ "nauc_precision_at_100_max": 0.229317,
122
+ "nauc_precision_at_100_std": 0.273075,
123
+ "nauc_precision_at_100_diff1": -0.000666,
124
+ "nauc_precision_at_1000_max": 0.139545,
125
+ "nauc_precision_at_1000_std": 0.264182,
126
+ "nauc_precision_at_1000_diff1": -0.103752,
127
+ "nauc_mrr_at_1_max": 0.386829,
128
+ "nauc_mrr_at_1_std": 0.074424,
129
+ "nauc_mrr_at_1_diff1": 0.528493,
130
+ "nauc_mrr_at_3_max": 0.387082,
131
+ "nauc_mrr_at_3_std": 0.075774,
132
+ "nauc_mrr_at_3_diff1": 0.478237,
133
+ "nauc_mrr_at_5_max": 0.385714,
134
+ "nauc_mrr_at_5_std": 0.078757,
135
+ "nauc_mrr_at_5_diff1": 0.46983,
136
+ "nauc_mrr_at_10_max": 0.385054,
137
+ "nauc_mrr_at_10_std": 0.079545,
138
+ "nauc_mrr_at_10_diff1": 0.469465,
139
+ "nauc_mrr_at_20_max": 0.383847,
140
+ "nauc_mrr_at_20_std": 0.079177,
141
+ "nauc_mrr_at_20_diff1": 0.468517,
142
+ "nauc_mrr_at_100_max": 0.38363,
143
+ "nauc_mrr_at_100_std": 0.081133,
144
+ "nauc_mrr_at_100_diff1": 0.467934,
145
+ "nauc_mrr_at_1000_max": 0.383753,
146
+ "nauc_mrr_at_1000_std": 0.08116,
147
+ "nauc_mrr_at_1000_diff1": 0.468037,
148
+ "main_score": 0.38867,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 41.29191493988037,
157
+ "kg_co2_emissions": null
158
+ }
mteb/CQADupstackGamingRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "4885aa143210c98657558c04aaf3dc47cfb54340",
3
+ "task_name": "CQADupstackGamingRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.37555,
9
+ "ndcg_at_3": 0.43752,
10
+ "ndcg_at_5": 0.4595,
11
+ "ndcg_at_10": 0.48593,
12
+ "ndcg_at_20": 0.50274,
13
+ "ndcg_at_100": 0.52922,
14
+ "ndcg_at_1000": 0.5442,
15
+ "map_at_1": 0.32732,
16
+ "map_at_3": 0.40392,
17
+ "map_at_5": 0.41928,
18
+ "map_at_10": 0.43189,
19
+ "map_at_20": 0.4374,
20
+ "map_at_100": 0.44174,
21
+ "map_at_1000": 0.44242,
22
+ "recall_at_1": 0.32732,
23
+ "recall_at_3": 0.48,
24
+ "recall_at_5": 0.53475,
25
+ "recall_at_10": 0.61265,
26
+ "recall_at_20": 0.67586,
27
+ "recall_at_100": 0.80604,
28
+ "recall_at_1000": 0.91408,
29
+ "precision_at_1": 0.37555,
30
+ "precision_at_3": 0.19645,
31
+ "precision_at_5": 0.13455,
32
+ "precision_at_10": 0.079,
33
+ "precision_at_20": 0.04436,
34
+ "precision_at_100": 0.0109,
35
+ "precision_at_1000": 0.00127,
36
+ "mrr_at_1": 0.375549,
37
+ "mrr_at_3": 0.443574,
38
+ "mrr_at_5": 0.455455,
39
+ "mrr_at_10": 0.465721,
40
+ "mrr_at_20": 0.469989,
41
+ "mrr_at_100": 0.473172,
42
+ "mrr_at_1000": 0.473553,
43
+ "nauc_ndcg_at_1_max": 0.34123,
44
+ "nauc_ndcg_at_1_std": -0.017841,
45
+ "nauc_ndcg_at_1_diff1": 0.518322,
46
+ "nauc_ndcg_at_3_max": 0.35477,
47
+ "nauc_ndcg_at_3_std": -0.012643,
48
+ "nauc_ndcg_at_3_diff1": 0.475991,
49
+ "nauc_ndcg_at_5_max": 0.362862,
50
+ "nauc_ndcg_at_5_std": -3.8e-05,
51
+ "nauc_ndcg_at_5_diff1": 0.465251,
52
+ "nauc_ndcg_at_10_max": 0.367788,
53
+ "nauc_ndcg_at_10_std": 0.008117,
54
+ "nauc_ndcg_at_10_diff1": 0.467285,
55
+ "nauc_ndcg_at_20_max": 0.375701,
56
+ "nauc_ndcg_at_20_std": 0.016506,
57
+ "nauc_ndcg_at_20_diff1": 0.46824,
58
+ "nauc_ndcg_at_100_max": 0.378108,
59
+ "nauc_ndcg_at_100_std": 0.029329,
60
+ "nauc_ndcg_at_100_diff1": 0.466243,
61
+ "nauc_ndcg_at_1000_max": 0.378672,
62
+ "nauc_ndcg_at_1000_std": 0.029359,
63
+ "nauc_ndcg_at_1000_diff1": 0.467787,
64
+ "nauc_map_at_1_max": 0.309187,
65
+ "nauc_map_at_1_std": -0.026286,
66
+ "nauc_map_at_1_diff1": 0.51255,
67
+ "nauc_map_at_3_max": 0.339493,
68
+ "nauc_map_at_3_std": -0.024583,
69
+ "nauc_map_at_3_diff1": 0.485462,
70
+ "nauc_map_at_5_max": 0.34567,
71
+ "nauc_map_at_5_std": -0.015681,
72
+ "nauc_map_at_5_diff1": 0.478419,
73
+ "nauc_map_at_10_max": 0.350466,
74
+ "nauc_map_at_10_std": -0.010182,
75
+ "nauc_map_at_10_diff1": 0.479976,
76
+ "nauc_map_at_20_max": 0.354014,
77
+ "nauc_map_at_20_std": -0.006443,
78
+ "nauc_map_at_20_diff1": 0.480216,
79
+ "nauc_map_at_100_max": 0.355061,
80
+ "nauc_map_at_100_std": -0.003215,
81
+ "nauc_map_at_100_diff1": 0.479392,
82
+ "nauc_map_at_1000_max": 0.35521,
83
+ "nauc_map_at_1000_std": -0.003054,
84
+ "nauc_map_at_1000_diff1": 0.479491,
85
+ "nauc_recall_at_1_max": 0.309187,
86
+ "nauc_recall_at_1_std": -0.026286,
87
+ "nauc_recall_at_1_diff1": 0.51255,
88
+ "nauc_recall_at_3_max": 0.34792,
89
+ "nauc_recall_at_3_std": -0.017638,
90
+ "nauc_recall_at_3_diff1": 0.440913,
91
+ "nauc_recall_at_5_max": 0.364564,
92
+ "nauc_recall_at_5_std": 0.014875,
93
+ "nauc_recall_at_5_diff1": 0.409842,
94
+ "nauc_recall_at_10_max": 0.372517,
95
+ "nauc_recall_at_10_std": 0.038871,
96
+ "nauc_recall_at_10_diff1": 0.405621,
97
+ "nauc_recall_at_20_max": 0.408612,
98
+ "nauc_recall_at_20_std": 0.080092,
99
+ "nauc_recall_at_20_diff1": 0.400661,
100
+ "nauc_recall_at_100_max": 0.431074,
101
+ "nauc_recall_at_100_std": 0.195316,
102
+ "nauc_recall_at_100_diff1": 0.363097,
103
+ "nauc_recall_at_1000_max": 0.493019,
104
+ "nauc_recall_at_1000_std": 0.343645,
105
+ "nauc_recall_at_1000_diff1": 0.316154,
106
+ "nauc_precision_at_1_max": 0.34123,
107
+ "nauc_precision_at_1_std": -0.017841,
108
+ "nauc_precision_at_1_diff1": 0.518322,
109
+ "nauc_precision_at_3_max": 0.358498,
110
+ "nauc_precision_at_3_std": 0.027106,
111
+ "nauc_precision_at_3_diff1": 0.347859,
112
+ "nauc_precision_at_5_max": 0.34631,
113
+ "nauc_precision_at_5_std": 0.066776,
114
+ "nauc_precision_at_5_diff1": 0.274746,
115
+ "nauc_precision_at_10_max": 0.338077,
116
+ "nauc_precision_at_10_std": 0.112782,
117
+ "nauc_precision_at_10_diff1": 0.220539,
118
+ "nauc_precision_at_20_max": 0.340661,
119
+ "nauc_precision_at_20_std": 0.161584,
120
+ "nauc_precision_at_20_diff1": 0.172042,
121
+ "nauc_precision_at_100_max": 0.287824,
122
+ "nauc_precision_at_100_std": 0.236474,
123
+ "nauc_precision_at_100_diff1": 0.044878,
124
+ "nauc_precision_at_1000_max": 0.236745,
125
+ "nauc_precision_at_1000_std": 0.224593,
126
+ "nauc_precision_at_1000_diff1": -0.026638,
127
+ "nauc_mrr_at_1_max": 0.34123,
128
+ "nauc_mrr_at_1_std": -0.017841,
129
+ "nauc_mrr_at_1_diff1": 0.518322,
130
+ "nauc_mrr_at_3_max": 0.36619,
131
+ "nauc_mrr_at_3_std": -0.003463,
132
+ "nauc_mrr_at_3_diff1": 0.487033,
133
+ "nauc_mrr_at_5_max": 0.369399,
134
+ "nauc_mrr_at_5_std": 0.001776,
135
+ "nauc_mrr_at_5_diff1": 0.481842,
136
+ "nauc_mrr_at_10_max": 0.370071,
137
+ "nauc_mrr_at_10_std": 0.003426,
138
+ "nauc_mrr_at_10_diff1": 0.483223,
139
+ "nauc_mrr_at_20_max": 0.371624,
140
+ "nauc_mrr_at_20_std": 0.005601,
141
+ "nauc_mrr_at_20_diff1": 0.483756,
142
+ "nauc_mrr_at_100_max": 0.371384,
143
+ "nauc_mrr_at_100_std": 0.006386,
144
+ "nauc_mrr_at_100_diff1": 0.483777,
145
+ "nauc_mrr_at_1000_max": 0.371237,
146
+ "nauc_mrr_at_1000_std": 0.00627,
147
+ "nauc_mrr_at_1000_diff1": 0.483789,
148
+ "main_score": 0.48593,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 45.477328062057495,
157
+ "kg_co2_emissions": null
158
+ }
mteb/CQADupstackGisRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "5003b3064772da1887988e05400cf3806fe491f2",
3
+ "task_name": "CQADupstackGisRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.19322,
9
+ "ndcg_at_3": 0.2464,
10
+ "ndcg_at_5": 0.26398,
11
+ "ndcg_at_10": 0.28629,
12
+ "ndcg_at_20": 0.30424,
13
+ "ndcg_at_100": 0.33635,
14
+ "ndcg_at_1000": 0.36296,
15
+ "map_at_1": 0.1796,
16
+ "map_at_3": 0.22722,
17
+ "map_at_5": 0.23737,
18
+ "map_at_10": 0.24671,
19
+ "map_at_20": 0.25173,
20
+ "map_at_100": 0.25633,
21
+ "map_at_1000": 0.25725,
22
+ "recall_at_1": 0.1796,
23
+ "recall_at_3": 0.28451,
24
+ "recall_at_5": 0.32652,
25
+ "recall_at_10": 0.39283,
26
+ "recall_at_20": 0.46066,
27
+ "recall_at_100": 0.62619,
28
+ "recall_at_1000": 0.83354,
29
+ "precision_at_1": 0.19322,
30
+ "precision_at_3": 0.10395,
31
+ "precision_at_5": 0.07254,
32
+ "precision_at_10": 0.04475,
33
+ "precision_at_20": 0.02672,
34
+ "precision_at_100": 0.0074,
35
+ "precision_at_1000": 0.00101,
36
+ "mrr_at_1": 0.19322,
37
+ "mrr_at_3": 0.243315,
38
+ "mrr_at_5": 0.25371,
39
+ "mrr_at_10": 0.263167,
40
+ "mrr_at_20": 0.268103,
41
+ "mrr_at_100": 0.27234,
42
+ "mrr_at_1000": 0.273059,
43
+ "nauc_ndcg_at_1_max": 0.3509,
44
+ "nauc_ndcg_at_1_std": -0.028343,
45
+ "nauc_ndcg_at_1_diff1": 0.4556,
46
+ "nauc_ndcg_at_3_max": 0.332288,
47
+ "nauc_ndcg_at_3_std": -0.001513,
48
+ "nauc_ndcg_at_3_diff1": 0.383786,
49
+ "nauc_ndcg_at_5_max": 0.323156,
50
+ "nauc_ndcg_at_5_std": -0.002329,
51
+ "nauc_ndcg_at_5_diff1": 0.375401,
52
+ "nauc_ndcg_at_10_max": 0.319542,
53
+ "nauc_ndcg_at_10_std": 0.001314,
54
+ "nauc_ndcg_at_10_diff1": 0.371061,
55
+ "nauc_ndcg_at_20_max": 0.322781,
56
+ "nauc_ndcg_at_20_std": 0.007944,
57
+ "nauc_ndcg_at_20_diff1": 0.359749,
58
+ "nauc_ndcg_at_100_max": 0.321408,
59
+ "nauc_ndcg_at_100_std": 0.021193,
60
+ "nauc_ndcg_at_100_diff1": 0.353214,
61
+ "nauc_ndcg_at_1000_max": 0.333213,
62
+ "nauc_ndcg_at_1000_std": 0.028037,
63
+ "nauc_ndcg_at_1000_diff1": 0.362077,
64
+ "nauc_map_at_1_max": 0.334646,
65
+ "nauc_map_at_1_std": -0.046069,
66
+ "nauc_map_at_1_diff1": 0.475264,
67
+ "nauc_map_at_3_max": 0.330952,
68
+ "nauc_map_at_3_std": -0.014788,
69
+ "nauc_map_at_3_diff1": 0.405655,
70
+ "nauc_map_at_5_max": 0.326424,
71
+ "nauc_map_at_5_std": -0.015209,
72
+ "nauc_map_at_5_diff1": 0.400335,
73
+ "nauc_map_at_10_max": 0.325464,
74
+ "nauc_map_at_10_std": -0.014892,
75
+ "nauc_map_at_10_diff1": 0.398555,
76
+ "nauc_map_at_20_max": 0.326796,
77
+ "nauc_map_at_20_std": -0.013008,
78
+ "nauc_map_at_20_diff1": 0.395344,
79
+ "nauc_map_at_100_max": 0.326354,
80
+ "nauc_map_at_100_std": -0.01094,
81
+ "nauc_map_at_100_diff1": 0.394208,
82
+ "nauc_map_at_1000_max": 0.326999,
83
+ "nauc_map_at_1000_std": -0.01055,
84
+ "nauc_map_at_1000_diff1": 0.394564,
85
+ "nauc_recall_at_1_max": 0.334646,
86
+ "nauc_recall_at_1_std": -0.046069,
87
+ "nauc_recall_at_1_diff1": 0.475264,
88
+ "nauc_recall_at_3_max": 0.312987,
89
+ "nauc_recall_at_3_std": 0.018809,
90
+ "nauc_recall_at_3_diff1": 0.329535,
91
+ "nauc_recall_at_5_max": 0.291171,
92
+ "nauc_recall_at_5_std": 0.015371,
93
+ "nauc_recall_at_5_diff1": 0.310569,
94
+ "nauc_recall_at_10_max": 0.278967,
95
+ "nauc_recall_at_10_std": 0.027129,
96
+ "nauc_recall_at_10_diff1": 0.297162,
97
+ "nauc_recall_at_20_max": 0.285044,
98
+ "nauc_recall_at_20_std": 0.048128,
99
+ "nauc_recall_at_20_diff1": 0.255835,
100
+ "nauc_recall_at_100_max": 0.265933,
101
+ "nauc_recall_at_100_std": 0.119517,
102
+ "nauc_recall_at_100_diff1": 0.202748,
103
+ "nauc_recall_at_1000_max": 0.373161,
104
+ "nauc_recall_at_1000_std": 0.285925,
105
+ "nauc_recall_at_1000_diff1": 0.204139,
106
+ "nauc_precision_at_1_max": 0.3509,
107
+ "nauc_precision_at_1_std": -0.028343,
108
+ "nauc_precision_at_1_diff1": 0.4556,
109
+ "nauc_precision_at_3_max": 0.349653,
110
+ "nauc_precision_at_3_std": 0.037263,
111
+ "nauc_precision_at_3_diff1": 0.316443,
112
+ "nauc_precision_at_5_max": 0.337659,
113
+ "nauc_precision_at_5_std": 0.038751,
114
+ "nauc_precision_at_5_diff1": 0.294004,
115
+ "nauc_precision_at_10_max": 0.327502,
116
+ "nauc_precision_at_10_std": 0.043505,
117
+ "nauc_precision_at_10_diff1": 0.285592,
118
+ "nauc_precision_at_20_max": 0.331996,
119
+ "nauc_precision_at_20_std": 0.076739,
120
+ "nauc_precision_at_20_diff1": 0.232499,
121
+ "nauc_precision_at_100_max": 0.299848,
122
+ "nauc_precision_at_100_std": 0.144632,
123
+ "nauc_precision_at_100_diff1": 0.154486,
124
+ "nauc_precision_at_1000_max": 0.28517,
125
+ "nauc_precision_at_1000_std": 0.204971,
126
+ "nauc_precision_at_1000_diff1": 0.090991,
127
+ "nauc_mrr_at_1_max": 0.3509,
128
+ "nauc_mrr_at_1_std": -0.028343,
129
+ "nauc_mrr_at_1_diff1": 0.4556,
130
+ "nauc_mrr_at_3_max": 0.343039,
131
+ "nauc_mrr_at_3_std": 0.003194,
132
+ "nauc_mrr_at_3_diff1": 0.395025,
133
+ "nauc_mrr_at_5_max": 0.338967,
134
+ "nauc_mrr_at_5_std": 0.002291,
135
+ "nauc_mrr_at_5_diff1": 0.389964,
136
+ "nauc_mrr_at_10_max": 0.337191,
137
+ "nauc_mrr_at_10_std": 0.004711,
138
+ "nauc_mrr_at_10_diff1": 0.388168,
139
+ "nauc_mrr_at_20_max": 0.338323,
140
+ "nauc_mrr_at_20_std": 0.006217,
141
+ "nauc_mrr_at_20_diff1": 0.385364,
142
+ "nauc_mrr_at_100_max": 0.337901,
143
+ "nauc_mrr_at_100_std": 0.007768,
144
+ "nauc_mrr_at_100_diff1": 0.384638,
145
+ "nauc_mrr_at_1000_max": 0.338161,
146
+ "nauc_mrr_at_1000_std": 0.00777,
147
+ "nauc_mrr_at_1000_diff1": 0.384956,
148
+ "main_score": 0.28629,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 75.08722925186157,
157
+ "kg_co2_emissions": null
158
+ }
mteb/CQADupstackMathematicaRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "90fceea13679c63fe563ded68f3b6f06e50061de",
3
+ "task_name": "CQADupstackMathematicaRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.1592,
9
+ "ndcg_at_3": 0.18727,
10
+ "ndcg_at_5": 0.20573,
11
+ "ndcg_at_10": 0.23092,
12
+ "ndcg_at_20": 0.24856,
13
+ "ndcg_at_100": 0.28661,
14
+ "ndcg_at_1000": 0.31839,
15
+ "map_at_1": 0.126,
16
+ "map_at_3": 0.1645,
17
+ "map_at_5": 0.17543,
18
+ "map_at_10": 0.18641,
19
+ "map_at_20": 0.19162,
20
+ "map_at_100": 0.19739,
21
+ "map_at_1000": 0.19864,
22
+ "recall_at_1": 0.126,
23
+ "recall_at_3": 0.20946,
24
+ "recall_at_5": 0.2564,
25
+ "recall_at_10": 0.32953,
26
+ "recall_at_20": 0.392,
27
+ "recall_at_100": 0.57855,
28
+ "recall_at_1000": 0.80748,
29
+ "precision_at_1": 0.1592,
30
+ "precision_at_3": 0.08955,
31
+ "precision_at_5": 0.06592,
32
+ "precision_at_10": 0.04366,
33
+ "precision_at_20": 0.02668,
34
+ "precision_at_100": 0.00832,
35
+ "precision_at_1000": 0.00124,
36
+ "mrr_at_1": 0.159204,
37
+ "mrr_at_3": 0.204187,
38
+ "mrr_at_5": 0.216563,
39
+ "mrr_at_10": 0.228361,
40
+ "mrr_at_20": 0.233146,
41
+ "mrr_at_100": 0.237804,
42
+ "mrr_at_1000": 0.238608,
43
+ "nauc_ndcg_at_1_max": 0.173331,
44
+ "nauc_ndcg_at_1_std": -0.070922,
45
+ "nauc_ndcg_at_1_diff1": 0.313574,
46
+ "nauc_ndcg_at_3_max": 0.149226,
47
+ "nauc_ndcg_at_3_std": -0.044527,
48
+ "nauc_ndcg_at_3_diff1": 0.231067,
49
+ "nauc_ndcg_at_5_max": 0.147868,
50
+ "nauc_ndcg_at_5_std": -0.033304,
51
+ "nauc_ndcg_at_5_diff1": 0.21667,
52
+ "nauc_ndcg_at_10_max": 0.136413,
53
+ "nauc_ndcg_at_10_std": -0.030496,
54
+ "nauc_ndcg_at_10_diff1": 0.19898,
55
+ "nauc_ndcg_at_20_max": 0.140685,
56
+ "nauc_ndcg_at_20_std": -0.020516,
57
+ "nauc_ndcg_at_20_diff1": 0.199934,
58
+ "nauc_ndcg_at_100_max": 0.159372,
59
+ "nauc_ndcg_at_100_std": 0.004912,
60
+ "nauc_ndcg_at_100_diff1": 0.205706,
61
+ "nauc_ndcg_at_1000_max": 0.158797,
62
+ "nauc_ndcg_at_1000_std": 0.000364,
63
+ "nauc_ndcg_at_1000_diff1": 0.203824,
64
+ "nauc_map_at_1_max": 0.176756,
65
+ "nauc_map_at_1_std": -0.061246,
66
+ "nauc_map_at_1_diff1": 0.294157,
67
+ "nauc_map_at_3_max": 0.151356,
68
+ "nauc_map_at_3_std": -0.048515,
69
+ "nauc_map_at_3_diff1": 0.241311,
70
+ "nauc_map_at_5_max": 0.151841,
71
+ "nauc_map_at_5_std": -0.042908,
72
+ "nauc_map_at_5_diff1": 0.23564,
73
+ "nauc_map_at_10_max": 0.146342,
74
+ "nauc_map_at_10_std": -0.040837,
75
+ "nauc_map_at_10_diff1": 0.226564,
76
+ "nauc_map_at_20_max": 0.148085,
77
+ "nauc_map_at_20_std": -0.036882,
78
+ "nauc_map_at_20_diff1": 0.226222,
79
+ "nauc_map_at_100_max": 0.151301,
80
+ "nauc_map_at_100_std": -0.032596,
81
+ "nauc_map_at_100_diff1": 0.226917,
82
+ "nauc_map_at_1000_max": 0.151171,
83
+ "nauc_map_at_1000_std": -0.032835,
84
+ "nauc_map_at_1000_diff1": 0.226596,
85
+ "nauc_recall_at_1_max": 0.176756,
86
+ "nauc_recall_at_1_std": -0.061246,
87
+ "nauc_recall_at_1_diff1": 0.294157,
88
+ "nauc_recall_at_3_max": 0.129382,
89
+ "nauc_recall_at_3_std": -0.026964,
90
+ "nauc_recall_at_3_diff1": 0.182064,
91
+ "nauc_recall_at_5_max": 0.126607,
92
+ "nauc_recall_at_5_std": -0.010177,
93
+ "nauc_recall_at_5_diff1": 0.159092,
94
+ "nauc_recall_at_10_max": 0.100506,
95
+ "nauc_recall_at_10_std": -0.007447,
96
+ "nauc_recall_at_10_diff1": 0.123114,
97
+ "nauc_recall_at_20_max": 0.109598,
98
+ "nauc_recall_at_20_std": 0.022768,
99
+ "nauc_recall_at_20_diff1": 0.126061,
100
+ "nauc_recall_at_100_max": 0.192576,
101
+ "nauc_recall_at_100_std": 0.141059,
102
+ "nauc_recall_at_100_diff1": 0.148286,
103
+ "nauc_recall_at_1000_max": 0.2255,
104
+ "nauc_recall_at_1000_std": 0.2101,
105
+ "nauc_recall_at_1000_diff1": 0.097776,
106
+ "nauc_precision_at_1_max": 0.173331,
107
+ "nauc_precision_at_1_std": -0.070922,
108
+ "nauc_precision_at_1_diff1": 0.313574,
109
+ "nauc_precision_at_3_max": 0.128165,
110
+ "nauc_precision_at_3_std": -0.037662,
111
+ "nauc_precision_at_3_diff1": 0.186901,
112
+ "nauc_precision_at_5_max": 0.135809,
113
+ "nauc_precision_at_5_std": -0.01395,
114
+ "nauc_precision_at_5_diff1": 0.16978,
115
+ "nauc_precision_at_10_max": 0.111158,
116
+ "nauc_precision_at_10_std": -0.011867,
117
+ "nauc_precision_at_10_diff1": 0.126989,
118
+ "nauc_precision_at_20_max": 0.111932,
119
+ "nauc_precision_at_20_std": 0.005621,
120
+ "nauc_precision_at_20_diff1": 0.115231,
121
+ "nauc_precision_at_100_max": 0.106532,
122
+ "nauc_precision_at_100_std": 0.058503,
123
+ "nauc_precision_at_100_diff1": 0.072954,
124
+ "nauc_precision_at_1000_max": 0.065429,
125
+ "nauc_precision_at_1000_std": -0.003839,
126
+ "nauc_precision_at_1000_diff1": 0.017772,
127
+ "nauc_mrr_at_1_max": 0.173331,
128
+ "nauc_mrr_at_1_std": -0.070922,
129
+ "nauc_mrr_at_1_diff1": 0.313574,
130
+ "nauc_mrr_at_3_max": 0.156386,
131
+ "nauc_mrr_at_3_std": -0.049645,
132
+ "nauc_mrr_at_3_diff1": 0.249522,
133
+ "nauc_mrr_at_5_max": 0.154441,
134
+ "nauc_mrr_at_5_std": -0.043804,
135
+ "nauc_mrr_at_5_diff1": 0.240661,
136
+ "nauc_mrr_at_10_max": 0.14987,
137
+ "nauc_mrr_at_10_std": -0.044188,
138
+ "nauc_mrr_at_10_diff1": 0.23307,
139
+ "nauc_mrr_at_20_max": 0.151533,
140
+ "nauc_mrr_at_20_std": -0.042259,
141
+ "nauc_mrr_at_20_diff1": 0.234266,
142
+ "nauc_mrr_at_100_max": 0.153064,
143
+ "nauc_mrr_at_100_std": -0.039756,
144
+ "nauc_mrr_at_100_diff1": 0.234561,
145
+ "nauc_mrr_at_1000_max": 0.152904,
146
+ "nauc_mrr_at_1000_std": -0.039874,
147
+ "nauc_mrr_at_1000_diff1": 0.234452,
148
+ "main_score": 0.23092,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 45.40352916717529,
157
+ "kg_co2_emissions": null
158
+ }
mteb/CQADupstackPhysicsRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4",
3
+ "task_name": "CQADupstackPhysicsRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.27238,
9
+ "ndcg_at_3": 0.32157,
10
+ "ndcg_at_5": 0.34044,
11
+ "ndcg_at_10": 0.37013,
12
+ "ndcg_at_20": 0.39337,
13
+ "ndcg_at_100": 0.42811,
14
+ "ndcg_at_1000": 0.45276,
15
+ "map_at_1": 0.22537,
16
+ "map_at_3": 0.2879,
17
+ "map_at_5": 0.3009,
18
+ "map_at_10": 0.31509,
19
+ "map_at_20": 0.32265,
20
+ "map_at_100": 0.32835,
21
+ "map_at_1000": 0.3296,
22
+ "recall_at_1": 0.22537,
23
+ "recall_at_3": 0.35122,
24
+ "recall_at_5": 0.39946,
25
+ "recall_at_10": 0.48803,
26
+ "recall_at_20": 0.56927,
27
+ "recall_at_100": 0.73288,
28
+ "recall_at_1000": 0.89725,
29
+ "precision_at_1": 0.27238,
30
+ "precision_at_3": 0.15271,
31
+ "precision_at_5": 0.1078,
32
+ "precision_at_10": 0.06756,
33
+ "precision_at_20": 0.04139,
34
+ "precision_at_100": 0.01153,
35
+ "precision_at_1000": 0.00154,
36
+ "mrr_at_1": 0.272377,
37
+ "mrr_at_3": 0.336702,
38
+ "mrr_at_5": 0.350369,
39
+ "mrr_at_10": 0.36334,
40
+ "mrr_at_20": 0.368893,
41
+ "mrr_at_100": 0.372666,
42
+ "mrr_at_1000": 0.373281,
43
+ "nauc_ndcg_at_1_max": 0.434963,
44
+ "nauc_ndcg_at_1_std": -0.016834,
45
+ "nauc_ndcg_at_1_diff1": 0.587191,
46
+ "nauc_ndcg_at_3_max": 0.376659,
47
+ "nauc_ndcg_at_3_std": -0.021128,
48
+ "nauc_ndcg_at_3_diff1": 0.498754,
49
+ "nauc_ndcg_at_5_max": 0.371104,
50
+ "nauc_ndcg_at_5_std": -0.006056,
51
+ "nauc_ndcg_at_5_diff1": 0.484702,
52
+ "nauc_ndcg_at_10_max": 0.371137,
53
+ "nauc_ndcg_at_10_std": 0.000755,
54
+ "nauc_ndcg_at_10_diff1": 0.472322,
55
+ "nauc_ndcg_at_20_max": 0.374454,
56
+ "nauc_ndcg_at_20_std": 0.009248,
57
+ "nauc_ndcg_at_20_diff1": 0.471307,
58
+ "nauc_ndcg_at_100_max": 0.385982,
59
+ "nauc_ndcg_at_100_std": 0.032502,
60
+ "nauc_ndcg_at_100_diff1": 0.473894,
61
+ "nauc_ndcg_at_1000_max": 0.39129,
62
+ "nauc_ndcg_at_1000_std": 0.036103,
63
+ "nauc_ndcg_at_1000_diff1": 0.475898,
64
+ "nauc_map_at_1_max": 0.387388,
65
+ "nauc_map_at_1_std": -0.063459,
66
+ "nauc_map_at_1_diff1": 0.595917,
67
+ "nauc_map_at_3_max": 0.368626,
68
+ "nauc_map_at_3_std": -0.041075,
69
+ "nauc_map_at_3_diff1": 0.520431,
70
+ "nauc_map_at_5_max": 0.369652,
71
+ "nauc_map_at_5_std": -0.028134,
72
+ "nauc_map_at_5_diff1": 0.512029,
73
+ "nauc_map_at_10_max": 0.373143,
74
+ "nauc_map_at_10_std": -0.021212,
75
+ "nauc_map_at_10_diff1": 0.505536,
76
+ "nauc_map_at_20_max": 0.375873,
77
+ "nauc_map_at_20_std": -0.017235,
78
+ "nauc_map_at_20_diff1": 0.50533,
79
+ "nauc_map_at_100_max": 0.378729,
80
+ "nauc_map_at_100_std": -0.013049,
81
+ "nauc_map_at_100_diff1": 0.505814,
82
+ "nauc_map_at_1000_max": 0.379222,
83
+ "nauc_map_at_1000_std": -0.012218,
84
+ "nauc_map_at_1000_diff1": 0.505796,
85
+ "nauc_recall_at_1_max": 0.387388,
86
+ "nauc_recall_at_1_std": -0.063459,
87
+ "nauc_recall_at_1_diff1": 0.595917,
88
+ "nauc_recall_at_3_max": 0.317313,
89
+ "nauc_recall_at_3_std": -0.038849,
90
+ "nauc_recall_at_3_diff1": 0.442818,
91
+ "nauc_recall_at_5_max": 0.31284,
92
+ "nauc_recall_at_5_std": 0.002032,
93
+ "nauc_recall_at_5_diff1": 0.407502,
94
+ "nauc_recall_at_10_max": 0.310181,
95
+ "nauc_recall_at_10_std": 0.029815,
96
+ "nauc_recall_at_10_diff1": 0.357645,
97
+ "nauc_recall_at_20_max": 0.305273,
98
+ "nauc_recall_at_20_std": 0.052793,
99
+ "nauc_recall_at_20_diff1": 0.339285,
100
+ "nauc_recall_at_100_max": 0.336613,
101
+ "nauc_recall_at_100_std": 0.198697,
102
+ "nauc_recall_at_100_diff1": 0.314998,
103
+ "nauc_recall_at_1000_max": 0.399223,
104
+ "nauc_recall_at_1000_std": 0.384869,
105
+ "nauc_recall_at_1000_diff1": 0.223792,
106
+ "nauc_precision_at_1_max": 0.434963,
107
+ "nauc_precision_at_1_std": -0.016834,
108
+ "nauc_precision_at_1_diff1": 0.587191,
109
+ "nauc_precision_at_3_max": 0.379844,
110
+ "nauc_precision_at_3_std": 0.058961,
111
+ "nauc_precision_at_3_diff1": 0.369786,
112
+ "nauc_precision_at_5_max": 0.367037,
113
+ "nauc_precision_at_5_std": 0.113331,
114
+ "nauc_precision_at_5_diff1": 0.304295,
115
+ "nauc_precision_at_10_max": 0.353315,
116
+ "nauc_precision_at_10_std": 0.159411,
117
+ "nauc_precision_at_10_diff1": 0.216986,
118
+ "nauc_precision_at_20_max": 0.321937,
119
+ "nauc_precision_at_20_std": 0.210608,
120
+ "nauc_precision_at_20_diff1": 0.15191,
121
+ "nauc_precision_at_100_max": 0.261556,
122
+ "nauc_precision_at_100_std": 0.287677,
123
+ "nauc_precision_at_100_diff1": 0.038748,
124
+ "nauc_precision_at_1000_max": 0.144137,
125
+ "nauc_precision_at_1000_std": 0.283117,
126
+ "nauc_precision_at_1000_diff1": -0.06849,
127
+ "nauc_mrr_at_1_max": 0.434963,
128
+ "nauc_mrr_at_1_std": -0.016834,
129
+ "nauc_mrr_at_1_diff1": 0.587191,
130
+ "nauc_mrr_at_3_max": 0.401226,
131
+ "nauc_mrr_at_3_std": -0.007172,
132
+ "nauc_mrr_at_3_diff1": 0.516348,
133
+ "nauc_mrr_at_5_max": 0.402557,
134
+ "nauc_mrr_at_5_std": 0.003495,
135
+ "nauc_mrr_at_5_diff1": 0.506254,
136
+ "nauc_mrr_at_10_max": 0.403164,
137
+ "nauc_mrr_at_10_std": 0.006847,
138
+ "nauc_mrr_at_10_diff1": 0.501766,
139
+ "nauc_mrr_at_20_max": 0.40341,
140
+ "nauc_mrr_at_20_std": 0.008421,
141
+ "nauc_mrr_at_20_diff1": 0.502319,
142
+ "nauc_mrr_at_100_max": 0.403681,
143
+ "nauc_mrr_at_100_std": 0.009655,
144
+ "nauc_mrr_at_100_diff1": 0.503195,
145
+ "nauc_mrr_at_1000_max": 0.403715,
146
+ "nauc_mrr_at_1000_std": 0.00956,
147
+ "nauc_mrr_at_1000_diff1": 0.503187,
148
+ "main_score": 0.37013,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 59.619898080825806,
157
+ "kg_co2_emissions": null
158
+ }
mteb/CQADupstackProgrammersRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "6184bc1440d2dbc7612be22b50686b8826d22b32",
3
+ "task_name": "CQADupstackProgrammersRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.25228,
9
+ "ndcg_at_3": 0.29072,
10
+ "ndcg_at_5": 0.31289,
11
+ "ndcg_at_10": 0.33489,
12
+ "ndcg_at_20": 0.35566,
13
+ "ndcg_at_100": 0.39297,
14
+ "ndcg_at_1000": 0.41996,
15
+ "map_at_1": 0.20288,
16
+ "map_at_3": 0.25911,
17
+ "map_at_5": 0.27423,
18
+ "map_at_10": 0.2851,
19
+ "map_at_20": 0.2914,
20
+ "map_at_100": 0.29736,
21
+ "map_at_1000": 0.29862,
22
+ "recall_at_1": 0.20288,
23
+ "recall_at_3": 0.31413,
24
+ "recall_at_5": 0.37333,
25
+ "recall_at_10": 0.43861,
26
+ "recall_at_20": 0.51217,
27
+ "recall_at_100": 0.6923,
28
+ "recall_at_1000": 0.87747,
29
+ "precision_at_1": 0.25228,
30
+ "precision_at_3": 0.13889,
31
+ "precision_at_5": 0.10137,
32
+ "precision_at_10": 0.06084,
33
+ "precision_at_20": 0.03687,
34
+ "precision_at_100": 0.01065,
35
+ "precision_at_1000": 0.00146,
36
+ "mrr_at_1": 0.252283,
37
+ "mrr_at_3": 0.307078,
38
+ "mrr_at_5": 0.323002,
39
+ "mrr_at_10": 0.332576,
40
+ "mrr_at_20": 0.337973,
41
+ "mrr_at_100": 0.342415,
42
+ "mrr_at_1000": 0.343064,
43
+ "nauc_ndcg_at_1_max": 0.401716,
44
+ "nauc_ndcg_at_1_std": 0.065067,
45
+ "nauc_ndcg_at_1_diff1": 0.392721,
46
+ "nauc_ndcg_at_3_max": 0.383229,
47
+ "nauc_ndcg_at_3_std": 0.065445,
48
+ "nauc_ndcg_at_3_diff1": 0.331411,
49
+ "nauc_ndcg_at_5_max": 0.388223,
50
+ "nauc_ndcg_at_5_std": 0.079156,
51
+ "nauc_ndcg_at_5_diff1": 0.321325,
52
+ "nauc_ndcg_at_10_max": 0.382528,
53
+ "nauc_ndcg_at_10_std": 0.076964,
54
+ "nauc_ndcg_at_10_diff1": 0.319019,
55
+ "nauc_ndcg_at_20_max": 0.38324,
56
+ "nauc_ndcg_at_20_std": 0.088949,
57
+ "nauc_ndcg_at_20_diff1": 0.315701,
58
+ "nauc_ndcg_at_100_max": 0.394976,
59
+ "nauc_ndcg_at_100_std": 0.112611,
60
+ "nauc_ndcg_at_100_diff1": 0.318071,
61
+ "nauc_ndcg_at_1000_max": 0.400048,
62
+ "nauc_ndcg_at_1000_std": 0.116156,
63
+ "nauc_ndcg_at_1000_diff1": 0.320789,
64
+ "nauc_map_at_1_max": 0.357305,
65
+ "nauc_map_at_1_std": 0.015761,
66
+ "nauc_map_at_1_diff1": 0.384366,
67
+ "nauc_map_at_3_max": 0.373442,
68
+ "nauc_map_at_3_std": 0.047477,
69
+ "nauc_map_at_3_diff1": 0.342786,
70
+ "nauc_map_at_5_max": 0.37963,
71
+ "nauc_map_at_5_std": 0.058431,
72
+ "nauc_map_at_5_diff1": 0.336109,
73
+ "nauc_map_at_10_max": 0.379757,
74
+ "nauc_map_at_10_std": 0.059797,
75
+ "nauc_map_at_10_diff1": 0.336136,
76
+ "nauc_map_at_20_max": 0.381347,
77
+ "nauc_map_at_20_std": 0.064297,
78
+ "nauc_map_at_20_diff1": 0.335546,
79
+ "nauc_map_at_100_max": 0.383476,
80
+ "nauc_map_at_100_std": 0.069208,
81
+ "nauc_map_at_100_diff1": 0.335143,
82
+ "nauc_map_at_1000_max": 0.383756,
83
+ "nauc_map_at_1000_std": 0.069628,
84
+ "nauc_map_at_1000_diff1": 0.335194,
85
+ "nauc_recall_at_1_max": 0.357305,
86
+ "nauc_recall_at_1_std": 0.015761,
87
+ "nauc_recall_at_1_diff1": 0.384366,
88
+ "nauc_recall_at_3_max": 0.358924,
89
+ "nauc_recall_at_3_std": 0.056564,
90
+ "nauc_recall_at_3_diff1": 0.303315,
91
+ "nauc_recall_at_5_max": 0.367283,
92
+ "nauc_recall_at_5_std": 0.09274,
93
+ "nauc_recall_at_5_diff1": 0.273828,
94
+ "nauc_recall_at_10_max": 0.35053,
95
+ "nauc_recall_at_10_std": 0.089863,
96
+ "nauc_recall_at_10_diff1": 0.262794,
97
+ "nauc_recall_at_20_max": 0.339605,
98
+ "nauc_recall_at_20_std": 0.126648,
99
+ "nauc_recall_at_20_diff1": 0.242244,
100
+ "nauc_recall_at_100_max": 0.380486,
101
+ "nauc_recall_at_100_std": 0.249646,
102
+ "nauc_recall_at_100_diff1": 0.240527,
103
+ "nauc_recall_at_1000_max": 0.487529,
104
+ "nauc_recall_at_1000_std": 0.466888,
105
+ "nauc_recall_at_1000_diff1": 0.234965,
106
+ "nauc_precision_at_1_max": 0.401716,
107
+ "nauc_precision_at_1_std": 0.065067,
108
+ "nauc_precision_at_1_diff1": 0.392721,
109
+ "nauc_precision_at_3_max": 0.403259,
110
+ "nauc_precision_at_3_std": 0.142162,
111
+ "nauc_precision_at_3_diff1": 0.269316,
112
+ "nauc_precision_at_5_max": 0.404385,
113
+ "nauc_precision_at_5_std": 0.178013,
114
+ "nauc_precision_at_5_diff1": 0.224482,
115
+ "nauc_precision_at_10_max": 0.367242,
116
+ "nauc_precision_at_10_std": 0.17353,
117
+ "nauc_precision_at_10_diff1": 0.183739,
118
+ "nauc_precision_at_20_max": 0.327196,
119
+ "nauc_precision_at_20_std": 0.202914,
120
+ "nauc_precision_at_20_diff1": 0.143319,
121
+ "nauc_precision_at_100_max": 0.234403,
122
+ "nauc_precision_at_100_std": 0.246467,
123
+ "nauc_precision_at_100_diff1": 0.036827,
124
+ "nauc_precision_at_1000_max": 0.077956,
125
+ "nauc_precision_at_1000_std": 0.148892,
126
+ "nauc_precision_at_1000_diff1": -0.060821,
127
+ "nauc_mrr_at_1_max": 0.401716,
128
+ "nauc_mrr_at_1_std": 0.065067,
129
+ "nauc_mrr_at_1_diff1": 0.392721,
130
+ "nauc_mrr_at_3_max": 0.401914,
131
+ "nauc_mrr_at_3_std": 0.080097,
132
+ "nauc_mrr_at_3_diff1": 0.351381,
133
+ "nauc_mrr_at_5_max": 0.401255,
134
+ "nauc_mrr_at_5_std": 0.088183,
135
+ "nauc_mrr_at_5_diff1": 0.342425,
136
+ "nauc_mrr_at_10_max": 0.395882,
137
+ "nauc_mrr_at_10_std": 0.085347,
138
+ "nauc_mrr_at_10_diff1": 0.338442,
139
+ "nauc_mrr_at_20_max": 0.394937,
140
+ "nauc_mrr_at_20_std": 0.086579,
141
+ "nauc_mrr_at_20_diff1": 0.337513,
142
+ "nauc_mrr_at_100_max": 0.397282,
143
+ "nauc_mrr_at_100_std": 0.088992,
144
+ "nauc_mrr_at_100_diff1": 0.339029,
145
+ "nauc_mrr_at_1000_max": 0.397251,
146
+ "nauc_mrr_at_1000_std": 0.088846,
147
+ "nauc_mrr_at_1000_diff1": 0.339196,
148
+ "main_score": 0.33489,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 57.37495994567871,
157
+ "kg_co2_emissions": null
158
+ }
mteb/CQADupstackStatsRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a",
3
+ "task_name": "CQADupstackStatsRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.21166,
9
+ "ndcg_at_3": 0.24697,
10
+ "ndcg_at_5": 0.26206,
11
+ "ndcg_at_10": 0.28144,
12
+ "ndcg_at_20": 0.29877,
13
+ "ndcg_at_100": 0.32798,
14
+ "ndcg_at_1000": 0.35119,
15
+ "map_at_1": 0.18672,
16
+ "map_at_3": 0.2269,
17
+ "map_at_5": 0.23653,
18
+ "map_at_10": 0.24469,
19
+ "map_at_20": 0.2497,
20
+ "map_at_100": 0.25401,
21
+ "map_at_1000": 0.25481,
22
+ "recall_at_1": 0.18672,
23
+ "recall_at_3": 0.27173,
24
+ "recall_at_5": 0.30953,
25
+ "recall_at_10": 0.3688,
26
+ "recall_at_20": 0.43444,
27
+ "recall_at_100": 0.58217,
28
+ "recall_at_1000": 0.75725,
29
+ "precision_at_1": 0.21166,
30
+ "precision_at_3": 0.10685,
31
+ "precision_at_5": 0.07485,
32
+ "precision_at_10": 0.04479,
33
+ "precision_at_20": 0.02669,
34
+ "precision_at_100": 0.00729,
35
+ "precision_at_1000": 0.001,
36
+ "mrr_at_1": 0.211656,
37
+ "mrr_at_3": 0.255112,
38
+ "mrr_at_5": 0.263471,
39
+ "mrr_at_10": 0.272333,
40
+ "mrr_at_20": 0.276509,
41
+ "mrr_at_100": 0.280587,
42
+ "mrr_at_1000": 0.28122,
43
+ "nauc_ndcg_at_1_max": 0.265288,
44
+ "nauc_ndcg_at_1_std": 0.001032,
45
+ "nauc_ndcg_at_1_diff1": 0.50655,
46
+ "nauc_ndcg_at_3_max": 0.259172,
47
+ "nauc_ndcg_at_3_std": 0.0391,
48
+ "nauc_ndcg_at_3_diff1": 0.464349,
49
+ "nauc_ndcg_at_5_max": 0.253671,
50
+ "nauc_ndcg_at_5_std": 0.057907,
51
+ "nauc_ndcg_at_5_diff1": 0.453087,
52
+ "nauc_ndcg_at_10_max": 0.252675,
53
+ "nauc_ndcg_at_10_std": 0.068572,
54
+ "nauc_ndcg_at_10_diff1": 0.431656,
55
+ "nauc_ndcg_at_20_max": 0.252661,
56
+ "nauc_ndcg_at_20_std": 0.085497,
57
+ "nauc_ndcg_at_20_diff1": 0.417731,
58
+ "nauc_ndcg_at_100_max": 0.257578,
59
+ "nauc_ndcg_at_100_std": 0.09627,
60
+ "nauc_ndcg_at_100_diff1": 0.409369,
61
+ "nauc_ndcg_at_1000_max": 0.26693,
62
+ "nauc_ndcg_at_1000_std": 0.100071,
63
+ "nauc_ndcg_at_1000_diff1": 0.41458,
64
+ "nauc_map_at_1_max": 0.280534,
65
+ "nauc_map_at_1_std": -0.03001,
66
+ "nauc_map_at_1_diff1": 0.538926,
67
+ "nauc_map_at_3_max": 0.264728,
68
+ "nauc_map_at_3_std": 0.014403,
69
+ "nauc_map_at_3_diff1": 0.48676,
70
+ "nauc_map_at_5_max": 0.262414,
71
+ "nauc_map_at_5_std": 0.030925,
72
+ "nauc_map_at_5_diff1": 0.479445,
73
+ "nauc_map_at_10_max": 0.262277,
74
+ "nauc_map_at_10_std": 0.037764,
75
+ "nauc_map_at_10_diff1": 0.470099,
76
+ "nauc_map_at_20_max": 0.262636,
77
+ "nauc_map_at_20_std": 0.04315,
78
+ "nauc_map_at_20_diff1": 0.465854,
79
+ "nauc_map_at_100_max": 0.263191,
80
+ "nauc_map_at_100_std": 0.045135,
81
+ "nauc_map_at_100_diff1": 0.463709,
82
+ "nauc_map_at_1000_max": 0.263864,
83
+ "nauc_map_at_1000_std": 0.045534,
84
+ "nauc_map_at_1000_diff1": 0.463922,
85
+ "nauc_recall_at_1_max": 0.280534,
86
+ "nauc_recall_at_1_std": -0.03001,
87
+ "nauc_recall_at_1_diff1": 0.538926,
88
+ "nauc_recall_at_3_max": 0.236962,
89
+ "nauc_recall_at_3_std": 0.054231,
90
+ "nauc_recall_at_3_diff1": 0.425405,
91
+ "nauc_recall_at_5_max": 0.224468,
92
+ "nauc_recall_at_5_std": 0.098421,
93
+ "nauc_recall_at_5_diff1": 0.39783,
94
+ "nauc_recall_at_10_max": 0.218317,
95
+ "nauc_recall_at_10_std": 0.128712,
96
+ "nauc_recall_at_10_diff1": 0.337508,
97
+ "nauc_recall_at_20_max": 0.213863,
98
+ "nauc_recall_at_20_std": 0.183436,
99
+ "nauc_recall_at_20_diff1": 0.285907,
100
+ "nauc_recall_at_100_max": 0.227284,
101
+ "nauc_recall_at_100_std": 0.243125,
102
+ "nauc_recall_at_100_diff1": 0.233571,
103
+ "nauc_recall_at_1000_max": 0.277326,
104
+ "nauc_recall_at_1000_std": 0.328397,
105
+ "nauc_recall_at_1000_diff1": 0.20892,
106
+ "nauc_precision_at_1_max": 0.265288,
107
+ "nauc_precision_at_1_std": 0.001032,
108
+ "nauc_precision_at_1_diff1": 0.50655,
109
+ "nauc_precision_at_3_max": 0.253493,
110
+ "nauc_precision_at_3_std": 0.115181,
111
+ "nauc_precision_at_3_diff1": 0.392497,
112
+ "nauc_precision_at_5_max": 0.234647,
113
+ "nauc_precision_at_5_std": 0.187151,
114
+ "nauc_precision_at_5_diff1": 0.338881,
115
+ "nauc_precision_at_10_max": 0.237545,
116
+ "nauc_precision_at_10_std": 0.213893,
117
+ "nauc_precision_at_10_diff1": 0.285351,
118
+ "nauc_precision_at_20_max": 0.230892,
119
+ "nauc_precision_at_20_std": 0.262866,
120
+ "nauc_precision_at_20_diff1": 0.210742,
121
+ "nauc_precision_at_100_max": 0.232832,
122
+ "nauc_precision_at_100_std": 0.294046,
123
+ "nauc_precision_at_100_diff1": 0.140878,
124
+ "nauc_precision_at_1000_max": 0.239556,
125
+ "nauc_precision_at_1000_std": 0.263343,
126
+ "nauc_precision_at_1000_diff1": 0.042393,
127
+ "nauc_mrr_at_1_max": 0.265288,
128
+ "nauc_mrr_at_1_std": 0.001032,
129
+ "nauc_mrr_at_1_diff1": 0.50655,
130
+ "nauc_mrr_at_3_max": 0.257994,
131
+ "nauc_mrr_at_3_std": 0.04097,
132
+ "nauc_mrr_at_3_diff1": 0.456167,
133
+ "nauc_mrr_at_5_max": 0.255945,
134
+ "nauc_mrr_at_5_std": 0.049945,
135
+ "nauc_mrr_at_5_diff1": 0.450037,
136
+ "nauc_mrr_at_10_max": 0.256333,
137
+ "nauc_mrr_at_10_std": 0.054663,
138
+ "nauc_mrr_at_10_diff1": 0.440708,
139
+ "nauc_mrr_at_20_max": 0.255786,
140
+ "nauc_mrr_at_20_std": 0.058257,
141
+ "nauc_mrr_at_20_diff1": 0.437883,
142
+ "nauc_mrr_at_100_max": 0.256685,
143
+ "nauc_mrr_at_100_std": 0.05889,
144
+ "nauc_mrr_at_100_diff1": 0.436408,
145
+ "nauc_mrr_at_1000_max": 0.256891,
146
+ "nauc_mrr_at_1000_std": 0.059147,
147
+ "nauc_mrr_at_1000_diff1": 0.436692,
148
+ "main_score": 0.28144,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 86.29045724868774,
157
+ "kg_co2_emissions": null
158
+ }
mteb/CQADupstackTexRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "46989137a86843e03a6195de44b09deda022eec7",
3
+ "task_name": "CQADupstackTexRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.14178,
9
+ "ndcg_at_3": 0.17057,
10
+ "ndcg_at_5": 0.1856,
11
+ "ndcg_at_10": 0.20471,
12
+ "ndcg_at_20": 0.22044,
13
+ "ndcg_at_100": 0.2503,
14
+ "ndcg_at_1000": 0.28231,
15
+ "map_at_1": 0.11553,
16
+ "map_at_3": 0.15025,
17
+ "map_at_5": 0.15972,
18
+ "map_at_10": 0.16841,
19
+ "map_at_20": 0.17291,
20
+ "map_at_100": 0.17711,
21
+ "map_at_1000": 0.17832,
22
+ "recall_at_1": 0.11553,
23
+ "recall_at_3": 0.18958,
24
+ "recall_at_5": 0.22824,
25
+ "recall_at_10": 0.28569,
26
+ "recall_at_20": 0.34504,
27
+ "recall_at_100": 0.49491,
28
+ "recall_at_1000": 0.72826,
29
+ "precision_at_1": 0.14178,
30
+ "precision_at_3": 0.08098,
31
+ "precision_at_5": 0.05953,
32
+ "precision_at_10": 0.03823,
33
+ "precision_at_20": 0.02326,
34
+ "precision_at_100": 0.0071,
35
+ "precision_at_1000": 0.00114,
36
+ "mrr_at_1": 0.141776,
37
+ "mrr_at_3": 0.179456,
38
+ "mrr_at_5": 0.189694,
39
+ "mrr_at_10": 0.198676,
40
+ "mrr_at_20": 0.203388,
41
+ "mrr_at_100": 0.207227,
42
+ "mrr_at_1000": 0.208095,
43
+ "nauc_ndcg_at_1_max": 0.273731,
44
+ "nauc_ndcg_at_1_std": -0.000848,
45
+ "nauc_ndcg_at_1_diff1": 0.402551,
46
+ "nauc_ndcg_at_3_max": 0.254625,
47
+ "nauc_ndcg_at_3_std": 0.005476,
48
+ "nauc_ndcg_at_3_diff1": 0.343983,
49
+ "nauc_ndcg_at_5_max": 0.251083,
50
+ "nauc_ndcg_at_5_std": 0.011607,
51
+ "nauc_ndcg_at_5_diff1": 0.328852,
52
+ "nauc_ndcg_at_10_max": 0.24845,
53
+ "nauc_ndcg_at_10_std": 0.024927,
54
+ "nauc_ndcg_at_10_diff1": 0.316519,
55
+ "nauc_ndcg_at_20_max": 0.251553,
56
+ "nauc_ndcg_at_20_std": 0.038963,
57
+ "nauc_ndcg_at_20_diff1": 0.306872,
58
+ "nauc_ndcg_at_100_max": 0.25391,
59
+ "nauc_ndcg_at_100_std": 0.049506,
60
+ "nauc_ndcg_at_100_diff1": 0.300219,
61
+ "nauc_ndcg_at_1000_max": 0.25447,
62
+ "nauc_ndcg_at_1000_std": 0.057198,
63
+ "nauc_ndcg_at_1000_diff1": 0.297107,
64
+ "nauc_map_at_1_max": 0.24317,
65
+ "nauc_map_at_1_std": -0.013325,
66
+ "nauc_map_at_1_diff1": 0.430924,
67
+ "nauc_map_at_3_max": 0.245786,
68
+ "nauc_map_at_3_std": -0.001987,
69
+ "nauc_map_at_3_diff1": 0.367577,
70
+ "nauc_map_at_5_max": 0.245709,
71
+ "nauc_map_at_5_std": 0.001682,
72
+ "nauc_map_at_5_diff1": 0.356493,
73
+ "nauc_map_at_10_max": 0.24637,
74
+ "nauc_map_at_10_std": 0.009096,
75
+ "nauc_map_at_10_diff1": 0.348743,
76
+ "nauc_map_at_20_max": 0.247864,
77
+ "nauc_map_at_20_std": 0.013651,
78
+ "nauc_map_at_20_diff1": 0.345685,
79
+ "nauc_map_at_100_max": 0.248458,
80
+ "nauc_map_at_100_std": 0.015959,
81
+ "nauc_map_at_100_diff1": 0.344828,
82
+ "nauc_map_at_1000_max": 0.248587,
83
+ "nauc_map_at_1000_std": 0.016378,
84
+ "nauc_map_at_1000_diff1": 0.344495,
85
+ "nauc_recall_at_1_max": 0.24317,
86
+ "nauc_recall_at_1_std": -0.013325,
87
+ "nauc_recall_at_1_diff1": 0.430924,
88
+ "nauc_recall_at_3_max": 0.239063,
89
+ "nauc_recall_at_3_std": 0.009005,
90
+ "nauc_recall_at_3_diff1": 0.313126,
91
+ "nauc_recall_at_5_max": 0.228969,
92
+ "nauc_recall_at_5_std": 0.018906,
93
+ "nauc_recall_at_5_diff1": 0.274421,
94
+ "nauc_recall_at_10_max": 0.224291,
95
+ "nauc_recall_at_10_std": 0.047312,
96
+ "nauc_recall_at_10_diff1": 0.244958,
97
+ "nauc_recall_at_20_max": 0.229385,
98
+ "nauc_recall_at_20_std": 0.086611,
99
+ "nauc_recall_at_20_diff1": 0.212233,
100
+ "nauc_recall_at_100_max": 0.237652,
101
+ "nauc_recall_at_100_std": 0.122032,
102
+ "nauc_recall_at_100_diff1": 0.180425,
103
+ "nauc_recall_at_1000_max": 0.232788,
104
+ "nauc_recall_at_1000_std": 0.208114,
105
+ "nauc_recall_at_1000_diff1": 0.120909,
106
+ "nauc_precision_at_1_max": 0.273731,
107
+ "nauc_precision_at_1_std": -0.000848,
108
+ "nauc_precision_at_1_diff1": 0.402551,
109
+ "nauc_precision_at_3_max": 0.275815,
110
+ "nauc_precision_at_3_std": 0.023241,
111
+ "nauc_precision_at_3_diff1": 0.281068,
112
+ "nauc_precision_at_5_max": 0.271773,
113
+ "nauc_precision_at_5_std": 0.037777,
114
+ "nauc_precision_at_5_diff1": 0.24271,
115
+ "nauc_precision_at_10_max": 0.264364,
116
+ "nauc_precision_at_10_std": 0.073815,
117
+ "nauc_precision_at_10_diff1": 0.210453,
118
+ "nauc_precision_at_20_max": 0.270537,
119
+ "nauc_precision_at_20_std": 0.109882,
120
+ "nauc_precision_at_20_diff1": 0.180007,
121
+ "nauc_precision_at_100_max": 0.247712,
122
+ "nauc_precision_at_100_std": 0.141652,
123
+ "nauc_precision_at_100_diff1": 0.110556,
124
+ "nauc_precision_at_1000_max": 0.195539,
125
+ "nauc_precision_at_1000_std": 0.152915,
126
+ "nauc_precision_at_1000_diff1": 0.008828,
127
+ "nauc_mrr_at_1_max": 0.273731,
128
+ "nauc_mrr_at_1_std": -0.000848,
129
+ "nauc_mrr_at_1_diff1": 0.402551,
130
+ "nauc_mrr_at_3_max": 0.264306,
131
+ "nauc_mrr_at_3_std": 0.01339,
132
+ "nauc_mrr_at_3_diff1": 0.344013,
133
+ "nauc_mrr_at_5_max": 0.262192,
134
+ "nauc_mrr_at_5_std": 0.017274,
135
+ "nauc_mrr_at_5_diff1": 0.333294,
136
+ "nauc_mrr_at_10_max": 0.261096,
137
+ "nauc_mrr_at_10_std": 0.022398,
138
+ "nauc_mrr_at_10_diff1": 0.32811,
139
+ "nauc_mrr_at_20_max": 0.262244,
140
+ "nauc_mrr_at_20_std": 0.026409,
141
+ "nauc_mrr_at_20_diff1": 0.32533,
142
+ "nauc_mrr_at_100_max": 0.262496,
143
+ "nauc_mrr_at_100_std": 0.027576,
144
+ "nauc_mrr_at_100_diff1": 0.324713,
145
+ "nauc_mrr_at_1000_max": 0.262422,
146
+ "nauc_mrr_at_1000_std": 0.027625,
147
+ "nauc_mrr_at_1000_diff1": 0.324694,
148
+ "main_score": 0.20471,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 198.2237286567688,
157
+ "kg_co2_emissions": null
158
+ }
mteb/CQADupstackUnixRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53",
3
+ "task_name": "CQADupstackUnixRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.24347,
9
+ "ndcg_at_3": 0.2739,
10
+ "ndcg_at_5": 0.29021,
11
+ "ndcg_at_10": 0.3119,
12
+ "ndcg_at_20": 0.33183,
13
+ "ndcg_at_100": 0.36465,
14
+ "ndcg_at_1000": 0.39555,
15
+ "map_at_1": 0.20973,
16
+ "map_at_3": 0.25159,
17
+ "map_at_5": 0.2616,
18
+ "map_at_10": 0.27125,
19
+ "map_at_20": 0.27681,
20
+ "map_at_100": 0.28175,
21
+ "map_at_1000": 0.28303,
22
+ "recall_at_1": 0.20973,
23
+ "recall_at_3": 0.29951,
24
+ "recall_at_5": 0.34009,
25
+ "recall_at_10": 0.40363,
26
+ "recall_at_20": 0.47728,
27
+ "recall_at_100": 0.63718,
28
+ "recall_at_1000": 0.8574,
29
+ "precision_at_1": 0.24347,
30
+ "precision_at_3": 0.11971,
31
+ "precision_at_5": 0.08321,
32
+ "precision_at_10": 0.05028,
33
+ "precision_at_20": 0.03032,
34
+ "precision_at_100": 0.00867,
35
+ "precision_at_1000": 0.00126,
36
+ "mrr_at_1": 0.24347,
37
+ "mrr_at_3": 0.285914,
38
+ "mrr_at_5": 0.297015,
39
+ "mrr_at_10": 0.306006,
40
+ "mrr_at_20": 0.311716,
41
+ "mrr_at_100": 0.315759,
42
+ "mrr_at_1000": 0.316592,
43
+ "nauc_ndcg_at_1_max": 0.366074,
44
+ "nauc_ndcg_at_1_std": -0.002938,
45
+ "nauc_ndcg_at_1_diff1": 0.511682,
46
+ "nauc_ndcg_at_3_max": 0.355856,
47
+ "nauc_ndcg_at_3_std": 0.033005,
48
+ "nauc_ndcg_at_3_diff1": 0.455243,
49
+ "nauc_ndcg_at_5_max": 0.342306,
50
+ "nauc_ndcg_at_5_std": 0.024515,
51
+ "nauc_ndcg_at_5_diff1": 0.434606,
52
+ "nauc_ndcg_at_10_max": 0.340939,
53
+ "nauc_ndcg_at_10_std": 0.036514,
54
+ "nauc_ndcg_at_10_diff1": 0.421622,
55
+ "nauc_ndcg_at_20_max": 0.339306,
56
+ "nauc_ndcg_at_20_std": 0.041006,
57
+ "nauc_ndcg_at_20_diff1": 0.418551,
58
+ "nauc_ndcg_at_100_max": 0.339351,
59
+ "nauc_ndcg_at_100_std": 0.058532,
60
+ "nauc_ndcg_at_100_diff1": 0.413425,
61
+ "nauc_ndcg_at_1000_max": 0.345893,
62
+ "nauc_ndcg_at_1000_std": 0.063455,
63
+ "nauc_ndcg_at_1000_diff1": 0.41893,
64
+ "nauc_map_at_1_max": 0.364348,
65
+ "nauc_map_at_1_std": -0.01631,
66
+ "nauc_map_at_1_diff1": 0.529856,
67
+ "nauc_map_at_3_max": 0.356602,
68
+ "nauc_map_at_3_std": 0.017935,
69
+ "nauc_map_at_3_diff1": 0.473642,
70
+ "nauc_map_at_5_max": 0.349516,
71
+ "nauc_map_at_5_std": 0.015001,
72
+ "nauc_map_at_5_diff1": 0.461318,
73
+ "nauc_map_at_10_max": 0.350392,
74
+ "nauc_map_at_10_std": 0.022047,
75
+ "nauc_map_at_10_diff1": 0.455432,
76
+ "nauc_map_at_20_max": 0.3504,
77
+ "nauc_map_at_20_std": 0.023646,
78
+ "nauc_map_at_20_diff1": 0.454156,
79
+ "nauc_map_at_100_max": 0.349922,
80
+ "nauc_map_at_100_std": 0.026192,
81
+ "nauc_map_at_100_diff1": 0.4536,
82
+ "nauc_map_at_1000_max": 0.350113,
83
+ "nauc_map_at_1000_std": 0.026625,
84
+ "nauc_map_at_1000_diff1": 0.453732,
85
+ "nauc_recall_at_1_max": 0.364348,
86
+ "nauc_recall_at_1_std": -0.01631,
87
+ "nauc_recall_at_1_diff1": 0.529856,
88
+ "nauc_recall_at_3_max": 0.343817,
89
+ "nauc_recall_at_3_std": 0.054702,
90
+ "nauc_recall_at_3_diff1": 0.414897,
91
+ "nauc_recall_at_5_max": 0.312585,
92
+ "nauc_recall_at_5_std": 0.040548,
93
+ "nauc_recall_at_5_diff1": 0.367072,
94
+ "nauc_recall_at_10_max": 0.302046,
95
+ "nauc_recall_at_10_std": 0.067438,
96
+ "nauc_recall_at_10_diff1": 0.328025,
97
+ "nauc_recall_at_20_max": 0.289,
98
+ "nauc_recall_at_20_std": 0.079,
99
+ "nauc_recall_at_20_diff1": 0.309696,
100
+ "nauc_recall_at_100_max": 0.279702,
101
+ "nauc_recall_at_100_std": 0.178914,
102
+ "nauc_recall_at_100_diff1": 0.259407,
103
+ "nauc_recall_at_1000_max": 0.358183,
104
+ "nauc_recall_at_1000_std": 0.36287,
105
+ "nauc_recall_at_1000_diff1": 0.232753,
106
+ "nauc_precision_at_1_max": 0.366074,
107
+ "nauc_precision_at_1_std": -0.002938,
108
+ "nauc_precision_at_1_diff1": 0.511682,
109
+ "nauc_precision_at_3_max": 0.342215,
110
+ "nauc_precision_at_3_std": 0.079457,
111
+ "nauc_precision_at_3_diff1": 0.379603,
112
+ "nauc_precision_at_5_max": 0.308996,
113
+ "nauc_precision_at_5_std": 0.065465,
114
+ "nauc_precision_at_5_diff1": 0.327939,
115
+ "nauc_precision_at_10_max": 0.293998,
116
+ "nauc_precision_at_10_std": 0.10973,
117
+ "nauc_precision_at_10_diff1": 0.268081,
118
+ "nauc_precision_at_20_max": 0.265752,
119
+ "nauc_precision_at_20_std": 0.123159,
120
+ "nauc_precision_at_20_diff1": 0.234389,
121
+ "nauc_precision_at_100_max": 0.178161,
122
+ "nauc_precision_at_100_std": 0.176852,
123
+ "nauc_precision_at_100_diff1": 0.089218,
124
+ "nauc_precision_at_1000_max": 0.055458,
125
+ "nauc_precision_at_1000_std": 0.142567,
126
+ "nauc_precision_at_1000_diff1": -0.047612,
127
+ "nauc_mrr_at_1_max": 0.366074,
128
+ "nauc_mrr_at_1_std": -0.002938,
129
+ "nauc_mrr_at_1_diff1": 0.511682,
130
+ "nauc_mrr_at_3_max": 0.354753,
131
+ "nauc_mrr_at_3_std": 0.024454,
132
+ "nauc_mrr_at_3_diff1": 0.456803,
133
+ "nauc_mrr_at_5_max": 0.349086,
134
+ "nauc_mrr_at_5_std": 0.021245,
135
+ "nauc_mrr_at_5_diff1": 0.444838,
136
+ "nauc_mrr_at_10_max": 0.346014,
137
+ "nauc_mrr_at_10_std": 0.024307,
138
+ "nauc_mrr_at_10_diff1": 0.440129,
139
+ "nauc_mrr_at_20_max": 0.345043,
140
+ "nauc_mrr_at_20_std": 0.026122,
141
+ "nauc_mrr_at_20_diff1": 0.439379,
142
+ "nauc_mrr_at_100_max": 0.345434,
143
+ "nauc_mrr_at_100_std": 0.028055,
144
+ "nauc_mrr_at_100_diff1": 0.439384,
145
+ "nauc_mrr_at_1000_max": 0.34558,
146
+ "nauc_mrr_at_1000_std": 0.028083,
147
+ "nauc_mrr_at_1000_diff1": 0.439607,
148
+ "main_score": 0.3119,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 103.79455494880676,
157
+ "kg_co2_emissions": null
158
+ }
mteb/CQADupstackWebmastersRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "160c094312a0e1facb97e55eeddb698c0abe3571",
3
+ "task_name": "CQADupstackWebmastersRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.22925,
9
+ "ndcg_at_3": 0.2741,
10
+ "ndcg_at_5": 0.2929,
11
+ "ndcg_at_10": 0.31232,
12
+ "ndcg_at_20": 0.33154,
13
+ "ndcg_at_100": 0.36606,
14
+ "ndcg_at_1000": 0.39806,
15
+ "map_at_1": 0.19137,
16
+ "map_at_3": 0.23966,
17
+ "map_at_5": 0.25369,
18
+ "map_at_10": 0.26413,
19
+ "map_at_20": 0.27109,
20
+ "map_at_100": 0.27726,
21
+ "map_at_1000": 0.27925,
22
+ "recall_at_1": 0.19137,
23
+ "recall_at_3": 0.29464,
24
+ "recall_at_5": 0.34544,
25
+ "recall_at_10": 0.40694,
26
+ "recall_at_20": 0.4791,
27
+ "recall_at_100": 0.65313,
28
+ "recall_at_1000": 0.86383,
29
+ "precision_at_1": 0.22925,
30
+ "precision_at_3": 0.13109,
31
+ "precision_at_5": 0.09684,
32
+ "precision_at_10": 0.06047,
33
+ "precision_at_20": 0.03903,
34
+ "precision_at_100": 0.01235,
35
+ "precision_at_1000": 0.00207,
36
+ "mrr_at_1": 0.229249,
37
+ "mrr_at_3": 0.278656,
38
+ "mrr_at_5": 0.291601,
39
+ "mrr_at_10": 0.300793,
40
+ "mrr_at_20": 0.30649,
41
+ "mrr_at_100": 0.310727,
42
+ "mrr_at_1000": 0.311487,
43
+ "nauc_ndcg_at_1_max": 0.278397,
44
+ "nauc_ndcg_at_1_std": 0.074006,
45
+ "nauc_ndcg_at_1_diff1": 0.513375,
46
+ "nauc_ndcg_at_3_max": 0.277863,
47
+ "nauc_ndcg_at_3_std": 0.105389,
48
+ "nauc_ndcg_at_3_diff1": 0.48273,
49
+ "nauc_ndcg_at_5_max": 0.262458,
50
+ "nauc_ndcg_at_5_std": 0.109897,
51
+ "nauc_ndcg_at_5_diff1": 0.465795,
52
+ "nauc_ndcg_at_10_max": 0.259559,
53
+ "nauc_ndcg_at_10_std": 0.112909,
54
+ "nauc_ndcg_at_10_diff1": 0.458508,
55
+ "nauc_ndcg_at_20_max": 0.259624,
56
+ "nauc_ndcg_at_20_std": 0.115503,
57
+ "nauc_ndcg_at_20_diff1": 0.448082,
58
+ "nauc_ndcg_at_100_max": 0.257458,
59
+ "nauc_ndcg_at_100_std": 0.133477,
60
+ "nauc_ndcg_at_100_diff1": 0.437838,
61
+ "nauc_ndcg_at_1000_max": 0.264933,
62
+ "nauc_ndcg_at_1000_std": 0.139622,
63
+ "nauc_ndcg_at_1000_diff1": 0.448966,
64
+ "nauc_map_at_1_max": 0.264657,
65
+ "nauc_map_at_1_std": 0.035551,
66
+ "nauc_map_at_1_diff1": 0.536002,
67
+ "nauc_map_at_3_max": 0.271682,
68
+ "nauc_map_at_3_std": 0.077661,
69
+ "nauc_map_at_3_diff1": 0.495551,
70
+ "nauc_map_at_5_max": 0.26631,
71
+ "nauc_map_at_5_std": 0.080968,
72
+ "nauc_map_at_5_diff1": 0.481763,
73
+ "nauc_map_at_10_max": 0.263891,
74
+ "nauc_map_at_10_std": 0.084954,
75
+ "nauc_map_at_10_diff1": 0.477934,
76
+ "nauc_map_at_20_max": 0.263787,
77
+ "nauc_map_at_20_std": 0.088554,
78
+ "nauc_map_at_20_diff1": 0.475802,
79
+ "nauc_map_at_100_max": 0.263258,
80
+ "nauc_map_at_100_std": 0.095955,
81
+ "nauc_map_at_100_diff1": 0.475416,
82
+ "nauc_map_at_1000_max": 0.263256,
83
+ "nauc_map_at_1000_std": 0.09769,
84
+ "nauc_map_at_1000_diff1": 0.475749,
85
+ "nauc_recall_at_1_max": 0.264657,
86
+ "nauc_recall_at_1_std": 0.035551,
87
+ "nauc_recall_at_1_diff1": 0.536002,
88
+ "nauc_recall_at_3_max": 0.263286,
89
+ "nauc_recall_at_3_std": 0.110583,
90
+ "nauc_recall_at_3_diff1": 0.454661,
91
+ "nauc_recall_at_5_max": 0.241204,
92
+ "nauc_recall_at_5_std": 0.125151,
93
+ "nauc_recall_at_5_diff1": 0.411045,
94
+ "nauc_recall_at_10_max": 0.240086,
95
+ "nauc_recall_at_10_std": 0.137833,
96
+ "nauc_recall_at_10_diff1": 0.388961,
97
+ "nauc_recall_at_20_max": 0.231511,
98
+ "nauc_recall_at_20_std": 0.151779,
99
+ "nauc_recall_at_20_diff1": 0.35641,
100
+ "nauc_recall_at_100_max": 0.205584,
101
+ "nauc_recall_at_100_std": 0.242135,
102
+ "nauc_recall_at_100_diff1": 0.267027,
103
+ "nauc_recall_at_1000_max": 0.259238,
104
+ "nauc_recall_at_1000_std": 0.406363,
105
+ "nauc_recall_at_1000_diff1": 0.28158,
106
+ "nauc_precision_at_1_max": 0.278397,
107
+ "nauc_precision_at_1_std": 0.074006,
108
+ "nauc_precision_at_1_diff1": 0.513375,
109
+ "nauc_precision_at_3_max": 0.254205,
110
+ "nauc_precision_at_3_std": 0.150633,
111
+ "nauc_precision_at_3_diff1": 0.384094,
112
+ "nauc_precision_at_5_max": 0.20648,
113
+ "nauc_precision_at_5_std": 0.158803,
114
+ "nauc_precision_at_5_diff1": 0.310389,
115
+ "nauc_precision_at_10_max": 0.175023,
116
+ "nauc_precision_at_10_std": 0.188265,
117
+ "nauc_precision_at_10_diff1": 0.268628,
118
+ "nauc_precision_at_20_max": 0.160018,
119
+ "nauc_precision_at_20_std": 0.229377,
120
+ "nauc_precision_at_20_diff1": 0.216145,
121
+ "nauc_precision_at_100_max": 0.039446,
122
+ "nauc_precision_at_100_std": 0.309237,
123
+ "nauc_precision_at_100_diff1": 0.081114,
124
+ "nauc_precision_at_1000_max": -0.078246,
125
+ "nauc_precision_at_1000_std": 0.226591,
126
+ "nauc_precision_at_1000_diff1": -0.029668,
127
+ "nauc_mrr_at_1_max": 0.278397,
128
+ "nauc_mrr_at_1_std": 0.074006,
129
+ "nauc_mrr_at_1_diff1": 0.513375,
130
+ "nauc_mrr_at_3_max": 0.271664,
131
+ "nauc_mrr_at_3_std": 0.105992,
132
+ "nauc_mrr_at_3_diff1": 0.475073,
133
+ "nauc_mrr_at_5_max": 0.264996,
134
+ "nauc_mrr_at_5_std": 0.109831,
135
+ "nauc_mrr_at_5_diff1": 0.463956,
136
+ "nauc_mrr_at_10_max": 0.265579,
137
+ "nauc_mrr_at_10_std": 0.111244,
138
+ "nauc_mrr_at_10_diff1": 0.464022,
139
+ "nauc_mrr_at_20_max": 0.26659,
140
+ "nauc_mrr_at_20_std": 0.111591,
141
+ "nauc_mrr_at_20_diff1": 0.462309,
142
+ "nauc_mrr_at_100_max": 0.265909,
143
+ "nauc_mrr_at_100_std": 0.113546,
144
+ "nauc_mrr_at_100_diff1": 0.461344,
145
+ "nauc_mrr_at_1000_max": 0.266114,
146
+ "nauc_mrr_at_1000_std": 0.113636,
147
+ "nauc_mrr_at_1000_diff1": 0.461867,
148
+ "main_score": 0.31232,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 23.93776535987854,
157
+ "kg_co2_emissions": null
158
+ }
mteb/CQADupstackWordpressRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4",
3
+ "task_name": "CQADupstackWordpressRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.18299,
9
+ "ndcg_at_3": 0.22947,
10
+ "ndcg_at_5": 0.24289,
11
+ "ndcg_at_10": 0.26195,
12
+ "ndcg_at_20": 0.28138,
13
+ "ndcg_at_100": 0.31194,
14
+ "ndcg_at_1000": 0.34148,
15
+ "map_at_1": 0.16833,
16
+ "map_at_3": 0.211,
17
+ "map_at_5": 0.2197,
18
+ "map_at_10": 0.22788,
19
+ "map_at_20": 0.23321,
20
+ "map_at_100": 0.2373,
21
+ "map_at_1000": 0.23838,
22
+ "recall_at_1": 0.16833,
23
+ "recall_at_3": 0.26334,
24
+ "recall_at_5": 0.29575,
25
+ "recall_at_10": 0.3512,
26
+ "recall_at_20": 0.42532,
27
+ "recall_at_100": 0.58598,
28
+ "recall_at_1000": 0.80809,
29
+ "precision_at_1": 0.18299,
30
+ "precision_at_3": 0.0992,
31
+ "precision_at_5": 0.06765,
32
+ "precision_at_10": 0.04067,
33
+ "precision_at_20": 0.02486,
34
+ "precision_at_100": 0.00701,
35
+ "precision_at_1000": 0.00105,
36
+ "mrr_at_1": 0.182994,
37
+ "mrr_at_3": 0.227665,
38
+ "mrr_at_5": 0.235059,
39
+ "mrr_at_10": 0.243175,
40
+ "mrr_at_20": 0.248732,
41
+ "mrr_at_100": 0.252759,
42
+ "mrr_at_1000": 0.253625,
43
+ "nauc_ndcg_at_1_max": 0.290492,
44
+ "nauc_ndcg_at_1_std": -0.037429,
45
+ "nauc_ndcg_at_1_diff1": 0.427622,
46
+ "nauc_ndcg_at_3_max": 0.243326,
47
+ "nauc_ndcg_at_3_std": -0.02747,
48
+ "nauc_ndcg_at_3_diff1": 0.340168,
49
+ "nauc_ndcg_at_5_max": 0.238634,
50
+ "nauc_ndcg_at_5_std": -0.02659,
51
+ "nauc_ndcg_at_5_diff1": 0.330531,
52
+ "nauc_ndcg_at_10_max": 0.247575,
53
+ "nauc_ndcg_at_10_std": -0.017911,
54
+ "nauc_ndcg_at_10_diff1": 0.320757,
55
+ "nauc_ndcg_at_20_max": 0.245705,
56
+ "nauc_ndcg_at_20_std": 0.004907,
57
+ "nauc_ndcg_at_20_diff1": 0.320415,
58
+ "nauc_ndcg_at_100_max": 0.240886,
59
+ "nauc_ndcg_at_100_std": 0.023006,
60
+ "nauc_ndcg_at_100_diff1": 0.31633,
61
+ "nauc_ndcg_at_1000_max": 0.242627,
62
+ "nauc_ndcg_at_1000_std": 0.029307,
63
+ "nauc_ndcg_at_1000_diff1": 0.318627,
64
+ "nauc_map_at_1_max": 0.265847,
65
+ "nauc_map_at_1_std": -0.039069,
66
+ "nauc_map_at_1_diff1": 0.430649,
67
+ "nauc_map_at_3_max": 0.244435,
68
+ "nauc_map_at_3_std": -0.027471,
69
+ "nauc_map_at_3_diff1": 0.358874,
70
+ "nauc_map_at_5_max": 0.244507,
71
+ "nauc_map_at_5_std": -0.02695,
72
+ "nauc_map_at_5_diff1": 0.352913,
73
+ "nauc_map_at_10_max": 0.248881,
74
+ "nauc_map_at_10_std": -0.023212,
75
+ "nauc_map_at_10_diff1": 0.348928,
76
+ "nauc_map_at_20_max": 0.248784,
77
+ "nauc_map_at_20_std": -0.016656,
78
+ "nauc_map_at_20_diff1": 0.348707,
79
+ "nauc_map_at_100_max": 0.248009,
80
+ "nauc_map_at_100_std": -0.014712,
81
+ "nauc_map_at_100_diff1": 0.348352,
82
+ "nauc_map_at_1000_max": 0.248034,
83
+ "nauc_map_at_1000_std": -0.014321,
84
+ "nauc_map_at_1000_diff1": 0.348601,
85
+ "nauc_recall_at_1_max": 0.265847,
86
+ "nauc_recall_at_1_std": -0.039069,
87
+ "nauc_recall_at_1_diff1": 0.430649,
88
+ "nauc_recall_at_3_max": 0.213529,
89
+ "nauc_recall_at_3_std": -0.025467,
90
+ "nauc_recall_at_3_diff1": 0.286084,
91
+ "nauc_recall_at_5_max": 0.205815,
92
+ "nauc_recall_at_5_std": -0.025665,
93
+ "nauc_recall_at_5_diff1": 0.265638,
94
+ "nauc_recall_at_10_max": 0.223811,
95
+ "nauc_recall_at_10_std": -0.002912,
96
+ "nauc_recall_at_10_diff1": 0.236385,
97
+ "nauc_recall_at_20_max": 0.211582,
98
+ "nauc_recall_at_20_std": 0.067677,
99
+ "nauc_recall_at_20_diff1": 0.234745,
100
+ "nauc_recall_at_100_max": 0.182712,
101
+ "nauc_recall_at_100_std": 0.17472,
102
+ "nauc_recall_at_100_diff1": 0.201213,
103
+ "nauc_recall_at_1000_max": 0.174778,
104
+ "nauc_recall_at_1000_std": 0.340317,
105
+ "nauc_recall_at_1000_diff1": 0.125604,
106
+ "nauc_precision_at_1_max": 0.290492,
107
+ "nauc_precision_at_1_std": -0.037429,
108
+ "nauc_precision_at_1_diff1": 0.427622,
109
+ "nauc_precision_at_3_max": 0.227398,
110
+ "nauc_precision_at_3_std": -0.028379,
111
+ "nauc_precision_at_3_diff1": 0.266137,
112
+ "nauc_precision_at_5_max": 0.230675,
113
+ "nauc_precision_at_5_std": -0.017098,
114
+ "nauc_precision_at_5_diff1": 0.241378,
115
+ "nauc_precision_at_10_max": 0.243564,
116
+ "nauc_precision_at_10_std": 0.005694,
117
+ "nauc_precision_at_10_diff1": 0.209227,
118
+ "nauc_precision_at_20_max": 0.239849,
119
+ "nauc_precision_at_20_std": 0.089154,
120
+ "nauc_precision_at_20_diff1": 0.210395,
121
+ "nauc_precision_at_100_max": 0.187676,
122
+ "nauc_precision_at_100_std": 0.159012,
123
+ "nauc_precision_at_100_diff1": 0.145642,
124
+ "nauc_precision_at_1000_max": 0.040961,
125
+ "nauc_precision_at_1000_std": 0.149076,
126
+ "nauc_precision_at_1000_diff1": 0.029816,
127
+ "nauc_mrr_at_1_max": 0.290492,
128
+ "nauc_mrr_at_1_std": -0.037429,
129
+ "nauc_mrr_at_1_diff1": 0.427622,
130
+ "nauc_mrr_at_3_max": 0.261245,
131
+ "nauc_mrr_at_3_std": -0.029722,
132
+ "nauc_mrr_at_3_diff1": 0.363549,
133
+ "nauc_mrr_at_5_max": 0.259592,
134
+ "nauc_mrr_at_5_std": -0.028622,
135
+ "nauc_mrr_at_5_diff1": 0.357855,
136
+ "nauc_mrr_at_10_max": 0.264184,
137
+ "nauc_mrr_at_10_std": -0.024333,
138
+ "nauc_mrr_at_10_diff1": 0.353717,
139
+ "nauc_mrr_at_20_max": 0.264148,
140
+ "nauc_mrr_at_20_std": -0.017082,
141
+ "nauc_mrr_at_20_diff1": 0.353438,
142
+ "nauc_mrr_at_100_max": 0.263672,
143
+ "nauc_mrr_at_100_std": -0.015025,
144
+ "nauc_mrr_at_100_diff1": 0.353221,
145
+ "nauc_mrr_at_1000_max": 0.263556,
146
+ "nauc_mrr_at_1000_std": -0.014899,
147
+ "nauc_mrr_at_1000_diff1": 0.353607,
148
+ "main_score": 0.26195,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 105.96243500709534,
157
+ "kg_co2_emissions": null
158
+ }
mteb/ClimateFEVER.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380",
3
+ "task_name": "ClimateFEVER",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.21042,
9
+ "ndcg_at_3": 0.18007,
10
+ "ndcg_at_5": 0.19365,
11
+ "ndcg_at_10": 0.22698,
12
+ "ndcg_at_20": 0.25245,
13
+ "ndcg_at_100": 0.29216,
14
+ "ndcg_at_1000": 0.32348,
15
+ "map_at_1": 0.09732,
16
+ "map_at_3": 0.13098,
17
+ "map_at_5": 0.14246,
18
+ "map_at_10": 0.15738,
19
+ "map_at_20": 0.16557,
20
+ "map_at_100": 0.17281,
21
+ "map_at_1000": 0.17442,
22
+ "recall_at_1": 0.09732,
23
+ "recall_at_3": 0.16374,
24
+ "recall_at_5": 0.20321,
25
+ "recall_at_10": 0.27863,
26
+ "recall_at_20": 0.35101,
27
+ "recall_at_100": 0.50446,
28
+ "recall_at_1000": 0.68127,
29
+ "precision_at_1": 0.21042,
30
+ "precision_at_3": 0.12942,
31
+ "precision_at_5": 0.09967,
32
+ "precision_at_10": 0.07114,
33
+ "precision_at_20": 0.04635,
34
+ "precision_at_100": 0.014,
35
+ "precision_at_1000": 0.00197,
36
+ "mrr_at_1": 0.210423,
37
+ "mrr_at_3": 0.276439,
38
+ "mrr_at_5": 0.293377,
39
+ "mrr_at_10": 0.309231,
40
+ "mrr_at_20": 0.315901,
41
+ "mrr_at_100": 0.319553,
42
+ "mrr_at_1000": 0.320057,
43
+ "nauc_ndcg_at_1_max": 0.253668,
44
+ "nauc_ndcg_at_1_std": 0.041982,
45
+ "nauc_ndcg_at_1_diff1": 0.263596,
46
+ "nauc_ndcg_at_3_max": 0.303375,
47
+ "nauc_ndcg_at_3_std": 0.071602,
48
+ "nauc_ndcg_at_3_diff1": 0.231569,
49
+ "nauc_ndcg_at_5_max": 0.31039,
50
+ "nauc_ndcg_at_5_std": 0.08631,
51
+ "nauc_ndcg_at_5_diff1": 0.234282,
52
+ "nauc_ndcg_at_10_max": 0.321631,
53
+ "nauc_ndcg_at_10_std": 0.128647,
54
+ "nauc_ndcg_at_10_diff1": 0.224955,
55
+ "nauc_ndcg_at_20_max": 0.329699,
56
+ "nauc_ndcg_at_20_std": 0.159557,
57
+ "nauc_ndcg_at_20_diff1": 0.228594,
58
+ "nauc_ndcg_at_100_max": 0.345065,
59
+ "nauc_ndcg_at_100_std": 0.20812,
60
+ "nauc_ndcg_at_100_diff1": 0.230539,
61
+ "nauc_ndcg_at_1000_max": 0.351358,
62
+ "nauc_ndcg_at_1000_std": 0.224793,
63
+ "nauc_ndcg_at_1000_diff1": 0.228996,
64
+ "nauc_map_at_1_max": 0.281958,
65
+ "nauc_map_at_1_std": -0.021727,
66
+ "nauc_map_at_1_diff1": 0.313643,
67
+ "nauc_map_at_3_max": 0.298457,
68
+ "nauc_map_at_3_std": 0.022626,
69
+ "nauc_map_at_3_diff1": 0.257536,
70
+ "nauc_map_at_5_max": 0.304463,
71
+ "nauc_map_at_5_std": 0.038367,
72
+ "nauc_map_at_5_diff1": 0.257217,
73
+ "nauc_map_at_10_max": 0.31309,
74
+ "nauc_map_at_10_std": 0.066086,
75
+ "nauc_map_at_10_diff1": 0.251932,
76
+ "nauc_map_at_20_max": 0.31663,
77
+ "nauc_map_at_20_std": 0.07943,
78
+ "nauc_map_at_20_diff1": 0.253261,
79
+ "nauc_map_at_100_max": 0.320402,
80
+ "nauc_map_at_100_std": 0.092723,
81
+ "nauc_map_at_100_diff1": 0.252213,
82
+ "nauc_map_at_1000_max": 0.320828,
83
+ "nauc_map_at_1000_std": 0.094498,
84
+ "nauc_map_at_1000_diff1": 0.252002,
85
+ "nauc_recall_at_1_max": 0.281958,
86
+ "nauc_recall_at_1_std": -0.021727,
87
+ "nauc_recall_at_1_diff1": 0.313643,
88
+ "nauc_recall_at_3_max": 0.311157,
89
+ "nauc_recall_at_3_std": 0.060219,
90
+ "nauc_recall_at_3_diff1": 0.210486,
91
+ "nauc_recall_at_5_max": 0.297119,
92
+ "nauc_recall_at_5_std": 0.096385,
93
+ "nauc_recall_at_5_diff1": 0.194008,
94
+ "nauc_recall_at_10_max": 0.29758,
95
+ "nauc_recall_at_10_std": 0.167827,
96
+ "nauc_recall_at_10_diff1": 0.168048,
97
+ "nauc_recall_at_20_max": 0.302094,
98
+ "nauc_recall_at_20_std": 0.227934,
99
+ "nauc_recall_at_20_diff1": 0.167479,
100
+ "nauc_recall_at_100_max": 0.325903,
101
+ "nauc_recall_at_100_std": 0.356132,
102
+ "nauc_recall_at_100_diff1": 0.163043,
103
+ "nauc_recall_at_1000_max": 0.365571,
104
+ "nauc_recall_at_1000_std": 0.465573,
105
+ "nauc_recall_at_1000_diff1": 0.146935,
106
+ "nauc_precision_at_1_max": 0.253668,
107
+ "nauc_precision_at_1_std": 0.041982,
108
+ "nauc_precision_at_1_diff1": 0.263596,
109
+ "nauc_precision_at_3_max": 0.302638,
110
+ "nauc_precision_at_3_std": 0.164883,
111
+ "nauc_precision_at_3_diff1": 0.165101,
112
+ "nauc_precision_at_5_max": 0.299424,
113
+ "nauc_precision_at_5_std": 0.204229,
114
+ "nauc_precision_at_5_diff1": 0.152494,
115
+ "nauc_precision_at_10_max": 0.287718,
116
+ "nauc_precision_at_10_std": 0.276706,
117
+ "nauc_precision_at_10_diff1": 0.105829,
118
+ "nauc_precision_at_20_max": 0.276563,
119
+ "nauc_precision_at_20_std": 0.329983,
120
+ "nauc_precision_at_20_diff1": 0.10746,
121
+ "nauc_precision_at_100_max": 0.258573,
122
+ "nauc_precision_at_100_std": 0.432574,
123
+ "nauc_precision_at_100_diff1": 0.079839,
124
+ "nauc_precision_at_1000_max": 0.199198,
125
+ "nauc_precision_at_1000_std": 0.440233,
126
+ "nauc_precision_at_1000_diff1": 0.024789,
127
+ "nauc_mrr_at_1_max": 0.253668,
128
+ "nauc_mrr_at_1_std": 0.041982,
129
+ "nauc_mrr_at_1_diff1": 0.263596,
130
+ "nauc_mrr_at_3_max": 0.298863,
131
+ "nauc_mrr_at_3_std": 0.106849,
132
+ "nauc_mrr_at_3_diff1": 0.230179,
133
+ "nauc_mrr_at_5_max": 0.29585,
134
+ "nauc_mrr_at_5_std": 0.117695,
135
+ "nauc_mrr_at_5_diff1": 0.225112,
136
+ "nauc_mrr_at_10_max": 0.297507,
137
+ "nauc_mrr_at_10_std": 0.128053,
138
+ "nauc_mrr_at_10_diff1": 0.222167,
139
+ "nauc_mrr_at_20_max": 0.298771,
140
+ "nauc_mrr_at_20_std": 0.13317,
141
+ "nauc_mrr_at_20_diff1": 0.223668,
142
+ "nauc_mrr_at_100_max": 0.299334,
143
+ "nauc_mrr_at_100_std": 0.133076,
144
+ "nauc_mrr_at_100_diff1": 0.224883,
145
+ "nauc_mrr_at_1000_max": 0.299249,
146
+ "nauc_mrr_at_1000_std": 0.132749,
147
+ "nauc_mrr_at_1000_diff1": 0.224871,
148
+ "main_score": 0.22698,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 5528.608217954636,
157
+ "kg_co2_emissions": null
158
+ }
mteb/DBPedia.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659",
3
+ "task_name": "DBPedia",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.41375,
9
+ "ndcg_at_3": 0.32494,
10
+ "ndcg_at_5": 0.29511,
11
+ "ndcg_at_10": 0.27559,
12
+ "ndcg_at_20": 0.26821,
13
+ "ndcg_at_100": 0.29407,
14
+ "ndcg_at_1000": 0.34784,
15
+ "map_at_1": 0.07455,
16
+ "map_at_3": 0.10611,
17
+ "map_at_5": 0.1174,
18
+ "map_at_10": 0.13131,
19
+ "map_at_20": 0.14319,
20
+ "map_at_100": 0.1607,
21
+ "map_at_1000": 0.16731,
22
+ "recall_at_1": 0.07455,
23
+ "recall_at_3": 0.11476,
24
+ "recall_at_5": 0.13383,
25
+ "recall_at_10": 0.16742,
26
+ "recall_at_20": 0.2134,
27
+ "recall_at_100": 0.32125,
28
+ "recall_at_1000": 0.49776,
29
+ "precision_at_1": 0.51,
30
+ "precision_at_3": 0.3475,
31
+ "precision_at_5": 0.2705,
32
+ "precision_at_10": 0.19575,
33
+ "precision_at_20": 0.13988,
34
+ "precision_at_100": 0.0535,
35
+ "precision_at_1000": 0.01072,
36
+ "mrr_at_1": 0.51,
37
+ "mrr_at_3": 0.57875,
38
+ "mrr_at_5": 0.585375,
39
+ "mrr_at_10": 0.591851,
40
+ "mrr_at_20": 0.596647,
41
+ "mrr_at_100": 0.598812,
42
+ "mrr_at_1000": 0.599078,
43
+ "nauc_ndcg_at_1_max": 0.423916,
44
+ "nauc_ndcg_at_1_std": 0.155682,
45
+ "nauc_ndcg_at_1_diff1": 0.431251,
46
+ "nauc_ndcg_at_3_max": 0.438604,
47
+ "nauc_ndcg_at_3_std": 0.193398,
48
+ "nauc_ndcg_at_3_diff1": 0.308724,
49
+ "nauc_ndcg_at_5_max": 0.414941,
50
+ "nauc_ndcg_at_5_std": 0.182644,
51
+ "nauc_ndcg_at_5_diff1": 0.287275,
52
+ "nauc_ndcg_at_10_max": 0.390035,
53
+ "nauc_ndcg_at_10_std": 0.176473,
54
+ "nauc_ndcg_at_10_diff1": 0.279835,
55
+ "nauc_ndcg_at_20_max": 0.357069,
56
+ "nauc_ndcg_at_20_std": 0.165987,
57
+ "nauc_ndcg_at_20_diff1": 0.282564,
58
+ "nauc_ndcg_at_100_max": 0.343485,
59
+ "nauc_ndcg_at_100_std": 0.205251,
60
+ "nauc_ndcg_at_100_diff1": 0.258367,
61
+ "nauc_ndcg_at_1000_max": 0.380072,
62
+ "nauc_ndcg_at_1000_std": 0.265801,
63
+ "nauc_ndcg_at_1000_diff1": 0.257738,
64
+ "nauc_map_at_1_max": 0.118856,
65
+ "nauc_map_at_1_std": -0.093654,
66
+ "nauc_map_at_1_diff1": 0.383665,
67
+ "nauc_map_at_3_max": 0.163439,
68
+ "nauc_map_at_3_std": -0.050284,
69
+ "nauc_map_at_3_diff1": 0.323527,
70
+ "nauc_map_at_5_max": 0.17271,
71
+ "nauc_map_at_5_std": -0.034506,
72
+ "nauc_map_at_5_diff1": 0.305797,
73
+ "nauc_map_at_10_max": 0.187527,
74
+ "nauc_map_at_10_std": 0.003105,
75
+ "nauc_map_at_10_diff1": 0.281637,
76
+ "nauc_map_at_20_max": 0.206161,
77
+ "nauc_map_at_20_std": 0.043977,
78
+ "nauc_map_at_20_diff1": 0.26506,
79
+ "nauc_map_at_100_max": 0.236987,
80
+ "nauc_map_at_100_std": 0.115381,
81
+ "nauc_map_at_100_diff1": 0.233655,
82
+ "nauc_map_at_1000_max": 0.24553,
83
+ "nauc_map_at_1000_std": 0.132969,
84
+ "nauc_map_at_1000_diff1": 0.230466,
85
+ "nauc_recall_at_1_max": 0.118856,
86
+ "nauc_recall_at_1_std": -0.093654,
87
+ "nauc_recall_at_1_diff1": 0.383665,
88
+ "nauc_recall_at_3_max": 0.144271,
89
+ "nauc_recall_at_3_std": -0.053232,
90
+ "nauc_recall_at_3_diff1": 0.297432,
91
+ "nauc_recall_at_5_max": 0.150162,
92
+ "nauc_recall_at_5_std": -0.041646,
93
+ "nauc_recall_at_5_diff1": 0.283241,
94
+ "nauc_recall_at_10_max": 0.155396,
95
+ "nauc_recall_at_10_std": -0.00623,
96
+ "nauc_recall_at_10_diff1": 0.246545,
97
+ "nauc_recall_at_20_max": 0.155934,
98
+ "nauc_recall_at_20_std": 0.03434,
99
+ "nauc_recall_at_20_diff1": 0.217566,
100
+ "nauc_recall_at_100_max": 0.183441,
101
+ "nauc_recall_at_100_std": 0.179899,
102
+ "nauc_recall_at_100_diff1": 0.154327,
103
+ "nauc_recall_at_1000_max": 0.253257,
104
+ "nauc_recall_at_1000_std": 0.313636,
105
+ "nauc_recall_at_1000_diff1": 0.147442,
106
+ "nauc_precision_at_1_max": 0.451052,
107
+ "nauc_precision_at_1_std": 0.189451,
108
+ "nauc_precision_at_1_diff1": 0.462767,
109
+ "nauc_precision_at_3_max": 0.441379,
110
+ "nauc_precision_at_3_std": 0.265232,
111
+ "nauc_precision_at_3_diff1": 0.183643,
112
+ "nauc_precision_at_5_max": 0.412908,
113
+ "nauc_precision_at_5_std": 0.282232,
114
+ "nauc_precision_at_5_diff1": 0.105613,
115
+ "nauc_precision_at_10_max": 0.389052,
116
+ "nauc_precision_at_10_std": 0.366196,
117
+ "nauc_precision_at_10_diff1": 0.030044,
118
+ "nauc_precision_at_20_max": 0.372606,
119
+ "nauc_precision_at_20_std": 0.450747,
120
+ "nauc_precision_at_20_diff1": -0.014095,
121
+ "nauc_precision_at_100_max": 0.315374,
122
+ "nauc_precision_at_100_std": 0.498572,
123
+ "nauc_precision_at_100_diff1": -0.082395,
124
+ "nauc_precision_at_1000_max": 0.199796,
125
+ "nauc_precision_at_1000_std": 0.386212,
126
+ "nauc_precision_at_1000_diff1": -0.119969,
127
+ "nauc_mrr_at_1_max": 0.451052,
128
+ "nauc_mrr_at_1_std": 0.189451,
129
+ "nauc_mrr_at_1_diff1": 0.462767,
130
+ "nauc_mrr_at_3_max": 0.483251,
131
+ "nauc_mrr_at_3_std": 0.253881,
132
+ "nauc_mrr_at_3_diff1": 0.434542,
133
+ "nauc_mrr_at_5_max": 0.483313,
134
+ "nauc_mrr_at_5_std": 0.251794,
135
+ "nauc_mrr_at_5_diff1": 0.435443,
136
+ "nauc_mrr_at_10_max": 0.480391,
137
+ "nauc_mrr_at_10_std": 0.249537,
138
+ "nauc_mrr_at_10_diff1": 0.431642,
139
+ "nauc_mrr_at_20_max": 0.479453,
140
+ "nauc_mrr_at_20_std": 0.251964,
141
+ "nauc_mrr_at_20_diff1": 0.431792,
142
+ "nauc_mrr_at_100_max": 0.4801,
143
+ "nauc_mrr_at_100_std": 0.252416,
144
+ "nauc_mrr_at_100_diff1": 0.433313,
145
+ "nauc_mrr_at_1000_max": 0.479913,
146
+ "nauc_mrr_at_1000_std": 0.252061,
147
+ "nauc_mrr_at_1000_diff1": 0.433367,
148
+ "main_score": 0.27559,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 3010.4430978298187,
157
+ "kg_co2_emissions": null
158
+ }
mteb/EmotionClassification.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37",
3
+ "task_name": "EmotionClassification",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "accuracy": 0.437,
9
+ "f1": 0.39358,
10
+ "f1_weighted": 0.45983,
11
+ "scores_per_experiment": [
12
+ {
13
+ "accuracy": 0.4555,
14
+ "f1": 0.399212,
15
+ "f1_weighted": 0.484673
16
+ },
17
+ {
18
+ "accuracy": 0.432,
19
+ "f1": 0.394225,
20
+ "f1_weighted": 0.447477
21
+ },
22
+ {
23
+ "accuracy": 0.415,
24
+ "f1": 0.379312,
25
+ "f1_weighted": 0.440126
26
+ },
27
+ {
28
+ "accuracy": 0.413,
29
+ "f1": 0.374642,
30
+ "f1_weighted": 0.440663
31
+ },
32
+ {
33
+ "accuracy": 0.456,
34
+ "f1": 0.41365,
35
+ "f1_weighted": 0.47793
36
+ },
37
+ {
38
+ "accuracy": 0.4375,
39
+ "f1": 0.390637,
40
+ "f1_weighted": 0.45846
41
+ },
42
+ {
43
+ "accuracy": 0.447,
44
+ "f1": 0.402599,
45
+ "f1_weighted": 0.470503
46
+ },
47
+ {
48
+ "accuracy": 0.4135,
49
+ "f1": 0.376153,
50
+ "f1_weighted": 0.428264
51
+ },
52
+ {
53
+ "accuracy": 0.461,
54
+ "f1": 0.414495,
55
+ "f1_weighted": 0.481941
56
+ },
57
+ {
58
+ "accuracy": 0.4395,
59
+ "f1": 0.390877,
60
+ "f1_weighted": 0.468265
61
+ }
62
+ ],
63
+ "main_score": 0.437,
64
+ "hf_subset": "default",
65
+ "languages": [
66
+ "eng-Latn"
67
+ ]
68
+ }
69
+ ]
70
+ },
71
+ "evaluation_time": 7.992861747741699,
72
+ "kg_co2_emissions": null
73
+ }
mteb/FEVER.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12",
3
+ "task_name": "FEVER",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.52625,
9
+ "ndcg_at_3": 0.61126,
10
+ "ndcg_at_5": 0.63731,
11
+ "ndcg_at_10": 0.65721,
12
+ "ndcg_at_20": 0.66718,
13
+ "ndcg_at_100": 0.67996,
14
+ "ndcg_at_1000": 0.68554,
15
+ "map_at_1": 0.48757,
16
+ "map_at_3": 0.57408,
17
+ "map_at_5": 0.58912,
18
+ "map_at_10": 0.59765,
19
+ "map_at_20": 0.60053,
20
+ "map_at_100": 0.60235,
21
+ "map_at_1000": 0.60258,
22
+ "recall_at_1": 0.48757,
23
+ "recall_at_3": 0.67712,
24
+ "recall_at_5": 0.74102,
25
+ "recall_at_10": 0.80181,
26
+ "recall_at_20": 0.83964,
27
+ "recall_at_100": 0.90629,
28
+ "recall_at_1000": 0.9479,
29
+ "precision_at_1": 0.52625,
30
+ "precision_at_3": 0.24482,
31
+ "precision_at_5": 0.16112,
32
+ "precision_at_10": 0.08738,
33
+ "precision_at_20": 0.0459,
34
+ "precision_at_100": 0.00997,
35
+ "precision_at_1000": 0.00105,
36
+ "mrr_at_1": 0.526253,
37
+ "mrr_at_3": 0.615912,
38
+ "mrr_at_5": 0.631183,
39
+ "mrr_at_10": 0.639444,
40
+ "mrr_at_20": 0.642131,
41
+ "mrr_at_100": 0.643808,
42
+ "mrr_at_1000": 0.643955,
43
+ "nauc_ndcg_at_1_max": 0.245061,
44
+ "nauc_ndcg_at_1_std": -0.137009,
45
+ "nauc_ndcg_at_1_diff1": 0.55709,
46
+ "nauc_ndcg_at_3_max": 0.248319,
47
+ "nauc_ndcg_at_3_std": -0.104366,
48
+ "nauc_ndcg_at_3_diff1": 0.457855,
49
+ "nauc_ndcg_at_5_max": 0.255093,
50
+ "nauc_ndcg_at_5_std": -0.091996,
51
+ "nauc_ndcg_at_5_diff1": 0.456859,
52
+ "nauc_ndcg_at_10_max": 0.250742,
53
+ "nauc_ndcg_at_10_std": -0.080252,
54
+ "nauc_ndcg_at_10_diff1": 0.461142,
55
+ "nauc_ndcg_at_20_max": 0.247866,
56
+ "nauc_ndcg_at_20_std": -0.075957,
57
+ "nauc_ndcg_at_20_diff1": 0.459826,
58
+ "nauc_ndcg_at_100_max": 0.24549,
59
+ "nauc_ndcg_at_100_std": -0.07457,
60
+ "nauc_ndcg_at_100_diff1": 0.460618,
61
+ "nauc_ndcg_at_1000_max": 0.245226,
62
+ "nauc_ndcg_at_1000_std": -0.077771,
63
+ "nauc_ndcg_at_1000_diff1": 0.462122,
64
+ "nauc_map_at_1_max": 0.223226,
65
+ "nauc_map_at_1_std": -0.111902,
66
+ "nauc_map_at_1_diff1": 0.506577,
67
+ "nauc_map_at_3_max": 0.236994,
68
+ "nauc_map_at_3_std": -0.1019,
69
+ "nauc_map_at_3_diff1": 0.460434,
70
+ "nauc_map_at_5_max": 0.241242,
71
+ "nauc_map_at_5_std": -0.096404,
72
+ "nauc_map_at_5_diff1": 0.461683,
73
+ "nauc_map_at_10_max": 0.239856,
74
+ "nauc_map_at_10_std": -0.092091,
75
+ "nauc_map_at_10_diff1": 0.463714,
76
+ "nauc_map_at_20_max": 0.239147,
77
+ "nauc_map_at_20_std": -0.091015,
78
+ "nauc_map_at_20_diff1": 0.463512,
79
+ "nauc_map_at_100_max": 0.238856,
80
+ "nauc_map_at_100_std": -0.090851,
81
+ "nauc_map_at_100_diff1": 0.463642,
82
+ "nauc_map_at_1000_max": 0.238833,
83
+ "nauc_map_at_1000_std": -0.090946,
84
+ "nauc_map_at_1000_diff1": 0.463676,
85
+ "nauc_recall_at_1_max": 0.223226,
86
+ "nauc_recall_at_1_std": -0.111902,
87
+ "nauc_recall_at_1_diff1": 0.506577,
88
+ "nauc_recall_at_3_max": 0.240556,
89
+ "nauc_recall_at_3_std": -0.078741,
90
+ "nauc_recall_at_3_diff1": 0.372825,
91
+ "nauc_recall_at_5_max": 0.257287,
92
+ "nauc_recall_at_5_std": -0.039755,
93
+ "nauc_recall_at_5_diff1": 0.352184,
94
+ "nauc_recall_at_10_max": 0.236078,
95
+ "nauc_recall_at_10_std": 0.021126,
96
+ "nauc_recall_at_10_diff1": 0.340759,
97
+ "nauc_recall_at_20_max": 0.211544,
98
+ "nauc_recall_at_20_std": 0.068803,
99
+ "nauc_recall_at_20_diff1": 0.302302,
100
+ "nauc_recall_at_100_max": 0.146115,
101
+ "nauc_recall_at_100_std": 0.18329,
102
+ "nauc_recall_at_100_diff1": 0.20034,
103
+ "nauc_recall_at_1000_max": 0.071731,
104
+ "nauc_recall_at_1000_std": 0.257758,
105
+ "nauc_recall_at_1000_diff1": 0.069671,
106
+ "nauc_precision_at_1_max": 0.245061,
107
+ "nauc_precision_at_1_std": -0.137009,
108
+ "nauc_precision_at_1_diff1": 0.55709,
109
+ "nauc_precision_at_3_max": 0.280319,
110
+ "nauc_precision_at_3_std": -0.106341,
111
+ "nauc_precision_at_3_diff1": 0.421819,
112
+ "nauc_precision_at_5_max": 0.306886,
113
+ "nauc_precision_at_5_std": -0.063768,
114
+ "nauc_precision_at_5_diff1": 0.400642,
115
+ "nauc_precision_at_10_max": 0.281175,
116
+ "nauc_precision_at_10_std": 0.010407,
117
+ "nauc_precision_at_10_diff1": 0.373056,
118
+ "nauc_precision_at_20_max": 0.246245,
119
+ "nauc_precision_at_20_std": 0.065341,
120
+ "nauc_precision_at_20_diff1": 0.313258,
121
+ "nauc_precision_at_100_max": 0.161083,
122
+ "nauc_precision_at_100_std": 0.165796,
123
+ "nauc_precision_at_100_diff1": 0.14073,
124
+ "nauc_precision_at_1000_max": 0.055871,
125
+ "nauc_precision_at_1000_std": 0.147183,
126
+ "nauc_precision_at_1000_diff1": -0.017647,
127
+ "nauc_mrr_at_1_max": 0.245061,
128
+ "nauc_mrr_at_1_std": -0.137009,
129
+ "nauc_mrr_at_1_diff1": 0.55709,
130
+ "nauc_mrr_at_3_max": 0.26592,
131
+ "nauc_mrr_at_3_std": -0.128604,
132
+ "nauc_mrr_at_3_diff1": 0.515274,
133
+ "nauc_mrr_at_5_max": 0.270103,
134
+ "nauc_mrr_at_5_std": -0.122957,
135
+ "nauc_mrr_at_5_diff1": 0.516955,
136
+ "nauc_mrr_at_10_max": 0.268861,
137
+ "nauc_mrr_at_10_std": -0.119639,
138
+ "nauc_mrr_at_10_diff1": 0.520291,
139
+ "nauc_mrr_at_20_max": 0.268109,
140
+ "nauc_mrr_at_20_std": -0.11923,
141
+ "nauc_mrr_at_20_diff1": 0.520273,
142
+ "nauc_mrr_at_100_max": 0.267881,
143
+ "nauc_mrr_at_100_std": -0.119438,
144
+ "nauc_mrr_at_100_diff1": 0.520616,
145
+ "nauc_mrr_at_1000_max": 0.267828,
146
+ "nauc_mrr_at_1000_std": -0.119632,
147
+ "nauc_mrr_at_1000_diff1": 0.520671,
148
+ "main_score": 0.65721,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 5951.847348451614,
157
+ "kg_co2_emissions": null
158
+ }
mteb/FiQA2018.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06",
3
+ "task_name": "FiQA2018",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.29167,
9
+ "ndcg_at_3": 0.26738,
10
+ "ndcg_at_5": 0.27998,
11
+ "ndcg_at_10": 0.30568,
12
+ "ndcg_at_20": 0.32864,
13
+ "ndcg_at_100": 0.36517,
14
+ "ndcg_at_1000": 0.4022,
15
+ "map_at_1": 0.13794,
16
+ "map_at_3": 0.19969,
17
+ "map_at_5": 0.21926,
18
+ "map_at_10": 0.23496,
19
+ "map_at_20": 0.24335,
20
+ "map_at_100": 0.2499,
21
+ "map_at_1000": 0.25182,
22
+ "recall_at_1": 0.13794,
23
+ "recall_at_3": 0.24295,
24
+ "recall_at_5": 0.29735,
25
+ "recall_at_10": 0.38048,
26
+ "recall_at_20": 0.44955,
27
+ "recall_at_100": 0.60103,
28
+ "recall_at_1000": 0.82989,
29
+ "precision_at_1": 0.29167,
30
+ "precision_at_3": 0.18313,
31
+ "precision_at_5": 0.13827,
32
+ "precision_at_10": 0.08827,
33
+ "precision_at_20": 0.05386,
34
+ "precision_at_100": 0.01489,
35
+ "precision_at_1000": 0.00213,
36
+ "mrr_at_1": 0.291667,
37
+ "mrr_at_3": 0.344136,
38
+ "mrr_at_5": 0.357639,
39
+ "mrr_at_10": 0.370531,
40
+ "mrr_at_20": 0.375432,
41
+ "mrr_at_100": 0.379265,
42
+ "mrr_at_1000": 0.379918,
43
+ "nauc_ndcg_at_1_max": 0.382865,
44
+ "nauc_ndcg_at_1_std": -0.018401,
45
+ "nauc_ndcg_at_1_diff1": 0.497381,
46
+ "nauc_ndcg_at_3_max": 0.323251,
47
+ "nauc_ndcg_at_3_std": -0.015778,
48
+ "nauc_ndcg_at_3_diff1": 0.393907,
49
+ "nauc_ndcg_at_5_max": 0.321995,
50
+ "nauc_ndcg_at_5_std": 0.010656,
51
+ "nauc_ndcg_at_5_diff1": 0.390196,
52
+ "nauc_ndcg_at_10_max": 0.307513,
53
+ "nauc_ndcg_at_10_std": 0.025637,
54
+ "nauc_ndcg_at_10_diff1": 0.376314,
55
+ "nauc_ndcg_at_20_max": 0.308166,
56
+ "nauc_ndcg_at_20_std": 0.041018,
57
+ "nauc_ndcg_at_20_diff1": 0.366269,
58
+ "nauc_ndcg_at_100_max": 0.329718,
59
+ "nauc_ndcg_at_100_std": 0.06222,
60
+ "nauc_ndcg_at_100_diff1": 0.377411,
61
+ "nauc_ndcg_at_1000_max": 0.342405,
62
+ "nauc_ndcg_at_1000_std": 0.066923,
63
+ "nauc_ndcg_at_1000_diff1": 0.379008,
64
+ "nauc_map_at_1_max": 0.25955,
65
+ "nauc_map_at_1_std": -0.068494,
66
+ "nauc_map_at_1_diff1": 0.455645,
67
+ "nauc_map_at_3_max": 0.284944,
68
+ "nauc_map_at_3_std": -0.028556,
69
+ "nauc_map_at_3_diff1": 0.403951,
70
+ "nauc_map_at_5_max": 0.302179,
71
+ "nauc_map_at_5_std": -0.006055,
72
+ "nauc_map_at_5_diff1": 0.398122,
73
+ "nauc_map_at_10_max": 0.304114,
74
+ "nauc_map_at_10_std": 0.002683,
75
+ "nauc_map_at_10_diff1": 0.391394,
76
+ "nauc_map_at_20_max": 0.305538,
77
+ "nauc_map_at_20_std": 0.008069,
78
+ "nauc_map_at_20_diff1": 0.387448,
79
+ "nauc_map_at_100_max": 0.311454,
80
+ "nauc_map_at_100_std": 0.013278,
81
+ "nauc_map_at_100_diff1": 0.389779,
82
+ "nauc_map_at_1000_max": 0.312485,
83
+ "nauc_map_at_1000_std": 0.014026,
84
+ "nauc_map_at_1000_diff1": 0.389806,
85
+ "nauc_recall_at_1_max": 0.25955,
86
+ "nauc_recall_at_1_std": -0.068494,
87
+ "nauc_recall_at_1_diff1": 0.455645,
88
+ "nauc_recall_at_3_max": 0.240749,
89
+ "nauc_recall_at_3_std": -0.009688,
90
+ "nauc_recall_at_3_diff1": 0.338397,
91
+ "nauc_recall_at_5_max": 0.249765,
92
+ "nauc_recall_at_5_std": 0.041173,
93
+ "nauc_recall_at_5_diff1": 0.306595,
94
+ "nauc_recall_at_10_max": 0.194711,
95
+ "nauc_recall_at_10_std": 0.068708,
96
+ "nauc_recall_at_10_diff1": 0.235346,
97
+ "nauc_recall_at_20_max": 0.187834,
98
+ "nauc_recall_at_20_std": 0.115026,
99
+ "nauc_recall_at_20_diff1": 0.195265,
100
+ "nauc_recall_at_100_max": 0.248323,
101
+ "nauc_recall_at_100_std": 0.209214,
102
+ "nauc_recall_at_100_diff1": 0.222041,
103
+ "nauc_recall_at_1000_max": 0.30524,
104
+ "nauc_recall_at_1000_std": 0.365714,
105
+ "nauc_recall_at_1000_diff1": 0.148893,
106
+ "nauc_precision_at_1_max": 0.382865,
107
+ "nauc_precision_at_1_std": -0.018401,
108
+ "nauc_precision_at_1_diff1": 0.497381,
109
+ "nauc_precision_at_3_max": 0.367225,
110
+ "nauc_precision_at_3_std": 0.013713,
111
+ "nauc_precision_at_3_diff1": 0.337091,
112
+ "nauc_precision_at_5_max": 0.378168,
113
+ "nauc_precision_at_5_std": 0.063933,
114
+ "nauc_precision_at_5_diff1": 0.305218,
115
+ "nauc_precision_at_10_max": 0.365227,
116
+ "nauc_precision_at_10_std": 0.099963,
117
+ "nauc_precision_at_10_diff1": 0.252887,
118
+ "nauc_precision_at_20_max": 0.342121,
119
+ "nauc_precision_at_20_std": 0.122905,
120
+ "nauc_precision_at_20_diff1": 0.214998,
121
+ "nauc_precision_at_100_max": 0.342554,
122
+ "nauc_precision_at_100_std": 0.163248,
123
+ "nauc_precision_at_100_diff1": 0.1778,
124
+ "nauc_precision_at_1000_max": 0.28898,
125
+ "nauc_precision_at_1000_std": 0.146869,
126
+ "nauc_precision_at_1000_diff1": 0.081735,
127
+ "nauc_mrr_at_1_max": 0.382865,
128
+ "nauc_mrr_at_1_std": -0.018401,
129
+ "nauc_mrr_at_1_diff1": 0.497381,
130
+ "nauc_mrr_at_3_max": 0.361507,
131
+ "nauc_mrr_at_3_std": -0.010684,
132
+ "nauc_mrr_at_3_diff1": 0.448131,
133
+ "nauc_mrr_at_5_max": 0.364134,
134
+ "nauc_mrr_at_5_std": 0.0015,
135
+ "nauc_mrr_at_5_diff1": 0.448739,
136
+ "nauc_mrr_at_10_max": 0.358277,
137
+ "nauc_mrr_at_10_std": 0.003988,
138
+ "nauc_mrr_at_10_diff1": 0.439851,
139
+ "nauc_mrr_at_20_max": 0.358842,
140
+ "nauc_mrr_at_20_std": 0.008399,
141
+ "nauc_mrr_at_20_diff1": 0.438136,
142
+ "nauc_mrr_at_100_max": 0.359867,
143
+ "nauc_mrr_at_100_std": 0.009229,
144
+ "nauc_mrr_at_100_diff1": 0.439474,
145
+ "nauc_mrr_at_1000_max": 0.359973,
146
+ "nauc_mrr_at_1000_std": 0.008946,
147
+ "nauc_mrr_at_1000_diff1": 0.439639,
148
+ "main_score": 0.30568,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 80.82175087928772,
157
+ "kg_co2_emissions": null
158
+ }
mteb/HotpotQA.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "ab518f4d6fcca38d87c25209f94beba119d02014",
3
+ "task_name": "HotpotQA",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.60162,
9
+ "ndcg_at_3": 0.44502,
10
+ "ndcg_at_5": 0.46295,
11
+ "ndcg_at_10": 0.47809,
12
+ "ndcg_at_20": 0.48903,
13
+ "ndcg_at_100": 0.50535,
14
+ "ndcg_at_1000": 0.51923,
15
+ "map_at_1": 0.30081,
16
+ "map_at_3": 0.36953,
17
+ "map_at_5": 0.38167,
18
+ "map_at_10": 0.38938,
19
+ "map_at_20": 0.39314,
20
+ "map_at_100": 0.39603,
21
+ "map_at_1000": 0.39663,
22
+ "recall_at_1": 0.30081,
23
+ "recall_at_3": 0.40614,
24
+ "recall_at_5": 0.44159,
25
+ "recall_at_10": 0.47961,
26
+ "recall_at_20": 0.51499,
27
+ "recall_at_100": 0.58731,
28
+ "recall_at_1000": 0.68035,
29
+ "precision_at_1": 0.60162,
30
+ "precision_at_3": 0.27076,
31
+ "precision_at_5": 0.17664,
32
+ "precision_at_10": 0.09592,
33
+ "precision_at_20": 0.0515,
34
+ "precision_at_100": 0.01175,
35
+ "precision_at_1000": 0.00136,
36
+ "mrr_at_1": 0.601621,
37
+ "mrr_at_3": 0.652622,
38
+ "mrr_at_5": 0.660826,
39
+ "mrr_at_10": 0.665901,
40
+ "mrr_at_20": 0.668133,
41
+ "mrr_at_100": 0.669551,
42
+ "mrr_at_1000": 0.669784,
43
+ "nauc_ndcg_at_1_max": 0.5116,
44
+ "nauc_ndcg_at_1_std": 0.073935,
45
+ "nauc_ndcg_at_1_diff1": 0.715694,
46
+ "nauc_ndcg_at_3_max": 0.378233,
47
+ "nauc_ndcg_at_3_std": 0.067468,
48
+ "nauc_ndcg_at_3_diff1": 0.494853,
49
+ "nauc_ndcg_at_5_max": 0.359559,
50
+ "nauc_ndcg_at_5_std": 0.073257,
51
+ "nauc_ndcg_at_5_diff1": 0.465013,
52
+ "nauc_ndcg_at_10_max": 0.339948,
53
+ "nauc_ndcg_at_10_std": 0.076042,
54
+ "nauc_ndcg_at_10_diff1": 0.443659,
55
+ "nauc_ndcg_at_20_max": 0.328131,
56
+ "nauc_ndcg_at_20_std": 0.074644,
57
+ "nauc_ndcg_at_20_diff1": 0.432222,
58
+ "nauc_ndcg_at_100_max": 0.312544,
59
+ "nauc_ndcg_at_100_std": 0.078449,
60
+ "nauc_ndcg_at_100_diff1": 0.415341,
61
+ "nauc_ndcg_at_1000_max": 0.305183,
62
+ "nauc_ndcg_at_1000_std": 0.078385,
63
+ "nauc_ndcg_at_1000_diff1": 0.409027,
64
+ "nauc_map_at_1_max": 0.5116,
65
+ "nauc_map_at_1_std": 0.073935,
66
+ "nauc_map_at_1_diff1": 0.715694,
67
+ "nauc_map_at_3_max": 0.334791,
68
+ "nauc_map_at_3_std": 0.058541,
69
+ "nauc_map_at_3_diff1": 0.444158,
70
+ "nauc_map_at_5_max": 0.31872,
71
+ "nauc_map_at_5_std": 0.060961,
72
+ "nauc_map_at_5_diff1": 0.421822,
73
+ "nauc_map_at_10_max": 0.307675,
74
+ "nauc_map_at_10_std": 0.06241,
75
+ "nauc_map_at_10_diff1": 0.410375,
76
+ "nauc_map_at_20_max": 0.303193,
77
+ "nauc_map_at_20_std": 0.061659,
78
+ "nauc_map_at_20_diff1": 0.406205,
79
+ "nauc_map_at_100_max": 0.299934,
80
+ "nauc_map_at_100_std": 0.062056,
81
+ "nauc_map_at_100_diff1": 0.402882,
82
+ "nauc_map_at_1000_max": 0.299604,
83
+ "nauc_map_at_1000_std": 0.06212,
84
+ "nauc_map_at_1000_diff1": 0.402604,
85
+ "nauc_recall_at_1_max": 0.5116,
86
+ "nauc_recall_at_1_std": 0.073935,
87
+ "nauc_recall_at_1_diff1": 0.715694,
88
+ "nauc_recall_at_3_max": 0.295614,
89
+ "nauc_recall_at_3_std": 0.061532,
90
+ "nauc_recall_at_3_diff1": 0.365997,
91
+ "nauc_recall_at_5_max": 0.250882,
92
+ "nauc_recall_at_5_std": 0.072136,
93
+ "nauc_recall_at_5_diff1": 0.296869,
94
+ "nauc_recall_at_10_max": 0.187916,
95
+ "nauc_recall_at_10_std": 0.075065,
96
+ "nauc_recall_at_10_diff1": 0.227623,
97
+ "nauc_recall_at_20_max": 0.139061,
98
+ "nauc_recall_at_20_std": 0.067366,
99
+ "nauc_recall_at_20_diff1": 0.176912,
100
+ "nauc_recall_at_100_max": 0.053559,
101
+ "nauc_recall_at_100_std": 0.077563,
102
+ "nauc_recall_at_100_diff1": 0.080251,
103
+ "nauc_recall_at_1000_max": -0.03957,
104
+ "nauc_recall_at_1000_std": 0.069824,
105
+ "nauc_recall_at_1000_diff1": -0.020025,
106
+ "nauc_precision_at_1_max": 0.5116,
107
+ "nauc_precision_at_1_std": 0.073935,
108
+ "nauc_precision_at_1_diff1": 0.715694,
109
+ "nauc_precision_at_3_max": 0.295614,
110
+ "nauc_precision_at_3_std": 0.061532,
111
+ "nauc_precision_at_3_diff1": 0.365997,
112
+ "nauc_precision_at_5_max": 0.250882,
113
+ "nauc_precision_at_5_std": 0.072136,
114
+ "nauc_precision_at_5_diff1": 0.296869,
115
+ "nauc_precision_at_10_max": 0.187916,
116
+ "nauc_precision_at_10_std": 0.075065,
117
+ "nauc_precision_at_10_diff1": 0.227623,
118
+ "nauc_precision_at_20_max": 0.139061,
119
+ "nauc_precision_at_20_std": 0.067366,
120
+ "nauc_precision_at_20_diff1": 0.176912,
121
+ "nauc_precision_at_100_max": 0.053559,
122
+ "nauc_precision_at_100_std": 0.077563,
123
+ "nauc_precision_at_100_diff1": 0.080251,
124
+ "nauc_precision_at_1000_max": -0.03957,
125
+ "nauc_precision_at_1000_std": 0.069824,
126
+ "nauc_precision_at_1000_diff1": -0.020025,
127
+ "nauc_mrr_at_1_max": 0.5116,
128
+ "nauc_mrr_at_1_std": 0.073935,
129
+ "nauc_mrr_at_1_diff1": 0.715694,
130
+ "nauc_mrr_at_3_max": 0.516213,
131
+ "nauc_mrr_at_3_std": 0.084836,
132
+ "nauc_mrr_at_3_diff1": 0.68779,
133
+ "nauc_mrr_at_5_max": 0.518266,
134
+ "nauc_mrr_at_5_std": 0.090287,
135
+ "nauc_mrr_at_5_diff1": 0.684759,
136
+ "nauc_mrr_at_10_max": 0.516795,
137
+ "nauc_mrr_at_10_std": 0.091841,
138
+ "nauc_mrr_at_10_diff1": 0.682263,
139
+ "nauc_mrr_at_20_max": 0.516508,
140
+ "nauc_mrr_at_20_std": 0.092047,
141
+ "nauc_mrr_at_20_diff1": 0.682142,
142
+ "nauc_mrr_at_100_max": 0.516723,
143
+ "nauc_mrr_at_100_std": 0.093048,
144
+ "nauc_mrr_at_100_diff1": 0.682074,
145
+ "nauc_mrr_at_1000_max": 0.516691,
146
+ "nauc_mrr_at_1000_std": 0.093012,
147
+ "nauc_mrr_at_1000_diff1": 0.682124,
148
+ "main_score": 0.47809,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 3732.872656583786,
157
+ "kg_co2_emissions": null
158
+ }
mteb/ImdbClassification.json ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7",
3
+ "task_name": "ImdbClassification",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "accuracy": 0.61454,
9
+ "f1": 0.611751,
10
+ "f1_weighted": 0.611751,
11
+ "ap": 0.572238,
12
+ "ap_weighted": 0.572238,
13
+ "scores_per_experiment": [
14
+ {
15
+ "accuracy": 0.64184,
16
+ "f1": 0.641046,
17
+ "f1_weighted": 0.641046,
18
+ "ap": 0.593128,
19
+ "ap_weighted": 0.593128
20
+ },
21
+ {
22
+ "accuracy": 0.64316,
23
+ "f1": 0.643142,
24
+ "f1_weighted": 0.643142,
25
+ "ap": 0.591785,
26
+ "ap_weighted": 0.591785
27
+ },
28
+ {
29
+ "accuracy": 0.5602,
30
+ "f1": 0.556495,
31
+ "f1_weighted": 0.556495,
32
+ "ap": 0.534535,
33
+ "ap_weighted": 0.534535
34
+ },
35
+ {
36
+ "accuracy": 0.63604,
37
+ "f1": 0.635737,
38
+ "f1_weighted": 0.635737,
39
+ "ap": 0.58766,
40
+ "ap_weighted": 0.58766
41
+ },
42
+ {
43
+ "accuracy": 0.60992,
44
+ "f1": 0.604073,
45
+ "f1_weighted": 0.604073,
46
+ "ap": 0.56468,
47
+ "ap_weighted": 0.56468
48
+ },
49
+ {
50
+ "accuracy": 0.60764,
51
+ "f1": 0.606284,
52
+ "f1_weighted": 0.606284,
53
+ "ap": 0.566947,
54
+ "ap_weighted": 0.566947
55
+ },
56
+ {
57
+ "accuracy": 0.60716,
58
+ "f1": 0.592887,
59
+ "f1_weighted": 0.592887,
60
+ "ap": 0.571938,
61
+ "ap_weighted": 0.571938
62
+ },
63
+ {
64
+ "accuracy": 0.60104,
65
+ "f1": 0.599904,
66
+ "f1_weighted": 0.599904,
67
+ "ap": 0.561947,
68
+ "ap_weighted": 0.561947
69
+ },
70
+ {
71
+ "accuracy": 0.5806,
72
+ "f1": 0.580167,
73
+ "f1_weighted": 0.580167,
74
+ "ap": 0.546404,
75
+ "ap_weighted": 0.546404
76
+ },
77
+ {
78
+ "accuracy": 0.6578,
79
+ "f1": 0.657772,
80
+ "f1_weighted": 0.657772,
81
+ "ap": 0.603361,
82
+ "ap_weighted": 0.603361
83
+ }
84
+ ],
85
+ "main_score": 0.61454,
86
+ "hf_subset": "default",
87
+ "languages": [
88
+ "eng-Latn"
89
+ ]
90
+ }
91
+ ]
92
+ },
93
+ "evaluation_time": 65.28035974502563,
94
+ "kg_co2_emissions": null
95
+ }
mteb/MSMARCO.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "c5a29a104738b98a9e76336939199e264163d4a0",
3
+ "task_name": "MSMARCO",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.76744,
9
+ "ndcg_at_3": 0.71949,
10
+ "ndcg_at_5": 0.68963,
11
+ "ndcg_at_10": 0.66652,
12
+ "ndcg_at_20": 0.63025,
13
+ "ndcg_at_100": 0.57169,
14
+ "ndcg_at_1000": 0.64349,
15
+ "map_at_1": 0.02314,
16
+ "map_at_3": 0.05259,
17
+ "map_at_5": 0.07976,
18
+ "map_at_10": 0.13288,
19
+ "map_at_20": 0.19885,
20
+ "map_at_100": 0.32109,
21
+ "map_at_1000": 0.38117,
22
+ "recall_at_1": 0.02314,
23
+ "recall_at_3": 0.05397,
24
+ "recall_at_5": 0.08814,
25
+ "recall_at_10": 0.15347,
26
+ "recall_at_20": 0.23955,
27
+ "recall_at_100": 0.44498,
28
+ "recall_at_1000": 0.69357,
29
+ "precision_at_1": 0.95349,
30
+ "precision_at_3": 0.86047,
31
+ "precision_at_5": 0.79535,
32
+ "precision_at_10": 0.72558,
33
+ "precision_at_20": 0.62907,
34
+ "precision_at_100": 0.31977,
35
+ "precision_at_1000": 0.06049,
36
+ "mrr_at_1": 0.953488,
37
+ "mrr_at_3": 0.965116,
38
+ "mrr_at_5": 0.969767,
39
+ "mrr_at_10": 0.969767,
40
+ "mrr_at_20": 0.969767,
41
+ "mrr_at_100": 0.969767,
42
+ "mrr_at_1000": 0.969767,
43
+ "nauc_ndcg_at_1_max": 0.335394,
44
+ "nauc_ndcg_at_1_std": 0.12295,
45
+ "nauc_ndcg_at_1_diff1": 0.160874,
46
+ "nauc_ndcg_at_3_max": 0.329494,
47
+ "nauc_ndcg_at_3_std": 0.212066,
48
+ "nauc_ndcg_at_3_diff1": 0.019303,
49
+ "nauc_ndcg_at_5_max": 0.268626,
50
+ "nauc_ndcg_at_5_std": 0.198698,
51
+ "nauc_ndcg_at_5_diff1": -0.130704,
52
+ "nauc_ndcg_at_10_max": 0.258121,
53
+ "nauc_ndcg_at_10_std": 0.314913,
54
+ "nauc_ndcg_at_10_diff1": -0.321204,
55
+ "nauc_ndcg_at_20_max": 0.308474,
56
+ "nauc_ndcg_at_20_std": 0.486443,
57
+ "nauc_ndcg_at_20_diff1": -0.412659,
58
+ "nauc_ndcg_at_100_max": 0.319625,
59
+ "nauc_ndcg_at_100_std": 0.561189,
60
+ "nauc_ndcg_at_100_diff1": -0.451756,
61
+ "nauc_ndcg_at_1000_max": 0.388789,
62
+ "nauc_ndcg_at_1000_std": 0.603502,
63
+ "nauc_ndcg_at_1000_diff1": -0.463901,
64
+ "nauc_map_at_1_max": -0.286523,
65
+ "nauc_map_at_1_std": -0.331061,
66
+ "nauc_map_at_1_diff1": -0.11994,
67
+ "nauc_map_at_3_max": -0.160122,
68
+ "nauc_map_at_3_std": -0.219302,
69
+ "nauc_map_at_3_diff1": -0.038522,
70
+ "nauc_map_at_5_max": -0.120035,
71
+ "nauc_map_at_5_std": -0.209837,
72
+ "nauc_map_at_5_diff1": -0.05801,
73
+ "nauc_map_at_10_max": -0.044091,
74
+ "nauc_map_at_10_std": -0.134428,
75
+ "nauc_map_at_10_diff1": -0.104398,
76
+ "nauc_map_at_20_max": 0.029722,
77
+ "nauc_map_at_20_std": -0.021077,
78
+ "nauc_map_at_20_diff1": -0.205479,
79
+ "nauc_map_at_100_max": 0.181352,
80
+ "nauc_map_at_100_std": 0.363231,
81
+ "nauc_map_at_100_diff1": -0.383395,
82
+ "nauc_map_at_1000_max": 0.28169,
83
+ "nauc_map_at_1000_std": 0.600619,
84
+ "nauc_map_at_1000_diff1": -0.438413,
85
+ "nauc_recall_at_1_max": -0.286523,
86
+ "nauc_recall_at_1_std": -0.331061,
87
+ "nauc_recall_at_1_diff1": -0.11994,
88
+ "nauc_recall_at_3_max": -0.162157,
89
+ "nauc_recall_at_3_std": -0.230294,
90
+ "nauc_recall_at_3_diff1": -0.042975,
91
+ "nauc_recall_at_5_max": -0.183205,
92
+ "nauc_recall_at_5_std": -0.268365,
93
+ "nauc_recall_at_5_diff1": -0.101943,
94
+ "nauc_recall_at_10_max": -0.119993,
95
+ "nauc_recall_at_10_std": -0.20104,
96
+ "nauc_recall_at_10_diff1": -0.124464,
97
+ "nauc_recall_at_20_max": -0.068623,
98
+ "nauc_recall_at_20_std": -0.114267,
99
+ "nauc_recall_at_20_diff1": -0.202857,
100
+ "nauc_recall_at_100_max": 0.055972,
101
+ "nauc_recall_at_100_std": 0.207599,
102
+ "nauc_recall_at_100_diff1": -0.354162,
103
+ "nauc_recall_at_1000_max": 0.260786,
104
+ "nauc_recall_at_1000_std": 0.526085,
105
+ "nauc_recall_at_1000_diff1": -0.411833,
106
+ "nauc_precision_at_1_max": 0.558702,
107
+ "nauc_precision_at_1_std": 0.67915,
108
+ "nauc_precision_at_1_diff1": 0.614293,
109
+ "nauc_precision_at_3_max": 0.373974,
110
+ "nauc_precision_at_3_std": 0.616163,
111
+ "nauc_precision_at_3_diff1": -0.125103,
112
+ "nauc_precision_at_5_max": 0.317393,
113
+ "nauc_precision_at_5_std": 0.453637,
114
+ "nauc_precision_at_5_diff1": -0.301655,
115
+ "nauc_precision_at_10_max": 0.346793,
116
+ "nauc_precision_at_10_std": 0.589706,
117
+ "nauc_precision_at_10_diff1": -0.421495,
118
+ "nauc_precision_at_20_max": 0.339533,
119
+ "nauc_precision_at_20_std": 0.666475,
120
+ "nauc_precision_at_20_diff1": -0.418728,
121
+ "nauc_precision_at_100_max": 0.292942,
122
+ "nauc_precision_at_100_std": 0.742772,
123
+ "nauc_precision_at_100_diff1": -0.334855,
124
+ "nauc_precision_at_1000_max": 0.2712,
125
+ "nauc_precision_at_1000_std": 0.677077,
126
+ "nauc_precision_at_1000_diff1": -0.222352,
127
+ "nauc_mrr_at_1_max": 0.558702,
128
+ "nauc_mrr_at_1_std": 0.67915,
129
+ "nauc_mrr_at_1_diff1": 0.614293,
130
+ "nauc_mrr_at_3_max": 0.705801,
131
+ "nauc_mrr_at_3_std": 0.7861,
132
+ "nauc_mrr_at_3_diff1": 0.699624,
133
+ "nauc_mrr_at_5_max": 0.66054,
134
+ "nauc_mrr_at_5_std": 0.753192,
135
+ "nauc_mrr_at_5_diff1": 0.673369,
136
+ "nauc_mrr_at_10_max": 0.66054,
137
+ "nauc_mrr_at_10_std": 0.753192,
138
+ "nauc_mrr_at_10_diff1": 0.673369,
139
+ "nauc_mrr_at_20_max": 0.66054,
140
+ "nauc_mrr_at_20_std": 0.753192,
141
+ "nauc_mrr_at_20_diff1": 0.673369,
142
+ "nauc_mrr_at_100_max": 0.66054,
143
+ "nauc_mrr_at_100_std": 0.753192,
144
+ "nauc_mrr_at_100_diff1": 0.673369,
145
+ "nauc_mrr_at_1000_max": 0.66054,
146
+ "nauc_mrr_at_1000_std": 0.753192,
147
+ "nauc_mrr_at_1000_diff1": 0.673369,
148
+ "main_score": 0.66652,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 5779.706394433975,
157
+ "kg_co2_emissions": null
158
+ }
mteb/MTOPDomainClassification.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf",
3
+ "task_name": "MTOPDomainClassification",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "accuracy": 0.889124,
9
+ "f1": 0.88758,
10
+ "f1_weighted": 0.889747,
11
+ "scores_per_experiment": [
12
+ {
13
+ "accuracy": 0.854537,
14
+ "f1": 0.857721,
15
+ "f1_weighted": 0.855151
16
+ },
17
+ {
18
+ "accuracy": 0.900593,
19
+ "f1": 0.89766,
20
+ "f1_weighted": 0.901084
21
+ },
22
+ {
23
+ "accuracy": 0.901049,
24
+ "f1": 0.898648,
25
+ "f1_weighted": 0.900877
26
+ },
27
+ {
28
+ "accuracy": 0.897629,
29
+ "f1": 0.894282,
30
+ "f1_weighted": 0.898338
31
+ },
32
+ {
33
+ "accuracy": 0.891473,
34
+ "f1": 0.890253,
35
+ "f1_weighted": 0.892135
36
+ },
37
+ {
38
+ "accuracy": 0.882809,
39
+ "f1": 0.884257,
40
+ "f1_weighted": 0.885064
41
+ },
42
+ {
43
+ "accuracy": 0.887597,
44
+ "f1": 0.884408,
45
+ "f1_weighted": 0.887453
46
+ },
47
+ {
48
+ "accuracy": 0.891017,
49
+ "f1": 0.887588,
50
+ "f1_weighted": 0.890818
51
+ },
52
+ {
53
+ "accuracy": 0.891929,
54
+ "f1": 0.89255,
55
+ "f1_weighted": 0.892768
56
+ },
57
+ {
58
+ "accuracy": 0.892613,
59
+ "f1": 0.888437,
60
+ "f1_weighted": 0.893779
61
+ }
62
+ ],
63
+ "main_score": 0.889124,
64
+ "hf_subset": "en",
65
+ "languages": [
66
+ "eng-Latn"
67
+ ]
68
+ }
69
+ ]
70
+ },
71
+ "evaluation_time": 10.538909912109375,
72
+ "kg_co2_emissions": null
73
+ }
mteb/MTOPIntentClassification.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba",
3
+ "task_name": "MTOPIntentClassification",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "accuracy": 0.656475,
9
+ "f1": 0.490948,
10
+ "f1_weighted": 0.693994,
11
+ "scores_per_experiment": [
12
+ {
13
+ "accuracy": 0.641587,
14
+ "f1": 0.456483,
15
+ "f1_weighted": 0.683725
16
+ },
17
+ {
18
+ "accuracy": 0.671911,
19
+ "f1": 0.48881,
20
+ "f1_weighted": 0.709701
21
+ },
22
+ {
23
+ "accuracy": 0.673279,
24
+ "f1": 0.501567,
25
+ "f1_weighted": 0.711032
26
+ },
27
+ {
28
+ "accuracy": 0.656407,
29
+ "f1": 0.514271,
30
+ "f1_weighted": 0.692525
31
+ },
32
+ {
33
+ "accuracy": 0.639763,
34
+ "f1": 0.480901,
35
+ "f1_weighted": 0.671092
36
+ },
37
+ {
38
+ "accuracy": 0.645691,
39
+ "f1": 0.501634,
40
+ "f1_weighted": 0.685544
41
+ },
42
+ {
43
+ "accuracy": 0.663703,
44
+ "f1": 0.479191,
45
+ "f1_weighted": 0.704379
46
+ },
47
+ {
48
+ "accuracy": 0.674647,
49
+ "f1": 0.516005,
50
+ "f1_weighted": 0.709525
51
+ },
52
+ {
53
+ "accuracy": 0.650935,
54
+ "f1": 0.480141,
55
+ "f1_weighted": 0.689081
56
+ },
57
+ {
58
+ "accuracy": 0.646831,
59
+ "f1": 0.490481,
60
+ "f1_weighted": 0.683338
61
+ }
62
+ ],
63
+ "main_score": 0.656475,
64
+ "hf_subset": "en",
65
+ "languages": [
66
+ "eng-Latn"
67
+ ]
68
+ }
69
+ ]
70
+ },
71
+ "evaluation_time": 13.826844215393066,
72
+ "kg_co2_emissions": null
73
+ }
mteb/MassiveIntentClassification.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6",
3
+ "task_name": "MassiveIntentClassification",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "accuracy": 0.652354,
9
+ "f1": 0.635945,
10
+ "f1_weighted": 0.650395,
11
+ "scores_per_experiment": [
12
+ {
13
+ "accuracy": 0.644923,
14
+ "f1": 0.639513,
15
+ "f1_weighted": 0.640635
16
+ },
17
+ {
18
+ "accuracy": 0.677539,
19
+ "f1": 0.652575,
20
+ "f1_weighted": 0.67554
21
+ },
22
+ {
23
+ "accuracy": 0.649294,
24
+ "f1": 0.630992,
25
+ "f1_weighted": 0.645129
26
+ },
27
+ {
28
+ "accuracy": 0.680565,
29
+ "f1": 0.659978,
30
+ "f1_weighted": 0.675892
31
+ },
32
+ {
33
+ "accuracy": 0.653329,
34
+ "f1": 0.637184,
35
+ "f1_weighted": 0.649634
36
+ },
37
+ {
38
+ "accuracy": 0.641896,
39
+ "f1": 0.624381,
40
+ "f1_weighted": 0.640526
41
+ },
42
+ {
43
+ "accuracy": 0.642233,
44
+ "f1": 0.633383,
45
+ "f1_weighted": 0.643923
46
+ },
47
+ {
48
+ "accuracy": 0.638534,
49
+ "f1": 0.618431,
50
+ "f1_weighted": 0.633252
51
+ },
52
+ {
53
+ "accuracy": 0.64694,
54
+ "f1": 0.63405,
55
+ "f1_weighted": 0.649656
56
+ },
57
+ {
58
+ "accuracy": 0.648285,
59
+ "f1": 0.628963,
60
+ "f1_weighted": 0.649767
61
+ }
62
+ ],
63
+ "main_score": 0.652354,
64
+ "hf_subset": "en",
65
+ "languages": [
66
+ "eng-Latn"
67
+ ]
68
+ }
69
+ ]
70
+ },
71
+ "evaluation_time": 11.81202483177185,
72
+ "kg_co2_emissions": null
73
+ }
mteb/MassiveScenarioClassification.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8",
3
+ "task_name": "MassiveScenarioClassification",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "accuracy": 0.726126,
9
+ "f1": 0.724117,
10
+ "f1_weighted": 0.726333,
11
+ "scores_per_experiment": [
12
+ {
13
+ "accuracy": 0.726295,
14
+ "f1": 0.724368,
15
+ "f1_weighted": 0.72364
16
+ },
17
+ {
18
+ "accuracy": 0.734701,
19
+ "f1": 0.737144,
20
+ "f1_weighted": 0.733128
21
+ },
22
+ {
23
+ "accuracy": 0.711163,
24
+ "f1": 0.711532,
25
+ "f1_weighted": 0.709362
26
+ },
27
+ {
28
+ "accuracy": 0.705783,
29
+ "f1": 0.699515,
30
+ "f1_weighted": 0.709819
31
+ },
32
+ {
33
+ "accuracy": 0.737727,
34
+ "f1": 0.733102,
35
+ "f1_weighted": 0.739757
36
+ },
37
+ {
38
+ "accuracy": 0.740081,
39
+ "f1": 0.729691,
40
+ "f1_weighted": 0.74056
41
+ },
42
+ {
43
+ "accuracy": 0.72226,
44
+ "f1": 0.724384,
45
+ "f1_weighted": 0.726585
46
+ },
47
+ {
48
+ "accuracy": 0.722596,
49
+ "f1": 0.725063,
50
+ "f1_weighted": 0.721339
51
+ },
52
+ {
53
+ "accuracy": 0.736382,
54
+ "f1": 0.733771,
55
+ "f1_weighted": 0.736615
56
+ },
57
+ {
58
+ "accuracy": 0.724277,
59
+ "f1": 0.722599,
60
+ "f1_weighted": 0.722527
61
+ }
62
+ ],
63
+ "main_score": 0.726126,
64
+ "hf_subset": "en",
65
+ "languages": [
66
+ "eng-Latn"
67
+ ]
68
+ }
69
+ ]
70
+ },
71
+ "evaluation_time": 4.415860176086426,
72
+ "kg_co2_emissions": null
73
+ }
mteb/MedrxivClusteringP2P.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73",
3
+ "task_name": "MedrxivClusteringP2P",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "v_measure": 0.2895,
9
+ "v_measure_std": 0.013911,
10
+ "v_measures": [
11
+ 0.271819,
12
+ 0.271103,
13
+ 0.278713,
14
+ 0.272723,
15
+ 0.288926,
16
+ 0.304441,
17
+ 0.300211,
18
+ 0.303506,
19
+ 0.306792,
20
+ 0.296767
21
+ ],
22
+ "main_score": 0.2895,
23
+ "hf_subset": "default",
24
+ "languages": [
25
+ "eng-Latn"
26
+ ]
27
+ }
28
+ ]
29
+ },
30
+ "evaluation_time": 114.98579716682434,
31
+ "kg_co2_emissions": null
32
+ }
mteb/MedrxivClusteringS2S.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663",
3
+ "task_name": "MedrxivClusteringS2S",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "v_measure": 0.27076,
9
+ "v_measure_std": 0.017323,
10
+ "v_measures": [
11
+ 0.256262,
12
+ 0.252559,
13
+ 0.253638,
14
+ 0.253727,
15
+ 0.25903,
16
+ 0.301547,
17
+ 0.277187,
18
+ 0.287709,
19
+ 0.292024,
20
+ 0.273922
21
+ ],
22
+ "main_score": 0.27076,
23
+ "hf_subset": "default",
24
+ "languages": [
25
+ "eng-Latn"
26
+ ]
27
+ }
28
+ ]
29
+ },
30
+ "evaluation_time": 17.49012589454651,
31
+ "kg_co2_emissions": null
32
+ }
mteb/MindSmallReranking.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "59042f120c80e8afa9cdbb224f67076cec0fc9a7",
3
+ "task_name": "MindSmallReranking",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "map": 0.306437,
9
+ "mrr": 0.316768,
10
+ "nAUC_map_max": -0.182556,
11
+ "nAUC_map_std": -0.016765,
12
+ "nAUC_map_diff1": 0.134889,
13
+ "nAUC_mrr_max": -0.128279,
14
+ "nAUC_mrr_std": 0.003295,
15
+ "nAUC_mrr_diff1": 0.128104,
16
+ "main_score": 0.306437,
17
+ "hf_subset": "default",
18
+ "languages": [
19
+ "eng-Latn"
20
+ ]
21
+ }
22
+ ]
23
+ },
24
+ "evaluation_time": 120.96634793281555,
25
+ "kg_co2_emissions": null
26
+ }
mteb/NFCorpus.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814",
3
+ "task_name": "NFCorpus",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.34675,
9
+ "ndcg_at_3": 0.31048,
10
+ "ndcg_at_5": 0.29451,
11
+ "ndcg_at_10": 0.2663,
12
+ "ndcg_at_20": 0.24402,
13
+ "ndcg_at_100": 0.24319,
14
+ "ndcg_at_1000": 0.33458,
15
+ "map_at_1": 0.04323,
16
+ "map_at_3": 0.06706,
17
+ "map_at_5": 0.08085,
18
+ "map_at_10": 0.09278,
19
+ "map_at_20": 0.1011,
20
+ "map_at_100": 0.11312,
21
+ "map_at_1000": 0.12461,
22
+ "recall_at_1": 0.04323,
23
+ "recall_at_3": 0.07573,
24
+ "recall_at_5": 0.10248,
25
+ "recall_at_10": 0.12779,
26
+ "recall_at_20": 0.15931,
27
+ "recall_at_100": 0.25619,
28
+ "recall_at_1000": 0.57667,
29
+ "precision_at_1": 0.36223,
30
+ "precision_at_3": 0.29102,
31
+ "precision_at_5": 0.25573,
32
+ "precision_at_10": 0.19505,
33
+ "precision_at_20": 0.13824,
34
+ "precision_at_100": 0.06037,
35
+ "precision_at_1000": 0.0188,
36
+ "mrr_at_1": 0.362229,
37
+ "mrr_at_3": 0.427245,
38
+ "mrr_at_5": 0.443189,
39
+ "mrr_at_10": 0.452435,
40
+ "mrr_at_20": 0.45666,
41
+ "mrr_at_100": 0.459342,
42
+ "mrr_at_1000": 0.459762,
43
+ "nauc_ndcg_at_1_max": 0.489247,
44
+ "nauc_ndcg_at_1_std": 0.207882,
45
+ "nauc_ndcg_at_1_diff1": 0.455278,
46
+ "nauc_ndcg_at_3_max": 0.484014,
47
+ "nauc_ndcg_at_3_std": 0.279633,
48
+ "nauc_ndcg_at_3_diff1": 0.373273,
49
+ "nauc_ndcg_at_5_max": 0.499214,
50
+ "nauc_ndcg_at_5_std": 0.285193,
51
+ "nauc_ndcg_at_5_diff1": 0.377699,
52
+ "nauc_ndcg_at_10_max": 0.471546,
53
+ "nauc_ndcg_at_10_std": 0.303156,
54
+ "nauc_ndcg_at_10_diff1": 0.326637,
55
+ "nauc_ndcg_at_20_max": 0.452904,
56
+ "nauc_ndcg_at_20_std": 0.286146,
57
+ "nauc_ndcg_at_20_diff1": 0.311691,
58
+ "nauc_ndcg_at_100_max": 0.471783,
59
+ "nauc_ndcg_at_100_std": 0.276204,
60
+ "nauc_ndcg_at_100_diff1": 0.330981,
61
+ "nauc_ndcg_at_1000_max": 0.534989,
62
+ "nauc_ndcg_at_1000_std": 0.320777,
63
+ "nauc_ndcg_at_1000_diff1": 0.346119,
64
+ "nauc_map_at_1_max": 0.328405,
65
+ "nauc_map_at_1_std": -0.055909,
66
+ "nauc_map_at_1_diff1": 0.464563,
67
+ "nauc_map_at_3_max": 0.317683,
68
+ "nauc_map_at_3_std": -0.002782,
69
+ "nauc_map_at_3_diff1": 0.38562,
70
+ "nauc_map_at_5_max": 0.374515,
71
+ "nauc_map_at_5_std": 0.023936,
72
+ "nauc_map_at_5_diff1": 0.41153,
73
+ "nauc_map_at_10_max": 0.397591,
74
+ "nauc_map_at_10_std": 0.069354,
75
+ "nauc_map_at_10_diff1": 0.375102,
76
+ "nauc_map_at_20_max": 0.412683,
77
+ "nauc_map_at_20_std": 0.109156,
78
+ "nauc_map_at_20_diff1": 0.360845,
79
+ "nauc_map_at_100_max": 0.424984,
80
+ "nauc_map_at_100_std": 0.159183,
81
+ "nauc_map_at_100_diff1": 0.342464,
82
+ "nauc_map_at_1000_max": 0.43113,
83
+ "nauc_map_at_1000_std": 0.18875,
84
+ "nauc_map_at_1000_diff1": 0.333251,
85
+ "nauc_recall_at_1_max": 0.328405,
86
+ "nauc_recall_at_1_std": -0.055909,
87
+ "nauc_recall_at_1_diff1": 0.464563,
88
+ "nauc_recall_at_3_max": 0.266527,
89
+ "nauc_recall_at_3_std": -0.012704,
90
+ "nauc_recall_at_3_diff1": 0.333511,
91
+ "nauc_recall_at_5_max": 0.317243,
92
+ "nauc_recall_at_5_std": -0.002978,
93
+ "nauc_recall_at_5_diff1": 0.38356,
94
+ "nauc_recall_at_10_max": 0.307398,
95
+ "nauc_recall_at_10_std": 0.039968,
96
+ "nauc_recall_at_10_diff1": 0.313532,
97
+ "nauc_recall_at_20_max": 0.304815,
98
+ "nauc_recall_at_20_std": 0.072316,
99
+ "nauc_recall_at_20_diff1": 0.271323,
100
+ "nauc_recall_at_100_max": 0.278974,
101
+ "nauc_recall_at_100_std": 0.152896,
102
+ "nauc_recall_at_100_diff1": 0.216284,
103
+ "nauc_recall_at_1000_max": 0.242439,
104
+ "nauc_recall_at_1000_std": 0.170621,
105
+ "nauc_recall_at_1000_diff1": 0.144589,
106
+ "nauc_precision_at_1_max": 0.493121,
107
+ "nauc_precision_at_1_std": 0.211866,
108
+ "nauc_precision_at_1_diff1": 0.439682,
109
+ "nauc_precision_at_3_max": 0.482029,
110
+ "nauc_precision_at_3_std": 0.326316,
111
+ "nauc_precision_at_3_diff1": 0.30629,
112
+ "nauc_precision_at_5_max": 0.501486,
113
+ "nauc_precision_at_5_std": 0.359998,
114
+ "nauc_precision_at_5_diff1": 0.271906,
115
+ "nauc_precision_at_10_max": 0.437322,
116
+ "nauc_precision_at_10_std": 0.424984,
117
+ "nauc_precision_at_10_diff1": 0.151228,
118
+ "nauc_precision_at_20_max": 0.363724,
119
+ "nauc_precision_at_20_std": 0.454707,
120
+ "nauc_precision_at_20_diff1": 0.083594,
121
+ "nauc_precision_at_100_max": 0.213989,
122
+ "nauc_precision_at_100_std": 0.452484,
123
+ "nauc_precision_at_100_diff1": -0.018369,
124
+ "nauc_precision_at_1000_max": 0.062987,
125
+ "nauc_precision_at_1000_std": 0.315722,
126
+ "nauc_precision_at_1000_diff1": -0.063268,
127
+ "nauc_mrr_at_1_max": 0.493121,
128
+ "nauc_mrr_at_1_std": 0.211866,
129
+ "nauc_mrr_at_1_diff1": 0.439682,
130
+ "nauc_mrr_at_3_max": 0.502532,
131
+ "nauc_mrr_at_3_std": 0.249172,
132
+ "nauc_mrr_at_3_diff1": 0.410622,
133
+ "nauc_mrr_at_5_max": 0.515072,
134
+ "nauc_mrr_at_5_std": 0.250577,
135
+ "nauc_mrr_at_5_diff1": 0.409294,
136
+ "nauc_mrr_at_10_max": 0.510025,
137
+ "nauc_mrr_at_10_std": 0.25547,
138
+ "nauc_mrr_at_10_diff1": 0.404908,
139
+ "nauc_mrr_at_20_max": 0.510314,
140
+ "nauc_mrr_at_20_std": 0.258227,
141
+ "nauc_mrr_at_20_diff1": 0.404249,
142
+ "nauc_mrr_at_100_max": 0.511729,
143
+ "nauc_mrr_at_100_std": 0.258183,
144
+ "nauc_mrr_at_100_diff1": 0.405071,
145
+ "nauc_mrr_at_1000_max": 0.511509,
146
+ "nauc_mrr_at_1000_std": 0.257945,
147
+ "nauc_mrr_at_1000_diff1": 0.405027,
148
+ "main_score": 0.2663,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 9.661004543304443,
157
+ "kg_co2_emissions": null
158
+ }
mteb/NQ.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31",
3
+ "task_name": "NQ",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.28708,
9
+ "ndcg_at_3": 0.38322,
10
+ "ndcg_at_5": 0.42087,
11
+ "ndcg_at_10": 0.4532,
12
+ "ndcg_at_20": 0.4742,
13
+ "ndcg_at_100": 0.49995,
14
+ "ndcg_at_1000": 0.51263,
15
+ "map_at_1": 0.25702,
16
+ "map_at_3": 0.34911,
17
+ "map_at_5": 0.37137,
18
+ "map_at_10": 0.38587,
19
+ "map_at_20": 0.39226,
20
+ "map_at_100": 0.39638,
21
+ "map_at_1000": 0.39689,
22
+ "recall_at_1": 0.25702,
23
+ "recall_at_3": 0.45394,
24
+ "recall_at_5": 0.54089,
25
+ "recall_at_10": 0.63504,
26
+ "recall_at_20": 0.71323,
27
+ "recall_at_100": 0.84154,
28
+ "recall_at_1000": 0.93687,
29
+ "precision_at_1": 0.28708,
30
+ "precision_at_3": 0.17468,
31
+ "precision_at_5": 0.1263,
32
+ "precision_at_10": 0.07512,
33
+ "precision_at_20": 0.0425,
34
+ "precision_at_100": 0.01018,
35
+ "precision_at_1000": 0.00114,
36
+ "mrr_at_1": 0.28708,
37
+ "mrr_at_3": 0.377221,
38
+ "mrr_at_5": 0.396833,
39
+ "mrr_at_10": 0.409104,
40
+ "mrr_at_20": 0.41419,
41
+ "mrr_at_100": 0.417396,
42
+ "mrr_at_1000": 0.417789,
43
+ "nauc_ndcg_at_1_max": 0.250479,
44
+ "nauc_ndcg_at_1_std": 0.019234,
45
+ "nauc_ndcg_at_1_diff1": 0.335807,
46
+ "nauc_ndcg_at_3_max": 0.294583,
47
+ "nauc_ndcg_at_3_std": 0.050666,
48
+ "nauc_ndcg_at_3_diff1": 0.293529,
49
+ "nauc_ndcg_at_5_max": 0.307405,
50
+ "nauc_ndcg_at_5_std": 0.060516,
51
+ "nauc_ndcg_at_5_diff1": 0.287018,
52
+ "nauc_ndcg_at_10_max": 0.320593,
53
+ "nauc_ndcg_at_10_std": 0.084058,
54
+ "nauc_ndcg_at_10_diff1": 0.287482,
55
+ "nauc_ndcg_at_20_max": 0.328514,
56
+ "nauc_ndcg_at_20_std": 0.096083,
57
+ "nauc_ndcg_at_20_diff1": 0.286127,
58
+ "nauc_ndcg_at_100_max": 0.324329,
59
+ "nauc_ndcg_at_100_std": 0.105305,
60
+ "nauc_ndcg_at_100_diff1": 0.283013,
61
+ "nauc_ndcg_at_1000_max": 0.318549,
62
+ "nauc_ndcg_at_1000_std": 0.097215,
63
+ "nauc_ndcg_at_1000_diff1": 0.286643,
64
+ "nauc_map_at_1_max": 0.242088,
65
+ "nauc_map_at_1_std": 0.00276,
66
+ "nauc_map_at_1_diff1": 0.340467,
67
+ "nauc_map_at_3_max": 0.282923,
68
+ "nauc_map_at_3_std": 0.036139,
69
+ "nauc_map_at_3_diff1": 0.305555,
70
+ "nauc_map_at_5_max": 0.291093,
71
+ "nauc_map_at_5_std": 0.042332,
72
+ "nauc_map_at_5_diff1": 0.302144,
73
+ "nauc_map_at_10_max": 0.29688,
74
+ "nauc_map_at_10_std": 0.053188,
75
+ "nauc_map_at_10_diff1": 0.301168,
76
+ "nauc_map_at_20_max": 0.299107,
77
+ "nauc_map_at_20_std": 0.0566,
78
+ "nauc_map_at_20_diff1": 0.301049,
79
+ "nauc_map_at_100_max": 0.298118,
80
+ "nauc_map_at_100_std": 0.057722,
81
+ "nauc_map_at_100_diff1": 0.300482,
82
+ "nauc_map_at_1000_max": 0.297934,
83
+ "nauc_map_at_1000_std": 0.057475,
84
+ "nauc_map_at_1000_diff1": 0.300611,
85
+ "nauc_recall_at_1_max": 0.242088,
86
+ "nauc_recall_at_1_std": 0.00276,
87
+ "nauc_recall_at_1_diff1": 0.340467,
88
+ "nauc_recall_at_3_max": 0.314631,
89
+ "nauc_recall_at_3_std": 0.072352,
90
+ "nauc_recall_at_3_diff1": 0.260977,
91
+ "nauc_recall_at_5_max": 0.343255,
92
+ "nauc_recall_at_5_std": 0.094105,
93
+ "nauc_recall_at_5_diff1": 0.242641,
94
+ "nauc_recall_at_10_max": 0.389135,
95
+ "nauc_recall_at_10_std": 0.170459,
96
+ "nauc_recall_at_10_diff1": 0.240436,
97
+ "nauc_recall_at_20_max": 0.440327,
98
+ "nauc_recall_at_20_std": 0.243425,
99
+ "nauc_recall_at_20_diff1": 0.225185,
100
+ "nauc_recall_at_100_max": 0.488993,
101
+ "nauc_recall_at_100_std": 0.423469,
102
+ "nauc_recall_at_100_diff1": 0.170894,
103
+ "nauc_recall_at_1000_max": 0.535473,
104
+ "nauc_recall_at_1000_std": 0.585053,
105
+ "nauc_recall_at_1000_diff1": 0.161342,
106
+ "nauc_precision_at_1_max": 0.250479,
107
+ "nauc_precision_at_1_std": 0.019234,
108
+ "nauc_precision_at_1_diff1": 0.335807,
109
+ "nauc_precision_at_3_max": 0.312609,
110
+ "nauc_precision_at_3_std": 0.098992,
111
+ "nauc_precision_at_3_diff1": 0.232633,
112
+ "nauc_precision_at_5_max": 0.321341,
113
+ "nauc_precision_at_5_std": 0.121296,
114
+ "nauc_precision_at_5_diff1": 0.194865,
115
+ "nauc_precision_at_10_max": 0.316709,
116
+ "nauc_precision_at_10_std": 0.18397,
117
+ "nauc_precision_at_10_diff1": 0.157855,
118
+ "nauc_precision_at_20_max": 0.310475,
119
+ "nauc_precision_at_20_std": 0.225964,
120
+ "nauc_precision_at_20_diff1": 0.117234,
121
+ "nauc_precision_at_100_max": 0.217555,
122
+ "nauc_precision_at_100_std": 0.269504,
123
+ "nauc_precision_at_100_diff1": 0.029149,
124
+ "nauc_precision_at_1000_max": 0.094419,
125
+ "nauc_precision_at_1000_std": 0.203673,
126
+ "nauc_precision_at_1000_diff1": -0.029269,
127
+ "nauc_mrr_at_1_max": 0.250479,
128
+ "nauc_mrr_at_1_std": 0.019234,
129
+ "nauc_mrr_at_1_diff1": 0.335807,
130
+ "nauc_mrr_at_3_max": 0.285525,
131
+ "nauc_mrr_at_3_std": 0.053665,
132
+ "nauc_mrr_at_3_diff1": 0.296797,
133
+ "nauc_mrr_at_5_max": 0.290497,
134
+ "nauc_mrr_at_5_std": 0.058275,
135
+ "nauc_mrr_at_5_diff1": 0.293153,
136
+ "nauc_mrr_at_10_max": 0.29465,
137
+ "nauc_mrr_at_10_std": 0.065692,
138
+ "nauc_mrr_at_10_diff1": 0.295064,
139
+ "nauc_mrr_at_20_max": 0.295426,
140
+ "nauc_mrr_at_20_std": 0.066635,
141
+ "nauc_mrr_at_20_diff1": 0.295253,
142
+ "nauc_mrr_at_100_max": 0.294819,
143
+ "nauc_mrr_at_100_std": 0.067348,
144
+ "nauc_mrr_at_100_diff1": 0.295063,
145
+ "nauc_mrr_at_1000_max": 0.29466,
146
+ "nauc_mrr_at_1000_std": 0.067075,
147
+ "nauc_mrr_at_1000_diff1": 0.295171,
148
+ "main_score": 0.4532,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 2530.8694610595703,
157
+ "kg_co2_emissions": null
158
+ }
mteb/QuoraRetrieval.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "e4e08e0b7dbe3c8700f0daef558ff32256715259",
3
+ "task_name": "QuoraRetrieval",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.7981,
9
+ "ndcg_at_3": 0.83923,
10
+ "ndcg_at_5": 0.85516,
11
+ "ndcg_at_10": 0.86825,
12
+ "ndcg_at_20": 0.87562,
13
+ "ndcg_at_100": 0.88241,
14
+ "ndcg_at_1000": 0.88406,
15
+ "map_at_1": 0.69387,
16
+ "map_at_3": 0.80109,
17
+ "map_at_5": 0.81907,
18
+ "map_at_10": 0.82976,
19
+ "map_at_20": 0.83391,
20
+ "map_at_100": 0.83627,
21
+ "map_at_1000": 0.83648,
22
+ "recall_at_1": 0.69387,
23
+ "recall_at_3": 0.85762,
24
+ "recall_at_5": 0.90305,
25
+ "recall_at_10": 0.94156,
26
+ "recall_at_20": 0.9656,
27
+ "recall_at_100": 0.99099,
28
+ "recall_at_1000": 0.99905,
29
+ "precision_at_1": 0.7981,
30
+ "precision_at_3": 0.3661,
31
+ "precision_at_5": 0.24042,
32
+ "precision_at_10": 0.13102,
33
+ "precision_at_20": 0.06961,
34
+ "precision_at_100": 0.0151,
35
+ "precision_at_1000": 0.00156,
36
+ "mrr_at_1": 0.7979,
37
+ "mrr_at_3": 0.8504,
38
+ "mrr_at_5": 0.85789,
39
+ "mrr_at_10": 0.861217,
40
+ "mrr_at_20": 0.862272,
41
+ "mrr_at_100": 0.862594,
42
+ "mrr_at_1000": 0.862613,
43
+ "nauc_ndcg_at_1_max": 0.385323,
44
+ "nauc_ndcg_at_1_std": -0.352997,
45
+ "nauc_ndcg_at_1_diff1": 0.768904,
46
+ "nauc_ndcg_at_3_max": 0.358767,
47
+ "nauc_ndcg_at_3_std": -0.422362,
48
+ "nauc_ndcg_at_3_diff1": 0.750812,
49
+ "nauc_ndcg_at_5_max": 0.365901,
50
+ "nauc_ndcg_at_5_std": -0.437432,
51
+ "nauc_ndcg_at_5_diff1": 0.756063,
52
+ "nauc_ndcg_at_10_max": 0.371952,
53
+ "nauc_ndcg_at_10_std": -0.432721,
54
+ "nauc_ndcg_at_10_diff1": 0.758216,
55
+ "nauc_ndcg_at_20_max": 0.375014,
56
+ "nauc_ndcg_at_20_std": -0.419042,
57
+ "nauc_ndcg_at_20_diff1": 0.75704,
58
+ "nauc_ndcg_at_100_max": 0.380351,
59
+ "nauc_ndcg_at_100_std": -0.39988,
60
+ "nauc_ndcg_at_100_diff1": 0.756946,
61
+ "nauc_ndcg_at_1000_max": 0.380961,
62
+ "nauc_ndcg_at_1000_std": -0.395691,
63
+ "nauc_ndcg_at_1000_diff1": 0.756467,
64
+ "nauc_map_at_1_max": 0.276589,
65
+ "nauc_map_at_1_std": -0.386526,
66
+ "nauc_map_at_1_diff1": 0.7886,
67
+ "nauc_map_at_3_max": 0.332655,
68
+ "nauc_map_at_3_std": -0.448285,
69
+ "nauc_map_at_3_diff1": 0.762881,
70
+ "nauc_map_at_5_max": 0.348765,
71
+ "nauc_map_at_5_std": -0.448588,
72
+ "nauc_map_at_5_diff1": 0.761948,
73
+ "nauc_map_at_10_max": 0.357223,
74
+ "nauc_map_at_10_std": -0.437594,
75
+ "nauc_map_at_10_diff1": 0.760072,
76
+ "nauc_map_at_20_max": 0.360747,
77
+ "nauc_map_at_20_std": -0.427652,
78
+ "nauc_map_at_20_diff1": 0.75873,
79
+ "nauc_map_at_100_max": 0.363031,
80
+ "nauc_map_at_100_std": -0.420231,
81
+ "nauc_map_at_100_diff1": 0.758438,
82
+ "nauc_map_at_1000_max": 0.363384,
83
+ "nauc_map_at_1000_std": -0.41938,
84
+ "nauc_map_at_1000_diff1": 0.758393,
85
+ "nauc_recall_at_1_max": 0.276589,
86
+ "nauc_recall_at_1_std": -0.386526,
87
+ "nauc_recall_at_1_diff1": 0.7886,
88
+ "nauc_recall_at_3_max": 0.29953,
89
+ "nauc_recall_at_3_std": -0.507123,
90
+ "nauc_recall_at_3_diff1": 0.723936,
91
+ "nauc_recall_at_5_max": 0.318398,
92
+ "nauc_recall_at_5_std": -0.564493,
93
+ "nauc_recall_at_5_diff1": 0.717385,
94
+ "nauc_recall_at_10_max": 0.330489,
95
+ "nauc_recall_at_10_std": -0.596467,
96
+ "nauc_recall_at_10_diff1": 0.714574,
97
+ "nauc_recall_at_20_max": 0.34185,
98
+ "nauc_recall_at_20_std": -0.570183,
99
+ "nauc_recall_at_20_diff1": 0.707112,
100
+ "nauc_recall_at_100_max": 0.426654,
101
+ "nauc_recall_at_100_std": -0.408769,
102
+ "nauc_recall_at_100_diff1": 0.720717,
103
+ "nauc_recall_at_1000_max": 0.441165,
104
+ "nauc_recall_at_1000_std": 0.174422,
105
+ "nauc_recall_at_1000_diff1": 0.582066,
106
+ "nauc_precision_at_1_max": 0.385323,
107
+ "nauc_precision_at_1_std": -0.352997,
108
+ "nauc_precision_at_1_diff1": 0.768904,
109
+ "nauc_precision_at_3_max": 0.115283,
110
+ "nauc_precision_at_3_std": 0.034073,
111
+ "nauc_precision_at_3_diff1": -0.11805,
112
+ "nauc_precision_at_5_max": 0.067543,
113
+ "nauc_precision_at_5_std": 0.137509,
114
+ "nauc_precision_at_5_diff1": -0.254885,
115
+ "nauc_precision_at_10_max": 0.025493,
116
+ "nauc_precision_at_10_std": 0.236938,
117
+ "nauc_precision_at_10_diff1": -0.342161,
118
+ "nauc_precision_at_20_max": -0.001939,
119
+ "nauc_precision_at_20_std": 0.310031,
120
+ "nauc_precision_at_20_diff1": -0.388267,
121
+ "nauc_precision_at_100_max": -0.02205,
122
+ "nauc_precision_at_100_std": 0.38886,
123
+ "nauc_precision_at_100_diff1": -0.41661,
124
+ "nauc_precision_at_1000_max": -0.027452,
125
+ "nauc_precision_at_1000_std": 0.411492,
126
+ "nauc_precision_at_1000_diff1": -0.424162,
127
+ "nauc_mrr_at_1_max": 0.383192,
128
+ "nauc_mrr_at_1_std": -0.353824,
129
+ "nauc_mrr_at_1_diff1": 0.769376,
130
+ "nauc_mrr_at_3_max": 0.386992,
131
+ "nauc_mrr_at_3_std": -0.374313,
132
+ "nauc_mrr_at_3_diff1": 0.758136,
133
+ "nauc_mrr_at_5_max": 0.389249,
134
+ "nauc_mrr_at_5_std": -0.377732,
135
+ "nauc_mrr_at_5_diff1": 0.760258,
136
+ "nauc_mrr_at_10_max": 0.390148,
137
+ "nauc_mrr_at_10_std": -0.375749,
138
+ "nauc_mrr_at_10_diff1": 0.761165,
139
+ "nauc_mrr_at_20_max": 0.389778,
140
+ "nauc_mrr_at_20_std": -0.373585,
141
+ "nauc_mrr_at_20_diff1": 0.761091,
142
+ "nauc_mrr_at_100_max": 0.389507,
143
+ "nauc_mrr_at_100_std": -0.373448,
144
+ "nauc_mrr_at_100_diff1": 0.761166,
145
+ "nauc_mrr_at_1000_max": 0.389482,
146
+ "nauc_mrr_at_1000_std": -0.373441,
147
+ "nauc_mrr_at_1000_diff1": 0.76116,
148
+ "main_score": 0.86825,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 304.4700849056244,
157
+ "kg_co2_emissions": null
158
+ }
mteb/RedditClustering.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "24640382cdbf8abc73003fb0fa6d111a705499eb",
3
+ "task_name": "RedditClustering",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "v_measure": 0.411257,
9
+ "v_measure_std": 0.039637,
10
+ "v_measures": [
11
+ 0.47826,
12
+ 0.463098,
13
+ 0.378164,
14
+ 0.408031,
15
+ 0.375972,
16
+ 0.408828,
17
+ 0.442864,
18
+ 0.366117,
19
+ 0.364104,
20
+ 0.38536,
21
+ 0.404472,
22
+ 0.412944,
23
+ 0.391103,
24
+ 0.42435,
25
+ 0.401238,
26
+ 0.36973,
27
+ 0.433922,
28
+ 0.482263,
29
+ 0.399745,
30
+ 0.386039,
31
+ 0.37345,
32
+ 0.392968,
33
+ 0.523073,
34
+ 0.426964,
35
+ 0.388366
36
+ ],
37
+ "main_score": 0.411257,
38
+ "hf_subset": "default",
39
+ "languages": [
40
+ "eng-Latn"
41
+ ]
42
+ }
43
+ ]
44
+ },
45
+ "evaluation_time": 178.5294632911682,
46
+ "kg_co2_emissions": null
47
+ }
mteb/RedditClusteringP2P.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "385e3cb46b4cfa89021f56c4380204149d0efe33",
3
+ "task_name": "RedditClusteringP2P",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "v_measure": 0.513872,
9
+ "v_measure_std": 0.116705,
10
+ "v_measures": [
11
+ 0.563847,
12
+ 0.569759,
13
+ 0.580057,
14
+ 0.339426,
15
+ 0.569614,
16
+ 0.511353,
17
+ 0.24021,
18
+ 0.608951,
19
+ 0.56097,
20
+ 0.594533
21
+ ],
22
+ "main_score": 0.513872,
23
+ "hf_subset": "default",
24
+ "languages": [
25
+ "eng-Latn"
26
+ ]
27
+ }
28
+ ]
29
+ },
30
+ "evaluation_time": 662.1253287792206,
31
+ "kg_co2_emissions": null
32
+ }
mteb/SCIDOCS.json ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88",
3
+ "task_name": "SCIDOCS",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "ndcg_at_1": 0.173,
9
+ "ndcg_at_3": 0.13497,
10
+ "ndcg_at_5": 0.11591,
11
+ "ndcg_at_10": 0.13679,
12
+ "ndcg_at_20": 0.15566,
13
+ "ndcg_at_100": 0.19218,
14
+ "ndcg_at_1000": 0.23768,
15
+ "map_at_1": 0.0349,
16
+ "map_at_3": 0.05844,
17
+ "map_at_5": 0.069,
18
+ "map_at_10": 0.07798,
19
+ "map_at_20": 0.08404,
20
+ "map_at_100": 0.09027,
21
+ "map_at_1000": 0.09238,
22
+ "recall_at_1": 0.0349,
23
+ "recall_at_3": 0.0747,
24
+ "recall_at_5": 0.10045,
25
+ "recall_at_10": 0.1389,
26
+ "recall_at_20": 0.18338,
27
+ "recall_at_100": 0.3019,
28
+ "recall_at_1000": 0.52682,
29
+ "precision_at_1": 0.173,
30
+ "precision_at_3": 0.12333,
31
+ "precision_at_5": 0.0992,
32
+ "precision_at_10": 0.0685,
33
+ "precision_at_20": 0.0452,
34
+ "precision_at_100": 0.01486,
35
+ "precision_at_1000": 0.00259,
36
+ "mrr_at_1": 0.173,
37
+ "mrr_at_3": 0.231667,
38
+ "mrr_at_5": 0.244817,
39
+ "mrr_at_10": 0.256625,
40
+ "mrr_at_20": 0.262569,
41
+ "mrr_at_100": 0.267332,
42
+ "mrr_at_1000": 0.268125,
43
+ "nauc_ndcg_at_1_max": 0.242005,
44
+ "nauc_ndcg_at_1_std": 0.119834,
45
+ "nauc_ndcg_at_1_diff1": 0.24468,
46
+ "nauc_ndcg_at_3_max": 0.29079,
47
+ "nauc_ndcg_at_3_std": 0.16193,
48
+ "nauc_ndcg_at_3_diff1": 0.212417,
49
+ "nauc_ndcg_at_5_max": 0.302191,
50
+ "nauc_ndcg_at_5_std": 0.183763,
51
+ "nauc_ndcg_at_5_diff1": 0.194039,
52
+ "nauc_ndcg_at_10_max": 0.307147,
53
+ "nauc_ndcg_at_10_std": 0.215882,
54
+ "nauc_ndcg_at_10_diff1": 0.177705,
55
+ "nauc_ndcg_at_20_max": 0.311477,
56
+ "nauc_ndcg_at_20_std": 0.2469,
57
+ "nauc_ndcg_at_20_diff1": 0.169089,
58
+ "nauc_ndcg_at_100_max": 0.321562,
59
+ "nauc_ndcg_at_100_std": 0.30425,
60
+ "nauc_ndcg_at_100_diff1": 0.171303,
61
+ "nauc_ndcg_at_1000_max": 0.324754,
62
+ "nauc_ndcg_at_1000_std": 0.325746,
63
+ "nauc_ndcg_at_1000_diff1": 0.174281,
64
+ "nauc_map_at_1_max": 0.244036,
65
+ "nauc_map_at_1_std": 0.122252,
66
+ "nauc_map_at_1_diff1": 0.245346,
67
+ "nauc_map_at_3_max": 0.30043,
68
+ "nauc_map_at_3_std": 0.162703,
69
+ "nauc_map_at_3_diff1": 0.218286,
70
+ "nauc_map_at_5_max": 0.315919,
71
+ "nauc_map_at_5_std": 0.186765,
72
+ "nauc_map_at_5_diff1": 0.191174,
73
+ "nauc_map_at_10_max": 0.321404,
74
+ "nauc_map_at_10_std": 0.210341,
75
+ "nauc_map_at_10_diff1": 0.183068,
76
+ "nauc_map_at_20_max": 0.325777,
77
+ "nauc_map_at_20_std": 0.230543,
78
+ "nauc_map_at_20_diff1": 0.175999,
79
+ "nauc_map_at_100_max": 0.32757,
80
+ "nauc_map_at_100_std": 0.252143,
81
+ "nauc_map_at_100_diff1": 0.175648,
82
+ "nauc_map_at_1000_max": 0.32836,
83
+ "nauc_map_at_1000_std": 0.256243,
84
+ "nauc_map_at_1000_diff1": 0.176171,
85
+ "nauc_recall_at_1_max": 0.244036,
86
+ "nauc_recall_at_1_std": 0.122252,
87
+ "nauc_recall_at_1_diff1": 0.245346,
88
+ "nauc_recall_at_3_max": 0.304984,
89
+ "nauc_recall_at_3_std": 0.17664,
90
+ "nauc_recall_at_3_diff1": 0.193246,
91
+ "nauc_recall_at_5_max": 0.308366,
92
+ "nauc_recall_at_5_std": 0.203564,
93
+ "nauc_recall_at_5_diff1": 0.156459,
94
+ "nauc_recall_at_10_max": 0.303941,
95
+ "nauc_recall_at_10_std": 0.251872,
96
+ "nauc_recall_at_10_diff1": 0.122576,
97
+ "nauc_recall_at_20_max": 0.294931,
98
+ "nauc_recall_at_20_std": 0.298445,
99
+ "nauc_recall_at_20_diff1": 0.102911,
100
+ "nauc_recall_at_100_max": 0.295687,
101
+ "nauc_recall_at_100_std": 0.404675,
102
+ "nauc_recall_at_100_diff1": 0.108211,
103
+ "nauc_recall_at_1000_max": 0.261128,
104
+ "nauc_recall_at_1000_std": 0.42622,
105
+ "nauc_recall_at_1000_diff1": 0.095162,
106
+ "nauc_precision_at_1_max": 0.242005,
107
+ "nauc_precision_at_1_std": 0.119834,
108
+ "nauc_precision_at_1_diff1": 0.24468,
109
+ "nauc_precision_at_3_max": 0.305687,
110
+ "nauc_precision_at_3_std": 0.175571,
111
+ "nauc_precision_at_3_diff1": 0.193951,
112
+ "nauc_precision_at_5_max": 0.310911,
113
+ "nauc_precision_at_5_std": 0.203866,
114
+ "nauc_precision_at_5_diff1": 0.160506,
115
+ "nauc_precision_at_10_max": 0.310221,
116
+ "nauc_precision_at_10_std": 0.253927,
117
+ "nauc_precision_at_10_diff1": 0.128608,
118
+ "nauc_precision_at_20_max": 0.300773,
119
+ "nauc_precision_at_20_std": 0.299155,
120
+ "nauc_precision_at_20_diff1": 0.108257,
121
+ "nauc_precision_at_100_max": 0.299943,
122
+ "nauc_precision_at_100_std": 0.4021,
123
+ "nauc_precision_at_100_diff1": 0.112121,
124
+ "nauc_precision_at_1000_max": 0.263822,
125
+ "nauc_precision_at_1000_std": 0.417393,
126
+ "nauc_precision_at_1000_diff1": 0.099425,
127
+ "nauc_mrr_at_1_max": 0.242005,
128
+ "nauc_mrr_at_1_std": 0.119834,
129
+ "nauc_mrr_at_1_diff1": 0.24468,
130
+ "nauc_mrr_at_3_max": 0.265063,
131
+ "nauc_mrr_at_3_std": 0.146784,
132
+ "nauc_mrr_at_3_diff1": 0.224465,
133
+ "nauc_mrr_at_5_max": 0.270678,
134
+ "nauc_mrr_at_5_std": 0.151935,
135
+ "nauc_mrr_at_5_diff1": 0.226012,
136
+ "nauc_mrr_at_10_max": 0.269741,
137
+ "nauc_mrr_at_10_std": 0.159622,
138
+ "nauc_mrr_at_10_diff1": 0.217963,
139
+ "nauc_mrr_at_20_max": 0.270563,
140
+ "nauc_mrr_at_20_std": 0.162419,
141
+ "nauc_mrr_at_20_diff1": 0.217493,
142
+ "nauc_mrr_at_100_max": 0.270795,
143
+ "nauc_mrr_at_100_std": 0.164246,
144
+ "nauc_mrr_at_100_diff1": 0.218387,
145
+ "nauc_mrr_at_1000_max": 0.270832,
146
+ "nauc_mrr_at_1000_std": 0.164069,
147
+ "nauc_mrr_at_1000_diff1": 0.218429,
148
+ "main_score": 0.13679,
149
+ "hf_subset": "default",
150
+ "languages": [
151
+ "eng-Latn"
152
+ ]
153
+ }
154
+ ]
155
+ },
156
+ "evaluation_time": 46.585543632507324,
157
+ "kg_co2_emissions": null
158
+ }
mteb/SICK-R.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "20a6d6f312dd54037fe07a32d58e5e168867909d",
3
+ "task_name": "SICK-R",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "pearson": 0.782689,
9
+ "spearman": 0.709342,
10
+ "cosine_pearson": 0.782689,
11
+ "cosine_spearman": 0.709342,
12
+ "manhattan_pearson": 0.747536,
13
+ "manhattan_spearman": 0.708905,
14
+ "euclidean_pearson": 0.747187,
15
+ "euclidean_spearman": 0.70888,
16
+ "main_score": 0.709342,
17
+ "hf_subset": "default",
18
+ "languages": [
19
+ "eng-Latn"
20
+ ]
21
+ }
22
+ ]
23
+ },
24
+ "evaluation_time": 8.33829402923584,
25
+ "kg_co2_emissions": null
26
+ }
mteb/STS12.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384",
3
+ "task_name": "STS12",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "pearson": 0.733449,
9
+ "spearman": 0.672036,
10
+ "cosine_pearson": 0.733449,
11
+ "cosine_spearman": 0.672036,
12
+ "manhattan_pearson": 0.685626,
13
+ "manhattan_spearman": 0.668326,
14
+ "euclidean_pearson": 0.684606,
15
+ "euclidean_spearman": 0.668238,
16
+ "main_score": 0.672036,
17
+ "hf_subset": "default",
18
+ "languages": [
19
+ "eng-Latn"
20
+ ]
21
+ }
22
+ ]
23
+ },
24
+ "evaluation_time": 2.7828357219696045,
25
+ "kg_co2_emissions": null
26
+ }
mteb/STS13.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca",
3
+ "task_name": "STS13",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "pearson": 0.773137,
9
+ "spearman": 0.774032,
10
+ "cosine_pearson": 0.773137,
11
+ "cosine_spearman": 0.774032,
12
+ "manhattan_pearson": 0.762551,
13
+ "manhattan_spearman": 0.766366,
14
+ "euclidean_pearson": 0.761261,
15
+ "euclidean_spearman": 0.765129,
16
+ "main_score": 0.774032,
17
+ "hf_subset": "default",
18
+ "languages": [
19
+ "eng-Latn"
20
+ ]
21
+ }
22
+ ]
23
+ },
24
+ "evaluation_time": 1.3985404968261719,
25
+ "kg_co2_emissions": null
26
+ }
mteb/STS14.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375",
3
+ "task_name": "STS14",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "pearson": 0.771255,
9
+ "spearman": 0.723476,
10
+ "cosine_pearson": 0.771255,
11
+ "cosine_spearman": 0.723476,
12
+ "manhattan_pearson": 0.741639,
13
+ "manhattan_spearman": 0.721268,
14
+ "euclidean_pearson": 0.74118,
15
+ "euclidean_spearman": 0.721061,
16
+ "main_score": 0.723476,
17
+ "hf_subset": "default",
18
+ "languages": [
19
+ "eng-Latn"
20
+ ]
21
+ }
22
+ ]
23
+ },
24
+ "evaluation_time": 3.2774949073791504,
25
+ "kg_co2_emissions": null
26
+ }
mteb/STS15.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3",
3
+ "task_name": "STS15",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "pearson": 0.81992,
9
+ "spearman": 0.824958,
10
+ "cosine_pearson": 0.81992,
11
+ "cosine_spearman": 0.824958,
12
+ "manhattan_pearson": 0.813437,
13
+ "manhattan_spearman": 0.819727,
14
+ "euclidean_pearson": 0.813543,
15
+ "euclidean_spearman": 0.819557,
16
+ "main_score": 0.824958,
17
+ "hf_subset": "default",
18
+ "languages": [
19
+ "eng-Latn"
20
+ ]
21
+ }
22
+ ]
23
+ },
24
+ "evaluation_time": 2.9901087284088135,
25
+ "kg_co2_emissions": null
26
+ }
mteb/STS16.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_revision": "4d8694f8f0e0100860b497b999b3dbed754a0513",
3
+ "task_name": "STS16",
4
+ "mteb_version": "1.25.1",
5
+ "scores": {
6
+ "test": [
7
+ {
8
+ "pearson": 0.78755,
9
+ "spearman": 0.79408,
10
+ "cosine_pearson": 0.78755,
11
+ "cosine_spearman": 0.79408,
12
+ "manhattan_pearson": 0.800336,
13
+ "manhattan_spearman": 0.805912,
14
+ "euclidean_pearson": 0.800315,
15
+ "euclidean_spearman": 0.805413,
16
+ "main_score": 0.79408,
17
+ "hf_subset": "default",
18
+ "languages": [
19
+ "eng-Latn"
20
+ ]
21
+ }
22
+ ]
23
+ },
24
+ "evaluation_time": 1.2775132656097412,
25
+ "kg_co2_emissions": null
26
+ }