diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -1,5596 +1,5606 @@ ---- -model-index: -- name: stella_en_400M_v5 - results: - - dataset: - config: en - name: MTEB AmazonCounterfactualClassification (en) - revision: e8379541af4e31359cca9fbcf4b00f2671dba205 - split: test - type: mteb/amazon_counterfactual - metrics: - - type: accuracy - value: 92.35820895522387 - - type: ap - value: 70.81322736988783 - - type: ap_weighted - value: 70.81322736988783 - - type: f1 - value: 88.9505466159595 - - type: f1_weighted - value: 92.68630932872613 - - type: main_score - value: 92.35820895522387 - task: - type: Classification - - dataset: - config: default - name: MTEB AmazonPolarityClassification - revision: e2d317d38cd51312af73b3d32a06d1a08b442046 - split: test - type: mteb/amazon_polarity - metrics: - - type: accuracy - value: 97.1945 - - type: ap - value: 96.08192192244094 - - type: ap_weighted - value: 96.08192192244094 - - type: f1 - value: 97.1936887167346 - - type: f1_weighted - value: 97.1936887167346 - - type: main_score - value: 97.1945 - task: - type: Classification - - dataset: - config: en - name: MTEB AmazonReviewsClassification (en) - revision: 1399c76144fd37290681b995c656ef9b2e06e26d - split: test - type: mteb/amazon_reviews_multi - metrics: - - type: accuracy - value: 59.528000000000006 - - type: f1 - value: 59.21016819840188 - - type: f1_weighted - value: 59.21016819840188 - - type: main_score - value: 59.528000000000006 - task: - type: Classification - - dataset: - config: default - name: MTEB ArguAna - revision: c22ab2a51041ffd869aaddef7af8d8215647e41a - split: test - type: mteb/arguana - metrics: - - type: main_score - value: 64.24 - - type: map_at_1 - value: 40.398 - - type: map_at_10 - value: 56.215 - - type: map_at_100 - value: 56.833999999999996 - - type: map_at_1000 - value: 56.835 - - type: map_at_20 - value: 56.747 - - type: map_at_3 - value: 52.181 - - type: map_at_5 - value: 54.628 - - type: mrr_at_1 - value: 41.25177809388336 - - type: mrr_at_10 - value: 56.570762491815216 - - type: mrr_at_100 - value: 57.17548614361504 - - type: mrr_at_1000 - value: 57.176650626377466 - - type: mrr_at_20 - value: 57.08916253512566 - - type: mrr_at_3 - value: 52.47747747747754 - - type: mrr_at_5 - value: 54.94547178757718 - - type: nauc_map_at_1000_diff1 - value: 22.408086887100158 - - type: nauc_map_at_1000_max - value: -8.730419096847543 - - type: nauc_map_at_1000_std - value: -17.789262741255737 - - type: nauc_map_at_100_diff1 - value: 22.407371684274025 - - type: nauc_map_at_100_max - value: -8.732263549026266 - - type: nauc_map_at_100_std - value: -17.79550515579994 - - type: nauc_map_at_10_diff1 - value: 21.925005073301246 - - type: nauc_map_at_10_max - value: -8.990323944492134 - - type: nauc_map_at_10_std - value: -18.199246301671458 - - type: nauc_map_at_1_diff1 - value: 26.23276644969203 - - type: nauc_map_at_1_max - value: -12.376511389571245 - - type: nauc_map_at_1_std - value: -18.11411715207284 - - type: nauc_map_at_20_diff1 - value: 22.32455790850922 - - type: nauc_map_at_20_max - value: -8.664671547236034 - - type: nauc_map_at_20_std - value: -17.8290016125137 - - type: nauc_map_at_3_diff1 - value: 22.395462147465064 - - type: nauc_map_at_3_max - value: -8.206580750918844 - - type: nauc_map_at_3_std - value: -17.604490446911484 - - type: nauc_map_at_5_diff1 - value: 21.95307379904799 - - type: nauc_map_at_5_max - value: -8.03958102978443 - - type: nauc_map_at_5_std - value: -17.36578866595004 - - type: nauc_mrr_at_1000_diff1 - value: 20.124236798365587 - - type: nauc_mrr_at_1000_max - value: -9.587376069575898 - - type: nauc_mrr_at_1000_std - value: -17.79191612151833 - - type: nauc_mrr_at_100_diff1 - value: 20.123612603474033 - - type: nauc_mrr_at_100_max - value: -9.589187218607831 - - type: nauc_mrr_at_100_std - value: -17.7981617777748 - - type: nauc_mrr_at_10_diff1 - value: 19.723683875738075 - - type: nauc_mrr_at_10_max - value: -9.774151729178815 - - type: nauc_mrr_at_10_std - value: -18.168668675495162 - - type: nauc_mrr_at_1_diff1 - value: 23.945332059908132 - - type: nauc_mrr_at_1_max - value: -12.260461466152819 - - type: nauc_mrr_at_1_std - value: -18.007194922921148 - - type: nauc_mrr_at_20_diff1 - value: 20.04819461810257 - - type: nauc_mrr_at_20_max - value: -9.518368283588936 - - type: nauc_mrr_at_20_std - value: -17.831608149836136 - - type: nauc_mrr_at_3_diff1 - value: 19.8571785245832 - - type: nauc_mrr_at_3_max - value: -9.464375021240478 - - type: nauc_mrr_at_3_std - value: -17.728533927330453 - - type: nauc_mrr_at_5_diff1 - value: 19.670313652167827 - - type: nauc_mrr_at_5_max - value: -8.966372585728434 - - type: nauc_mrr_at_5_std - value: -17.468955834324817 - - type: nauc_ndcg_at_1000_diff1 - value: 21.863049281767417 - - type: nauc_ndcg_at_1000_max - value: -8.18698520924057 - - type: nauc_ndcg_at_1000_std - value: -17.634483364794804 - - type: nauc_ndcg_at_100_diff1 - value: 21.849924385738586 - - type: nauc_ndcg_at_100_max - value: -8.226437560889345 - - type: nauc_ndcg_at_100_std - value: -17.774648478087002 - - type: nauc_ndcg_at_10_diff1 - value: 19.888395590413573 - - type: nauc_ndcg_at_10_max - value: -8.968706085632382 - - type: nauc_ndcg_at_10_std - value: -19.31386964628115 - - type: nauc_ndcg_at_1_diff1 - value: 26.23276644969203 - - type: nauc_ndcg_at_1_max - value: -12.376511389571245 - - type: nauc_ndcg_at_1_std - value: -18.11411715207284 - - type: nauc_ndcg_at_20_diff1 - value: 21.38413342416933 - - type: nauc_ndcg_at_20_max - value: -7.636238194084164 - - type: nauc_ndcg_at_20_std - value: -17.946390844693028 - - type: nauc_ndcg_at_3_diff1 - value: 21.29169165029195 - - type: nauc_ndcg_at_3_max - value: -6.793840499730093 - - type: nauc_ndcg_at_3_std - value: -17.52359001586737 - - type: nauc_ndcg_at_5_diff1 - value: 20.238297656671364 - - type: nauc_ndcg_at_5_max - value: -6.424992706950072 - - type: nauc_ndcg_at_5_std - value: -17.082391132291356 - - type: nauc_precision_at_1000_diff1 - value: -7.05195108528572 - - type: nauc_precision_at_1000_max - value: 34.439879624882145 - - type: nauc_precision_at_1000_std - value: 68.72436351659353 - - type: nauc_precision_at_100_diff1 - value: -2.769464113932605 - - type: nauc_precision_at_100_max - value: 9.89562961226698 - - type: nauc_precision_at_100_std - value: -0.5880967482224028 - - type: nauc_precision_at_10_diff1 - value: 2.1371544726832323 - - type: nauc_precision_at_10_max - value: -11.93051325147756 - - type: nauc_precision_at_10_std - value: -30.83144187392059 - - type: nauc_precision_at_1_diff1 - value: 26.23276644969203 - - type: nauc_precision_at_1_max - value: -12.376511389571245 - - type: nauc_precision_at_1_std - value: -18.11411715207284 - - type: nauc_precision_at_20_diff1 - value: 3.780146814257504 - - type: nauc_precision_at_20_max - value: 17.06527540214615 - - type: nauc_precision_at_20_std - value: -20.36832563035565 - - type: nauc_precision_at_3_diff1 - value: 17.63894384012077 - - type: nauc_precision_at_3_max - value: -2.0220490624638887 - - type: nauc_precision_at_3_std - value: -17.285601413493918 - - type: nauc_precision_at_5_diff1 - value: 12.557855071944601 - - type: nauc_precision_at_5_max - value: 0.5840236463956658 - - type: nauc_precision_at_5_std - value: -15.827224420217846 - - type: nauc_recall_at_1000_diff1 - value: -7.051951085286463 - - type: nauc_recall_at_1000_max - value: 34.43987962487738 - - type: nauc_recall_at_1000_std - value: 68.724363516591 - - type: nauc_recall_at_100_diff1 - value: -2.769464113930314 - - type: nauc_recall_at_100_max - value: 9.895629612270017 - - type: nauc_recall_at_100_std - value: -0.58809674821745 - - type: nauc_recall_at_10_diff1 - value: 2.1371544726834495 - - type: nauc_recall_at_10_max - value: -11.930513251477253 - - type: nauc_recall_at_10_std - value: -30.83144187392047 - - type: nauc_recall_at_1_diff1 - value: 26.23276644969203 - - type: nauc_recall_at_1_max - value: -12.376511389571245 - - type: nauc_recall_at_1_std - value: -18.11411715207284 - - type: nauc_recall_at_20_diff1 - value: 3.7801468142575922 - - type: nauc_recall_at_20_max - value: 17.0652754021456 - - type: nauc_recall_at_20_std - value: -20.36832563035559 - - type: nauc_recall_at_3_diff1 - value: 17.63894384012074 - - type: nauc_recall_at_3_max - value: -2.02204906246383 - - type: nauc_recall_at_3_std - value: -17.28560141349386 - - type: nauc_recall_at_5_diff1 - value: 12.55785507194463 - - type: nauc_recall_at_5_max - value: 0.5840236463957296 - - type: nauc_recall_at_5_std - value: -15.827224420217856 - - type: ndcg_at_1 - value: 40.398 - - type: ndcg_at_10 - value: 64.24 - - type: ndcg_at_100 - value: 66.631 - - type: ndcg_at_1000 - value: 66.65100000000001 - - type: ndcg_at_20 - value: 66.086 - - type: ndcg_at_3 - value: 55.938 - - type: ndcg_at_5 - value: 60.370000000000005 - - type: precision_at_1 - value: 40.398 - - type: precision_at_10 - value: 8.962 - - type: precision_at_100 - value: 0.9950000000000001 - - type: precision_at_1000 - value: 0.1 - - type: precision_at_20 - value: 4.836 - - type: precision_at_3 - value: 22.262 - - type: precision_at_5 - value: 15.519 - - type: recall_at_1 - value: 40.398 - - type: recall_at_10 - value: 89.616 - - type: recall_at_100 - value: 99.502 - - type: recall_at_1000 - value: 99.644 - - type: recall_at_20 - value: 96.72800000000001 - - type: recall_at_3 - value: 66.78500000000001 - - type: recall_at_5 - value: 77.596 - task: - type: Retrieval - - dataset: - config: default - name: MTEB ArxivClusteringP2P - revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d - split: test - type: mteb/arxiv-clustering-p2p - metrics: - - type: main_score - value: 55.1564333205451 - - type: v_measure - value: 55.1564333205451 - - type: v_measure_std - value: 14.696883012214512 - task: - type: Clustering - - dataset: - config: default - name: MTEB ArxivClusteringS2S - revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 - split: test - type: mteb/arxiv-clustering-s2s - metrics: - - type: main_score - value: 49.823698316694795 - - type: v_measure - value: 49.823698316694795 - - type: v_measure_std - value: 14.951660654298186 - task: - type: Clustering - - dataset: - config: default - name: MTEB AskUbuntuDupQuestions - revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 - split: test - type: mteb/askubuntudupquestions-reranking - metrics: - - type: main_score - value: 66.15294503553424 - - type: map - value: 66.15294503553424 - - type: mrr - value: 78.53438420612935 - - type: nAUC_map_diff1 - value: 12.569697092717997 - - type: nAUC_map_max - value: 21.50670312412572 - - type: nAUC_map_std - value: 16.943786429229064 - - type: nAUC_mrr_diff1 - value: 15.590272897361238 - - type: nAUC_mrr_max - value: 34.96072022474653 - - type: nAUC_mrr_std - value: 21.649217605241045 - task: - type: Reranking - - dataset: - config: default - name: MTEB BIOSSES - revision: d3fb88f8f02e40887cd149695127462bbcf29b4a - split: test - type: mteb/biosses-sts - metrics: - - type: cosine_pearson - value: 85.7824546319275 - - type: cosine_spearman - value: 83.29587385660628 - - type: euclidean_pearson - value: 84.58764190565167 - - type: euclidean_spearman - value: 83.30069324352772 - - type: main_score - value: 83.29587385660628 - - type: manhattan_pearson - value: 84.95996839947179 - - type: manhattan_spearman - value: 83.87480271054358 - - type: pearson - value: 85.7824546319275 - - type: spearman - value: 83.29587385660628 - task: - type: STS - - dataset: - config: default - name: MTEB Banking77Classification - revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 - split: test - type: mteb/banking77 - metrics: - - type: accuracy - value: 89.30194805194806 - - type: f1 - value: 89.26182507266391 - - type: f1_weighted - value: 89.26182507266391 - - type: main_score - value: 89.30194805194806 - task: - type: Classification - - dataset: - config: default - name: MTEB BiorxivClusteringP2P - revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 - split: test - type: mteb/biorxiv-clustering-p2p - metrics: - - type: main_score - value: 50.67972171889736 - - type: v_measure - value: 50.67972171889736 - - type: v_measure_std - value: 0.7687409980036303 - task: - type: Clustering - - dataset: - config: default - name: MTEB BiorxivClusteringS2S - revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 - split: test - type: mteb/biorxiv-clustering-s2s - metrics: - - type: main_score - value: 45.80539715556144 - - type: v_measure - value: 45.80539715556144 - - type: v_measure_std - value: 0.9601346216579142 - task: - type: Clustering - - dataset: - config: default - name: MTEB CQADupstackRetrieval - revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 - split: test - type: mteb/cqadupstack - metrics: - - type: main_score - value: 44.361250000000005 - - type: map_at_1 - value: 28.304499999999997 - - type: map_at_10 - value: 38.54841666666666 - - type: map_at_100 - value: 39.83141666666667 - - type: map_at_1000 - value: 39.944750000000006 - - type: map_at_20 - value: 39.25341666666667 - - type: map_at_3 - value: 35.406749999999995 - - type: map_at_5 - value: 37.15558333333333 - - type: mrr_at_1 - value: 34.09077232860122 - - type: mrr_at_10 - value: 43.15445393211421 - - type: mrr_at_100 - value: 43.98645286848257 - - type: mrr_at_1000 - value: 44.037631313469404 - - type: mrr_at_20 - value: 43.64045813249614 - - type: mrr_at_3 - value: 40.674138648480486 - - type: mrr_at_5 - value: 42.106251182620255 - - type: nauc_map_at_1000_diff1 - value: 46.250011739434996 - - type: nauc_map_at_1000_max - value: 30.13664446260598 - - type: nauc_map_at_1000_std - value: 5.422301791618935 - - type: nauc_map_at_100_diff1 - value: 46.253631351999395 - - type: nauc_map_at_100_max - value: 30.12612918885181 - - type: nauc_map_at_100_std - value: 5.367077019987172 - - type: nauc_map_at_10_diff1 - value: 46.328171341741346 - - type: nauc_map_at_10_max - value: 29.80274612581464 - - type: nauc_map_at_10_std - value: 4.62996685176396 - - type: nauc_map_at_1_diff1 - value: 51.56118117729493 - - type: nauc_map_at_1_max - value: 27.94885243863768 - - type: nauc_map_at_1_std - value: 1.700366508927356 - - type: nauc_map_at_20_diff1 - value: 46.286750260299094 - - type: nauc_map_at_20_max - value: 29.979205290353278 - - type: nauc_map_at_20_std - value: 5.010588412441873 - - type: nauc_map_at_3_diff1 - value: 47.10018183619064 - - type: nauc_map_at_3_max - value: 29.062318206078753 - - type: nauc_map_at_3_std - value: 3.2235696254694197 - - type: nauc_map_at_5_diff1 - value: 46.41971733050039 - - type: nauc_map_at_5_max - value: 29.456798617695657 - - type: nauc_map_at_5_std - value: 4.0921691023077145 - - type: nauc_mrr_at_1000_diff1 - value: 45.88888977975723 - - type: nauc_mrr_at_1000_max - value: 32.162138978089544 - - type: nauc_mrr_at_1000_std - value: 6.2811943424217915 - - type: nauc_mrr_at_100_diff1 - value: 45.87480433011124 - - type: nauc_mrr_at_100_max - value: 32.16011334212834 - - type: nauc_mrr_at_100_std - value: 6.2865717772421785 - - type: nauc_mrr_at_10_diff1 - value: 45.849652904658825 - - type: nauc_mrr_at_10_max - value: 32.13847916232293 - - type: nauc_mrr_at_10_std - value: 6.105718728141999 - - type: nauc_mrr_at_1_diff1 - value: 51.013730325062156 - - type: nauc_mrr_at_1_max - value: 32.77457396492779 - - type: nauc_mrr_at_1_std - value: 4.415684893471724 - - type: nauc_mrr_at_20_diff1 - value: 45.86663046255274 - - type: nauc_mrr_at_20_max - value: 32.15219360697865 - - type: nauc_mrr_at_20_std - value: 6.19603046412763 - - type: nauc_mrr_at_3_diff1 - value: 46.522376582423185 - - type: nauc_mrr_at_3_max - value: 32.18259009733714 - - type: nauc_mrr_at_3_std - value: 5.288000648220897 - - type: nauc_mrr_at_5_diff1 - value: 45.86611481369745 - - type: nauc_mrr_at_5_max - value: 32.14261639054921 - - type: nauc_mrr_at_5_std - value: 5.8811238177073735 - - type: nauc_ndcg_at_1000_diff1 - value: 44.5055097547565 - - type: nauc_ndcg_at_1000_max - value: 31.149682057975458 - - type: nauc_ndcg_at_1000_std - value: 8.157937194901333 - - type: nauc_ndcg_at_100_diff1 - value: 44.12398363638596 - - type: nauc_ndcg_at_100_max - value: 30.878064321409994 - - type: nauc_ndcg_at_100_std - value: 8.40493441452808 - - type: nauc_ndcg_at_10_diff1 - value: 44.200093505221474 - - type: nauc_ndcg_at_10_max - value: 30.15267107733158 - - type: nauc_ndcg_at_10_std - value: 6.407495361566107 - - type: nauc_ndcg_at_1_diff1 - value: 51.013730325062156 - - type: nauc_ndcg_at_1_max - value: 32.77457396492779 - - type: nauc_ndcg_at_1_std - value: 4.415684893471724 - - type: nauc_ndcg_at_20_diff1 - value: 44.16988321564116 - - type: nauc_ndcg_at_20_max - value: 30.333532500651213 - - type: nauc_ndcg_at_20_std - value: 7.10024701386895 - - type: nauc_ndcg_at_3_diff1 - value: 45.35982873879988 - - type: nauc_ndcg_at_3_max - value: 30.288312457948702 - - type: nauc_ndcg_at_3_std - value: 4.653900898293395 - - type: nauc_ndcg_at_5_diff1 - value: 44.324558115380185 - - type: nauc_ndcg_at_5_max - value: 30.048149698941373 - - type: nauc_ndcg_at_5_std - value: 5.6684459618413205 - - type: nauc_precision_at_1000_diff1 - value: -7.282175798304458 - - type: nauc_precision_at_1000_max - value: 7.820142031765352 - - type: nauc_precision_at_1000_std - value: 11.736131836431172 - - type: nauc_precision_at_100_diff1 - value: 1.0222940256506976 - - type: nauc_precision_at_100_max - value: 16.12346497070298 - - type: nauc_precision_at_100_std - value: 18.202607395247874 - - type: nauc_precision_at_10_diff1 - value: 18.289439185857837 - - type: nauc_precision_at_10_max - value: 26.116517399154375 - - type: nauc_precision_at_10_std - value: 13.921214069982302 - - type: nauc_precision_at_1_diff1 - value: 51.013730325062156 - - type: nauc_precision_at_1_max - value: 32.77457396492779 - - type: nauc_precision_at_1_std - value: 4.415684893471724 - - type: nauc_precision_at_20_diff1 - value: 12.365165405210886 - - type: nauc_precision_at_20_max - value: 22.946297258937367 - - type: nauc_precision_at_20_std - value: 16.13862870358933 - - type: nauc_precision_at_3_diff1 - value: 32.063423642849685 - - type: nauc_precision_at_3_max - value: 30.140965811989407 - - type: nauc_precision_at_3_std - value: 8.501746262550146 - - type: nauc_precision_at_5_diff1 - value: 24.777203357717948 - - type: nauc_precision_at_5_max - value: 28.401579566848472 - - type: nauc_precision_at_5_std - value: 11.643246774390914 - - type: nauc_recall_at_1000_diff1 - value: 30.04216463401409 - - type: nauc_recall_at_1000_max - value: 34.98067760563842 - - type: nauc_recall_at_1000_std - value: 48.01453905250591 - - type: nauc_recall_at_100_diff1 - value: 31.193415507513972 - - type: nauc_recall_at_100_max - value: 28.69740149270981 - - type: nauc_recall_at_100_std - value: 25.20960758920368 - - type: nauc_recall_at_10_diff1 - value: 36.18870823636506 - - type: nauc_recall_at_10_max - value: 26.005625231341238 - - type: nauc_recall_at_10_std - value: 8.891983977041376 - - type: nauc_recall_at_1_diff1 - value: 51.56118117729493 - - type: nauc_recall_at_1_max - value: 27.94885243863768 - - type: nauc_recall_at_1_std - value: 1.700366508927356 - - type: nauc_recall_at_20_diff1 - value: 34.93996118564803 - - type: nauc_recall_at_20_max - value: 26.149961715956138 - - type: nauc_recall_at_20_std - value: 12.0657502367633 - - type: nauc_recall_at_3_diff1 - value: 40.80743946709512 - - type: nauc_recall_at_3_max - value: 26.443127773025783 - - type: nauc_recall_at_3_std - value: 3.7011448604241477 - - type: nauc_recall_at_5_diff1 - value: 37.608535157055776 - - type: nauc_recall_at_5_max - value: 26.168016189725822 - - type: nauc_recall_at_5_std - value: 6.344191564595316 - - type: ndcg_at_1 - value: 34.09083333333333 - - type: ndcg_at_10 - value: 44.361250000000005 - - type: ndcg_at_100 - value: 49.586166666666664 - - type: ndcg_at_1000 - value: 51.623583333333336 - - type: ndcg_at_20 - value: 46.40158333333333 - - type: ndcg_at_3 - value: 39.27733333333333 - - type: ndcg_at_5 - value: 41.662333333333336 - - type: precision_at_1 - value: 34.09083333333333 - - type: precision_at_10 - value: 7.957000000000002 - - type: precision_at_100 - value: 1.2521666666666669 - - type: precision_at_1000 - value: 0.16125 - - type: precision_at_20 - value: 4.6755 - - type: precision_at_3 - value: 18.402083333333334 - - type: precision_at_5 - value: 13.104333333333335 - - type: recall_at_1 - value: 28.304499999999997 - - type: recall_at_10 - value: 56.80666666666667 - - type: recall_at_100 - value: 79.66208333333334 - - type: recall_at_1000 - value: 93.6455 - - type: recall_at_20 - value: 64.2495 - - type: recall_at_3 - value: 42.431333333333335 - - type: recall_at_5 - value: 48.665416666666665 - task: - type: Retrieval - - dataset: - config: default - name: MTEB ClimateFEVER - revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 - split: test - type: mteb/climate-fever - metrics: - - type: main_score - value: 43.525999999999996 - - type: map_at_1 - value: 19.291 - - type: map_at_10 - value: 33.471000000000004 - - type: map_at_100 - value: 35.388999999999996 - - type: map_at_1000 - value: 35.568 - - type: map_at_20 - value: 34.496 - - type: map_at_3 - value: 28.713 - - type: map_at_5 - value: 31.384 - - type: mrr_at_1 - value: 43.77850162866449 - - type: mrr_at_10 - value: 56.28576598934912 - - type: mrr_at_100 - value: 56.8588518168194 - - type: mrr_at_1000 - value: 56.878236725973544 - - type: mrr_at_20 - value: 56.6409328120183 - - type: mrr_at_3 - value: 53.56134636264935 - - type: mrr_at_5 - value: 55.27795874049956 - - type: nauc_map_at_1000_diff1 - value: 27.262513153363876 - - type: nauc_map_at_1000_max - value: 40.099398684385584 - - type: nauc_map_at_1000_std - value: 18.847812394005512 - - type: nauc_map_at_100_diff1 - value: 27.238993503030745 - - type: nauc_map_at_100_max - value: 40.07730434492169 - - type: nauc_map_at_100_std - value: 18.795349250833684 - - type: nauc_map_at_10_diff1 - value: 27.70929180366227 - - type: nauc_map_at_10_max - value: 39.55987024970173 - - type: nauc_map_at_10_std - value: 17.214881544648996 - - type: nauc_map_at_1_diff1 - value: 43.34155892182403 - - type: nauc_map_at_1_max - value: 38.23324890148018 - - type: nauc_map_at_1_std - value: 6.0781444393516075 - - type: nauc_map_at_20_diff1 - value: 27.311577477800103 - - type: nauc_map_at_20_max - value: 39.624414083413456 - - type: nauc_map_at_20_std - value: 18.149811054163287 - - type: nauc_map_at_3_diff1 - value: 30.475965062734367 - - type: nauc_map_at_3_max - value: 38.49324825043695 - - type: nauc_map_at_3_std - value: 13.357656038648487 - - type: nauc_map_at_5_diff1 - value: 28.425110095017747 - - type: nauc_map_at_5_max - value: 39.017894870747796 - - type: nauc_map_at_5_std - value: 15.543817194122564 - - type: nauc_mrr_at_1000_diff1 - value: 33.16689354701644 - - type: nauc_mrr_at_1000_max - value: 41.70755363247148 - - type: nauc_mrr_at_1000_std - value: 24.61667417463176 - - type: nauc_mrr_at_100_diff1 - value: 33.147229262917506 - - type: nauc_mrr_at_100_max - value: 41.712455697170725 - - type: nauc_mrr_at_100_std - value: 24.6418922043652 - - type: nauc_mrr_at_10_diff1 - value: 32.94185191112572 - - type: nauc_mrr_at_10_max - value: 41.64272730141954 - - type: nauc_mrr_at_10_std - value: 24.663391015702707 - - type: nauc_mrr_at_1_diff1 - value: 39.571969559016395 - - type: nauc_mrr_at_1_max - value: 39.396249211263495 - - type: nauc_mrr_at_1_std - value: 16.984149923258357 - - type: nauc_mrr_at_20_diff1 - value: 33.10040770334742 - - type: nauc_mrr_at_20_max - value: 41.807565560083034 - - type: nauc_mrr_at_20_std - value: 24.8064180365271 - - type: nauc_mrr_at_3_diff1 - value: 33.065406161485704 - - type: nauc_mrr_at_3_max - value: 41.049510969934694 - - type: nauc_mrr_at_3_std - value: 23.18371458928609 - - type: nauc_mrr_at_5_diff1 - value: 33.2389593543916 - - type: nauc_mrr_at_5_max - value: 41.629486918949915 - - type: nauc_mrr_at_5_std - value: 24.5777253036149 - - type: nauc_ndcg_at_1000_diff1 - value: 25.868840609197637 - - type: nauc_ndcg_at_1000_max - value: 42.79564910784761 - - type: nauc_ndcg_at_1000_std - value: 27.035091271680113 - - type: nauc_ndcg_at_100_diff1 - value: 25.019789319579942 - - type: nauc_ndcg_at_100_max - value: 42.482345143533735 - - type: nauc_ndcg_at_100_std - value: 26.76872010731345 - - type: nauc_ndcg_at_10_diff1 - value: 25.949464660653238 - - type: nauc_ndcg_at_10_max - value: 40.79769544643906 - - type: nauc_ndcg_at_10_std - value: 22.486116508973204 - - type: nauc_ndcg_at_1_diff1 - value: 39.571969559016395 - - type: nauc_ndcg_at_1_max - value: 39.396249211263495 - - type: nauc_ndcg_at_1_std - value: 16.984149923258357 - - type: nauc_ndcg_at_20_diff1 - value: 25.173455685962214 - - type: nauc_ndcg_at_20_max - value: 40.88873540662413 - - type: nauc_ndcg_at_20_std - value: 24.4451041955519 - - type: nauc_ndcg_at_3_diff1 - value: 28.185416070726333 - - type: nauc_ndcg_at_3_max - value: 39.10600031163912 - - type: nauc_ndcg_at_3_std - value: 18.42694044215541 - - type: nauc_ndcg_at_5_diff1 - value: 27.112647584005583 - - type: nauc_ndcg_at_5_max - value: 40.154045682322526 - - type: nauc_ndcg_at_5_std - value: 20.26822517176828 - - type: nauc_precision_at_1000_diff1 - value: -16.42087927044017 - - type: nauc_precision_at_1000_max - value: 3.5326295053913 - - type: nauc_precision_at_1000_std - value: 24.406810708493197 - - type: nauc_precision_at_100_diff1 - value: -12.17648135724982 - - type: nauc_precision_at_100_max - value: 15.895489260126183 - - type: nauc_precision_at_100_std - value: 32.48346122610907 - - type: nauc_precision_at_10_diff1 - value: -1.2493131347748072 - - type: nauc_precision_at_10_max - value: 26.409459305604376 - - type: nauc_precision_at_10_std - value: 31.115432019300016 - - type: nauc_precision_at_1_diff1 - value: 39.571969559016395 - - type: nauc_precision_at_1_max - value: 39.396249211263495 - - type: nauc_precision_at_1_std - value: 16.984149923258357 - - type: nauc_precision_at_20_diff1 - value: -6.597509397240593 - - type: nauc_precision_at_20_max - value: 21.461984620659695 - - type: nauc_precision_at_20_std - value: 32.9450259748889 - - type: nauc_precision_at_3_diff1 - value: 9.46378764865453 - - type: nauc_precision_at_3_max - value: 32.03650819375425 - - type: nauc_precision_at_3_std - value: 26.489382638510765 - - type: nauc_precision_at_5_diff1 - value: 3.5987036728169537 - - type: nauc_precision_at_5_max - value: 30.633955978579703 - - type: nauc_precision_at_5_std - value: 30.532430088014443 - - type: nauc_recall_at_1000_diff1 - value: 10.714633106872254 - - type: nauc_recall_at_1000_max - value: 43.94958623961 - - type: nauc_recall_at_1000_std - value: 51.78914468954123 - - type: nauc_recall_at_100_diff1 - value: 9.63781472255557 - - type: nauc_recall_at_100_max - value: 38.50917465255336 - - type: nauc_recall_at_100_std - value: 37.78623984642377 - - type: nauc_recall_at_10_diff1 - value: 16.480342820841688 - - type: nauc_recall_at_10_max - value: 35.982566867357406 - - type: nauc_recall_at_10_std - value: 23.30688188788895 - - type: nauc_recall_at_1_diff1 - value: 43.34155892182403 - - type: nauc_recall_at_1_max - value: 38.23324890148018 - - type: nauc_recall_at_1_std - value: 6.0781444393516075 - - type: nauc_recall_at_20_diff1 - value: 13.521048985146367 - - type: nauc_recall_at_20_max - value: 34.62462209239834 - - type: nauc_recall_at_20_std - value: 27.85924191501618 - - type: nauc_recall_at_3_diff1 - value: 23.57032748533523 - - type: nauc_recall_at_3_max - value: 36.32703197635613 - - type: nauc_recall_at_3_std - value: 15.730238734014337 - - type: nauc_recall_at_5_diff1 - value: 19.61387036368584 - - type: nauc_recall_at_5_max - value: 36.22030835529556 - - type: nauc_recall_at_5_std - value: 19.76310648649897 - - type: ndcg_at_1 - value: 43.779 - - type: ndcg_at_10 - value: 43.525999999999996 - - type: ndcg_at_100 - value: 50.138000000000005 - - type: ndcg_at_1000 - value: 52.991 - - type: ndcg_at_20 - value: 46.083 - - type: ndcg_at_3 - value: 38.002 - - type: ndcg_at_5 - value: 39.842 - - type: precision_at_1 - value: 43.779 - - type: precision_at_10 - value: 13.205 - - type: precision_at_100 - value: 2.051 - - type: precision_at_1000 - value: 0.259 - - type: precision_at_20 - value: 7.722999999999999 - - type: precision_at_3 - value: 28.903000000000002 - - type: precision_at_5 - value: 21.368000000000002 - - type: recall_at_1 - value: 19.291 - - type: recall_at_10 - value: 48.754 - - type: recall_at_100 - value: 70.97200000000001 - - type: recall_at_1000 - value: 86.611 - - type: recall_at_20 - value: 55.884 - - type: recall_at_3 - value: 34.101 - - type: recall_at_5 - value: 40.784 - task: - type: Retrieval - - dataset: - config: default - name: MTEB DBPedia - revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 - split: test - type: mteb/dbpedia - metrics: - - type: main_score - value: 49.884 - - type: map_at_1 - value: 9.913 - - type: map_at_10 - value: 23.186999999999998 - - type: map_at_100 - value: 34.207 - - type: map_at_1000 - value: 36.318 - - type: map_at_20 - value: 27.419 - - type: map_at_3 - value: 15.656 - - type: map_at_5 - value: 18.945999999999998 - - type: mrr_at_1 - value: 75.75 - - type: mrr_at_10 - value: 82.16279761904761 - - type: mrr_at_100 - value: 82.48445635330299 - - type: mrr_at_1000 - value: 82.4870246719901 - - type: mrr_at_20 - value: 82.36203632968338 - - type: mrr_at_3 - value: 81.29166666666666 - - type: mrr_at_5 - value: 82.02916666666667 - - type: nauc_map_at_1000_diff1 - value: 17.0739966990996 - - type: nauc_map_at_1000_max - value: 28.440065298437133 - - type: nauc_map_at_1000_std - value: 20.83498154003865 - - type: nauc_map_at_100_diff1 - value: 17.75982086107111 - - type: nauc_map_at_100_max - value: 26.87850835673573 - - type: nauc_map_at_100_std - value: 18.350282298599275 - - type: nauc_map_at_10_diff1 - value: 17.15984258564116 - - type: nauc_map_at_10_max - value: 10.846179132675553 - - type: nauc_map_at_10_std - value: -6.263534464094614 - - type: nauc_map_at_1_diff1 - value: 24.014897777973694 - - type: nauc_map_at_1_max - value: -4.556638938723358 - - type: nauc_map_at_1_std - value: -22.7844467526989 - - type: nauc_map_at_20_diff1 - value: 16.3179372493187 - - type: nauc_map_at_20_max - value: 17.176378915498915 - - type: nauc_map_at_20_std - value: 1.9378637630340372 - - type: nauc_map_at_3_diff1 - value: 19.12786794046792 - - type: nauc_map_at_3_max - value: 0.09063919305677291 - - type: nauc_map_at_3_std - value: -16.713143158330492 - - type: nauc_map_at_5_diff1 - value: 18.76504725420023 - - type: nauc_map_at_5_max - value: 5.040867712207419 - - type: nauc_map_at_5_std - value: -12.382578318931165 - - type: nauc_mrr_at_1000_diff1 - value: 54.61266255011247 - - type: nauc_mrr_at_1000_max - value: 60.83961280977112 - - type: nauc_mrr_at_1000_std - value: 32.70429260443016 - - type: nauc_mrr_at_100_diff1 - value: 54.61346236538542 - - type: nauc_mrr_at_100_max - value: 60.8407974416647 - - type: nauc_mrr_at_100_std - value: 32.69272843993462 - - type: nauc_mrr_at_10_diff1 - value: 54.74633685810871 - - type: nauc_mrr_at_10_max - value: 61.084525933097865 - - type: nauc_mrr_at_10_std - value: 33.001220210025565 - - type: nauc_mrr_at_1_diff1 - value: 56.12708423835806 - - type: nauc_mrr_at_1_max - value: 58.9314540998289 - - type: nauc_mrr_at_1_std - value: 27.39422607651012 - - type: nauc_mrr_at_20_diff1 - value: 54.58896150245695 - - type: nauc_mrr_at_20_max - value: 60.890929983464815 - - type: nauc_mrr_at_20_std - value: 32.65559641276393 - - type: nauc_mrr_at_3_diff1 - value: 54.38229071443791 - - type: nauc_mrr_at_3_max - value: 59.987849044098596 - - type: nauc_mrr_at_3_std - value: 33.439813880719974 - - type: nauc_mrr_at_5_diff1 - value: 54.961790262449824 - - type: nauc_mrr_at_5_max - value: 61.17705173908951 - - type: nauc_mrr_at_5_std - value: 33.30939850734856 - - type: nauc_ndcg_at_1000_diff1 - value: 29.27465932507067 - - type: nauc_ndcg_at_1000_max - value: 47.952543312315214 - - type: nauc_ndcg_at_1000_std - value: 36.17132236391485 - - type: nauc_ndcg_at_100_diff1 - value: 28.63072328980134 - - type: nauc_ndcg_at_100_max - value: 41.460833419186564 - - type: nauc_ndcg_at_100_std - value: 27.157100358988135 - - type: nauc_ndcg_at_10_diff1 - value: 23.41488013023301 - - type: nauc_ndcg_at_10_max - value: 39.27798133072349 - - type: nauc_ndcg_at_10_std - value: 21.979241438928312 - - type: nauc_ndcg_at_1_diff1 - value: 46.12120543657642 - - type: nauc_ndcg_at_1_max - value: 47.28452124039853 - - type: nauc_ndcg_at_1_std - value: 19.799884708952543 - - type: nauc_ndcg_at_20_diff1 - value: 23.627669045115574 - - type: nauc_ndcg_at_20_max - value: 35.88225062457673 - - type: nauc_ndcg_at_20_std - value: 18.218628030529498 - - type: nauc_ndcg_at_3_diff1 - value: 25.37309228946118 - - type: nauc_ndcg_at_3_max - value: 40.64426332992231 - - type: nauc_ndcg_at_3_std - value: 24.608330645901482 - - type: nauc_ndcg_at_5_diff1 - value: 24.055798594999654 - - type: nauc_ndcg_at_5_max - value: 41.16180524175431 - - type: nauc_ndcg_at_5_std - value: 24.048305528761315 - - type: nauc_precision_at_1000_diff1 - value: -18.234943251015576 - - type: nauc_precision_at_1000_max - value: 0.48708502364659184 - - type: nauc_precision_at_1000_std - value: 2.4473601543134027 - - type: nauc_precision_at_100_diff1 - value: -3.0077810947381227 - - type: nauc_precision_at_100_max - value: 25.27249321108913 - - type: nauc_precision_at_100_std - value: 37.36575792126928 - - type: nauc_precision_at_10_diff1 - value: -0.2393778190297635 - - type: nauc_precision_at_10_max - value: 36.40513293547299 - - type: nauc_precision_at_10_std - value: 37.4827885766009 - - type: nauc_precision_at_1_diff1 - value: 56.12708423835806 - - type: nauc_precision_at_1_max - value: 58.9314540998289 - - type: nauc_precision_at_1_std - value: 27.39422607651012 - - type: nauc_precision_at_20_diff1 - value: -1.2010133229402933 - - type: nauc_precision_at_20_max - value: 34.117541814385966 - - type: nauc_precision_at_20_std - value: 39.13273254177449 - - type: nauc_precision_at_3_diff1 - value: 11.757378092198486 - - type: nauc_precision_at_3_max - value: 42.637962482588875 - - type: nauc_precision_at_3_std - value: 37.42465077352342 - - type: nauc_precision_at_5_diff1 - value: 7.233177203405101 - - type: nauc_precision_at_5_max - value: 43.1663582897407 - - type: nauc_precision_at_5_std - value: 38.848449220750055 - - type: nauc_recall_at_1000_diff1 - value: 27.33938551969145 - - type: nauc_recall_at_1000_max - value: 45.5614254479334 - - type: nauc_recall_at_1000_std - value: 50.58528916250458 - - type: nauc_recall_at_100_diff1 - value: 23.610383761920097 - - type: nauc_recall_at_100_max - value: 31.422168485847184 - - type: nauc_recall_at_100_std - value: 25.58649926458304 - - type: nauc_recall_at_10_diff1 - value: 14.62495111808408 - - type: nauc_recall_at_10_max - value: 7.4295041277681095 - - type: nauc_recall_at_10_std - value: -9.32297089600654 - - type: nauc_recall_at_1_diff1 - value: 24.014897777973694 - - type: nauc_recall_at_1_max - value: -4.556638938723358 - - type: nauc_recall_at_1_std - value: -22.7844467526989 - - type: nauc_recall_at_20_diff1 - value: 14.027862330014662 - - type: nauc_recall_at_20_max - value: 12.437478731690844 - - type: nauc_recall_at_20_std - value: -3.0740743798103676 - - type: nauc_recall_at_3_diff1 - value: 16.354018356566712 - - type: nauc_recall_at_3_max - value: -2.9812231240997917 - - type: nauc_recall_at_3_std - value: -18.27746460743442 - - type: nauc_recall_at_5_diff1 - value: 16.81486583473587 - - type: nauc_recall_at_5_max - value: 2.420128513974744 - - type: nauc_recall_at_5_std - value: -14.441820321214108 - - type: ndcg_at_1 - value: 63.87500000000001 - - type: ndcg_at_10 - value: 49.884 - - type: ndcg_at_100 - value: 54.738 - - type: ndcg_at_1000 - value: 61.635 - - type: ndcg_at_20 - value: 48.894999999999996 - - type: ndcg_at_3 - value: 54.287 - - type: ndcg_at_5 - value: 52.40899999999999 - - type: precision_at_1 - value: 75.75 - - type: precision_at_10 - value: 40.9 - - type: precision_at_100 - value: 13.139999999999999 - - type: precision_at_1000 - value: 2.533 - - type: precision_at_20 - value: 30.8 - - type: precision_at_3 - value: 57.667 - - type: precision_at_5 - value: 51.05 - - type: recall_at_1 - value: 9.913 - - type: recall_at_10 - value: 28.591 - - type: recall_at_100 - value: 61.017999999999994 - - type: recall_at_1000 - value: 83.383 - - type: recall_at_20 - value: 37.834 - - type: recall_at_3 - value: 17.049 - - type: recall_at_5 - value: 21.685 - task: - type: Retrieval - - dataset: - config: default - name: MTEB EmotionClassification - revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 - split: test - type: mteb/emotion - metrics: - - type: accuracy - value: 78.77499999999999 - - type: f1 - value: 73.74058240799386 - - type: f1_weighted - value: 79.78804377638227 - - type: main_score - value: 78.77499999999999 - task: - type: Classification - - dataset: - config: default - name: MTEB FEVER - revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 - split: test - type: mteb/fever - metrics: - - type: main_score - value: 90.986 - - type: map_at_1 - value: 81.601 - - type: map_at_10 - value: 88.242 - - type: map_at_100 - value: 88.46000000000001 - - type: map_at_1000 - value: 88.472 - - type: map_at_20 - value: 88.375 - - type: map_at_3 - value: 87.237 - - type: map_at_5 - value: 87.85300000000001 - - type: mrr_at_1 - value: 87.81878187818782 - - type: mrr_at_10 - value: 92.20301196786335 - - type: mrr_at_100 - value: 92.24884236673292 - - type: mrr_at_1000 - value: 92.2496338899362 - - type: mrr_at_20 - value: 92.23112073283473 - - type: mrr_at_3 - value: 91.77417741774165 - - type: mrr_at_5 - value: 92.03970397039689 - - type: nauc_map_at_1000_diff1 - value: 56.54670664910505 - - type: nauc_map_at_1000_max - value: 33.08375749975477 - - type: nauc_map_at_1000_std - value: 2.7491595418252865 - - type: nauc_map_at_100_diff1 - value: 56.50887688686924 - - type: nauc_map_at_100_max - value: 33.075487189958494 - - type: nauc_map_at_100_std - value: 2.7675869969253375 - - type: nauc_map_at_10_diff1 - value: 56.08080806610569 - - type: nauc_map_at_10_max - value: 32.776972098819066 - - type: nauc_map_at_10_std - value: 2.5904846711290097 - - type: nauc_map_at_1_diff1 - value: 60.645344065853145 - - type: nauc_map_at_1_max - value: 31.232776777514797 - - type: nauc_map_at_1_std - value: -1.1946138176109171 - - type: nauc_map_at_20_diff1 - value: 56.28378454162355 - - type: nauc_map_at_20_max - value: 32.98207150385811 - - type: nauc_map_at_20_std - value: 2.8469814040214025 - - type: nauc_map_at_3_diff1 - value: 55.81958007095375 - - type: nauc_map_at_3_max - value: 31.602707711038313 - - type: nauc_map_at_3_std - value: 0.8117019292273401 - - type: nauc_map_at_5_diff1 - value: 55.706025752316535 - - type: nauc_map_at_5_max - value: 32.16032683604737 - - type: nauc_map_at_5_std - value: 1.8853201503498669 - - type: nauc_mrr_at_1000_diff1 - value: 75.4997173366251 - - type: nauc_mrr_at_1000_max - value: 41.49117135484116 - - type: nauc_mrr_at_1000_std - value: -2.0636172883680852 - - type: nauc_mrr_at_100_diff1 - value: 75.50118860648519 - - type: nauc_mrr_at_100_max - value: 41.49490161517194 - - type: nauc_mrr_at_100_std - value: -2.057024385178682 - - type: nauc_mrr_at_10_diff1 - value: 75.47295153099428 - - type: nauc_mrr_at_10_max - value: 41.55003304042536 - - type: nauc_mrr_at_10_std - value: -2.0353663198929253 - - type: nauc_mrr_at_1_diff1 - value: 76.632058433229 - - type: nauc_mrr_at_1_max - value: 39.754483718891656 - - type: nauc_mrr_at_1_std - value: -2.962241058101701 - - type: nauc_mrr_at_20_diff1 - value: 75.47221882396194 - - type: nauc_mrr_at_20_max - value: 41.50779280480839 - - type: nauc_mrr_at_20_std - value: -1.9620212266426307 - - type: nauc_mrr_at_3_diff1 - value: 75.5682297897137 - - type: nauc_mrr_at_3_max - value: 41.53543801506081 - - type: nauc_mrr_at_3_std - value: -3.391681195945978 - - type: nauc_mrr_at_5_diff1 - value: 75.37562775183947 - - type: nauc_mrr_at_5_max - value: 41.42028509006753 - - type: nauc_mrr_at_5_std - value: -2.418698675622726 - - type: nauc_ndcg_at_1000_diff1 - value: 59.364557011624 - - type: nauc_ndcg_at_1000_max - value: 35.4112238125149 - - type: nauc_ndcg_at_1000_std - value: 3.717516193303376 - - type: nauc_ndcg_at_100_diff1 - value: 58.55706703023122 - - type: nauc_ndcg_at_100_max - value: 35.352285999934594 - - type: nauc_ndcg_at_100_std - value: 4.273437944266781 - - type: nauc_ndcg_at_10_diff1 - value: 56.77422701267037 - - type: nauc_ndcg_at_10_max - value: 34.24909893882957 - - type: nauc_ndcg_at_10_std - value: 4.178151434006727 - - type: nauc_ndcg_at_1_diff1 - value: 76.632058433229 - - type: nauc_ndcg_at_1_max - value: 39.754483718891656 - - type: nauc_ndcg_at_1_std - value: -2.962241058101701 - - type: nauc_ndcg_at_20_diff1 - value: 57.27343398231262 - - type: nauc_ndcg_at_20_max - value: 34.7416626740278 - - type: nauc_ndcg_at_20_std - value: 4.955858766014002 - - type: nauc_ndcg_at_3_diff1 - value: 57.69267803121093 - - type: nauc_ndcg_at_3_max - value: 33.13744317023105 - - type: nauc_ndcg_at_3_std - value: 0.40380284030057023 - - type: nauc_ndcg_at_5_diff1 - value: 56.57461019113917 - - type: nauc_ndcg_at_5_max - value: 33.244657840804386 - - type: nauc_ndcg_at_5_std - value: 2.5121440827702046 - - type: nauc_precision_at_1000_diff1 - value: -14.54492513449718 - - type: nauc_precision_at_1000_max - value: -5.94552147573623 - - type: nauc_precision_at_1000_std - value: 1.2446209816057374 - - type: nauc_precision_at_100_diff1 - value: -15.452676132568344 - - type: nauc_precision_at_100_max - value: -3.760241749847617 - - type: nauc_precision_at_100_std - value: 4.623534605290865 - - type: nauc_precision_at_10_diff1 - value: -12.712908026086176 - - type: nauc_precision_at_10_max - value: 0.45241316994816805 - - type: nauc_precision_at_10_std - value: 7.849478570138391 - - type: nauc_precision_at_1_diff1 - value: 76.632058433229 - - type: nauc_precision_at_1_max - value: 39.754483718891656 - - type: nauc_precision_at_1_std - value: -2.962241058101701 - - type: nauc_precision_at_20_diff1 - value: -14.514618673172041 - - type: nauc_precision_at_20_max - value: -1.113635490621818 - - type: nauc_precision_at_20_std - value: 8.599811730457576 - - type: nauc_precision_at_3_diff1 - value: 6.1367799850003815 - - type: nauc_precision_at_3_max - value: 8.466271950897857 - - type: nauc_precision_at_3_std - value: 1.7458051543195068 - - type: nauc_precision_at_5_diff1 - value: -5.804548945783379 - - type: nauc_precision_at_5_max - value: 3.4060251839074818 - - type: nauc_precision_at_5_std - value: 5.583410511782371 - - type: nauc_recall_at_1000_diff1 - value: 19.329432953574095 - - type: nauc_recall_at_1000_max - value: 43.260442595158736 - - type: nauc_recall_at_1000_std - value: 53.89644660661804 - - type: nauc_recall_at_100_diff1 - value: 21.265326296051235 - - type: nauc_recall_at_100_max - value: 38.573000195373695 - - type: nauc_recall_at_100_std - value: 42.169391082152785 - - type: nauc_recall_at_10_diff1 - value: 29.785129558987432 - - type: nauc_recall_at_10_max - value: 28.379657867558034 - - type: nauc_recall_at_10_std - value: 21.132574624091973 - - type: nauc_recall_at_1_diff1 - value: 60.645344065853145 - - type: nauc_recall_at_1_max - value: 31.232776777514797 - - type: nauc_recall_at_1_std - value: -1.1946138176109171 - - type: nauc_recall_at_20_diff1 - value: 25.88845612373954 - - type: nauc_recall_at_20_max - value: 30.24785945821152 - - type: nauc_recall_at_20_std - value: 31.73911437468067 - - type: nauc_recall_at_3_diff1 - value: 42.2968464797395 - - type: nauc_recall_at_3_max - value: 26.494318009870018 - - type: nauc_recall_at_3_std - value: 2.6045977160467544 - - type: nauc_recall_at_5_diff1 - value: 35.81340094401374 - - type: nauc_recall_at_5_max - value: 25.91082947510634 - - type: nauc_recall_at_5_std - value: 9.759404930864779 - - type: ndcg_at_1 - value: 87.819 - - type: ndcg_at_10 - value: 90.986 - - type: ndcg_at_100 - value: 91.69 - - type: ndcg_at_1000 - value: 91.863 - - type: ndcg_at_20 - value: 91.293 - - type: ndcg_at_3 - value: 89.621 - - type: ndcg_at_5 - value: 90.333 - - type: precision_at_1 - value: 87.819 - - type: precision_at_10 - value: 10.753 - - type: precision_at_100 - value: 1.138 - - type: precision_at_1000 - value: 0.117 - - type: precision_at_20 - value: 5.4879999999999995 - - type: precision_at_3 - value: 33.703 - - type: precision_at_5 - value: 20.831 - - type: recall_at_1 - value: 81.601 - - type: recall_at_10 - value: 95.44200000000001 - - type: recall_at_100 - value: 98.14399999999999 - - type: recall_at_1000 - value: 99.157 - - type: recall_at_20 - value: 96.43 - - type: recall_at_3 - value: 91.729 - - type: recall_at_5 - value: 93.552 - task: - type: Retrieval - - dataset: - config: default - name: MTEB FiQA2018 - revision: 27a168819829fe9bcd655c2df245fb19452e8e06 - split: test - type: mteb/fiqa - metrics: - - type: main_score - value: 56.056 - - type: map_at_1 - value: 28.666000000000004 - - type: map_at_10 - value: 47.437000000000005 - - type: map_at_100 - value: 49.537 - - type: map_at_1000 - value: 49.665 - - type: map_at_20 - value: 48.618 - - type: map_at_3 - value: 41.355 - - type: map_at_5 - value: 44.525 - - type: mrr_at_1 - value: 55.55555555555556 - - type: mrr_at_10 - value: 63.705173427395614 - - type: mrr_at_100 - value: 64.25449940779741 - - type: mrr_at_1000 - value: 64.27635581092147 - - type: mrr_at_20 - value: 64.03796029079103 - - type: mrr_at_3 - value: 61.49691358024688 - - type: mrr_at_5 - value: 62.73148148148143 - - type: nauc_map_at_1000_diff1 - value: 43.24282910397747 - - type: nauc_map_at_1000_max - value: 28.506093180265644 - - type: nauc_map_at_1000_std - value: -13.040508386155054 - - type: nauc_map_at_100_diff1 - value: 43.23650442904607 - - type: nauc_map_at_100_max - value: 28.470565635459156 - - type: nauc_map_at_100_std - value: -12.988098780714935 - - type: nauc_map_at_10_diff1 - value: 43.393840733087686 - - type: nauc_map_at_10_max - value: 26.637302062720153 - - type: nauc_map_at_10_std - value: -14.47500292113762 - - type: nauc_map_at_1_diff1 - value: 47.705150227211725 - - type: nauc_map_at_1_max - value: 15.354189686550129 - - type: nauc_map_at_1_std - value: -14.559819859039067 - - type: nauc_map_at_20_diff1 - value: 43.14121075706104 - - type: nauc_map_at_20_max - value: 27.811170590408395 - - type: nauc_map_at_20_std - value: -13.459413585283583 - - type: nauc_map_at_3_diff1 - value: 44.33938667720801 - - type: nauc_map_at_3_max - value: 21.785619884549398 - - type: nauc_map_at_3_std - value: -15.569980103071593 - - type: nauc_map_at_5_diff1 - value: 43.39280905665027 - - type: nauc_map_at_5_max - value: 25.021492190645017 - - type: nauc_map_at_5_std - value: -14.48856622187443 - - type: nauc_mrr_at_1000_diff1 - value: 52.971563939946286 - - type: nauc_mrr_at_1000_max - value: 38.88019486172324 - - type: nauc_mrr_at_1000_std - value: -12.412991642381616 - - type: nauc_mrr_at_100_diff1 - value: 52.978468139876945 - - type: nauc_mrr_at_100_max - value: 38.89751787948751 - - type: nauc_mrr_at_100_std - value: -12.3677876252269 - - type: nauc_mrr_at_10_diff1 - value: 52.78507148048174 - - type: nauc_mrr_at_10_max - value: 38.55079809310022 - - type: nauc_mrr_at_10_std - value: -12.944127025078755 - - type: nauc_mrr_at_1_diff1 - value: 55.52626805861546 - - type: nauc_mrr_at_1_max - value: 40.49306809164979 - - type: nauc_mrr_at_1_std - value: -12.886607701317681 - - type: nauc_mrr_at_20_diff1 - value: 52.9592152665678 - - type: nauc_mrr_at_20_max - value: 38.88514014589964 - - type: nauc_mrr_at_20_std - value: -12.434464359819444 - - type: nauc_mrr_at_3_diff1 - value: 52.73696844091174 - - type: nauc_mrr_at_3_max - value: 38.61018727252859 - - type: nauc_mrr_at_3_std - value: -13.123989867364166 - - type: nauc_mrr_at_5_diff1 - value: 53.037110010188 - - type: nauc_mrr_at_5_max - value: 38.44770729849151 - - type: nauc_mrr_at_5_std - value: -13.49318771828972 - - type: nauc_ndcg_at_1000_diff1 - value: 44.73813840091289 - - type: nauc_ndcg_at_1000_max - value: 33.70113904685389 - - type: nauc_ndcg_at_1000_std - value: -10.328687058192742 - - type: nauc_ndcg_at_100_diff1 - value: 44.595174119928835 - - type: nauc_ndcg_at_100_max - value: 33.4788285112467 - - type: nauc_ndcg_at_100_std - value: -8.695355259716946 - - type: nauc_ndcg_at_10_diff1 - value: 44.39837225263 - - type: nauc_ndcg_at_10_max - value: 29.188289725593393 - - type: nauc_ndcg_at_10_std - value: -13.67608323673103 - - type: nauc_ndcg_at_1_diff1 - value: 55.52626805861546 - - type: nauc_ndcg_at_1_max - value: 40.49306809164979 - - type: nauc_ndcg_at_1_std - value: -12.886607701317681 - - type: nauc_ndcg_at_20_diff1 - value: 44.24661739902305 - - type: nauc_ndcg_at_20_max - value: 31.667868318249965 - - type: nauc_ndcg_at_20_std - value: -10.65470780066342 - - type: nauc_ndcg_at_3_diff1 - value: 43.39857166975522 - - type: nauc_ndcg_at_3_max - value: 31.764668313577495 - - type: nauc_ndcg_at_3_std - value: -14.494866954678152 - - type: nauc_ndcg_at_5_diff1 - value: 43.16976647347281 - - type: nauc_ndcg_at_5_max - value: 29.878329062643143 - - type: nauc_ndcg_at_5_std - value: -13.987689089179739 - - type: nauc_precision_at_1000_diff1 - value: -9.807973252625484 - - type: nauc_precision_at_1000_max - value: 26.6279603849494 - - type: nauc_precision_at_1000_std - value: 7.113187103520632 - - type: nauc_precision_at_100_diff1 - value: -4.777149603323976 - - type: nauc_precision_at_100_max - value: 31.03410463692187 - - type: nauc_precision_at_100_std - value: 10.463144150275435 - - type: nauc_precision_at_10_diff1 - value: 8.691528703215962 - - type: nauc_precision_at_10_max - value: 33.329579434123374 - - type: nauc_precision_at_10_std - value: -0.8002015226329403 - - type: nauc_precision_at_1_diff1 - value: 55.52626805861546 - - type: nauc_precision_at_1_max - value: 40.49306809164979 - - type: nauc_precision_at_1_std - value: -12.886607701317681 - - type: nauc_precision_at_20_diff1 - value: 3.4564653474184284 - - type: nauc_precision_at_20_max - value: 34.401070158471136 - - type: nauc_precision_at_20_std - value: 5.813431200164549 - - type: nauc_precision_at_3_diff1 - value: 22.463219705462187 - - type: nauc_precision_at_3_max - value: 34.77413976546924 - - type: nauc_precision_at_3_std - value: -7.083890789741479 - - type: nauc_precision_at_5_diff1 - value: 14.011006004883154 - - type: nauc_precision_at_5_max - value: 35.73655466853702 - - type: nauc_precision_at_5_std - value: -2.8395172077771598 - - type: nauc_recall_at_1000_diff1 - value: 16.478046357391555 - - type: nauc_recall_at_1000_max - value: 43.231704288282344 - - type: nauc_recall_at_1000_std - value: 38.430684937573645 - - type: nauc_recall_at_100_diff1 - value: 30.764718344602436 - - type: nauc_recall_at_100_max - value: 31.769050487166655 - - type: nauc_recall_at_100_std - value: 23.48468311677149 - - type: nauc_recall_at_10_diff1 - value: 34.47339565324045 - - type: nauc_recall_at_10_max - value: 19.054212335800454 - - type: nauc_recall_at_10_std - value: -11.039734015330437 - - type: nauc_recall_at_1_diff1 - value: 47.705150227211725 - - type: nauc_recall_at_1_max - value: 15.354189686550129 - - type: nauc_recall_at_1_std - value: -14.559819859039067 - - type: nauc_recall_at_20_diff1 - value: 32.1011474016873 - - type: nauc_recall_at_20_max - value: 25.546372988304423 - - type: nauc_recall_at_20_std - value: -0.007233471152482897 - - type: nauc_recall_at_3_diff1 - value: 37.5708138019065 - - type: nauc_recall_at_3_max - value: 16.66410785756736 - - type: nauc_recall_at_3_std - value: -15.404817020108966 - - type: nauc_recall_at_5_diff1 - value: 35.714519648479595 - - type: nauc_recall_at_5_max - value: 19.02075233009296 - - type: nauc_recall_at_5_std - value: -13.180963359760725 - - type: ndcg_at_1 - value: 55.556000000000004 - - type: ndcg_at_10 - value: 56.056 - - type: ndcg_at_100 - value: 62.44 - - type: ndcg_at_1000 - value: 64.263 - - type: ndcg_at_20 - value: 58.638999999999996 - - type: ndcg_at_3 - value: 51.722 - - type: ndcg_at_5 - value: 52.701 - - type: precision_at_1 - value: 55.556000000000004 - - type: precision_at_10 - value: 15.679000000000002 - - type: precision_at_100 - value: 2.252 - - type: precision_at_1000 - value: 0.257 - - type: precision_at_20 - value: 9.02 - - type: precision_at_3 - value: 34.619 - - type: precision_at_5 - value: 25.093 - - type: recall_at_1 - value: 28.666000000000004 - - type: recall_at_10 - value: 63.717999999999996 - - type: recall_at_100 - value: 86.938 - - type: recall_at_1000 - value: 97.603 - - type: recall_at_20 - value: 71.649 - - type: recall_at_3 - value: 46.663 - - type: recall_at_5 - value: 53.313 - task: - type: Retrieval - - dataset: - config: default - name: MTEB HotpotQA - revision: ab518f4d6fcca38d87c25209f94beba119d02014 - split: test - type: mteb/hotpotqa - metrics: - - type: main_score - value: 71.74199999999999 - - type: map_at_1 - value: 41.729 - - type: map_at_10 - value: 63.168 - - type: map_at_100 - value: 64.132 - - type: map_at_1000 - value: 64.199 - - type: map_at_20 - value: 63.736000000000004 - - type: map_at_3 - value: 59.826 - - type: map_at_5 - value: 61.882000000000005 - - type: mrr_at_1 - value: 83.45712356515868 - - type: mrr_at_10 - value: 87.850342432719 - - type: mrr_at_100 - value: 88.0016320691113 - - type: mrr_at_1000 - value: 88.00576596968136 - - type: mrr_at_20 - value: 87.94463253190389 - - type: mrr_at_3 - value: 87.13706954760278 - - type: mrr_at_5 - value: 87.59419311276136 - - type: nauc_map_at_1000_diff1 - value: 13.635446621095054 - - type: nauc_map_at_1000_max - value: 18.670632529445633 - - type: nauc_map_at_1000_std - value: 10.444842636150575 - - type: nauc_map_at_100_diff1 - value: 13.599262398010783 - - type: nauc_map_at_100_max - value: 18.636389405484806 - - type: nauc_map_at_100_std - value: 10.460027483576043 - - type: nauc_map_at_10_diff1 - value: 13.235053919323942 - - type: nauc_map_at_10_max - value: 18.252140477080047 - - type: nauc_map_at_10_std - value: 9.9075337042203 - - type: nauc_map_at_1_diff1 - value: 76.51940497836482 - - type: nauc_map_at_1_max - value: 51.251419487235474 - - type: nauc_map_at_1_std - value: 0.16714896857146574 - - type: nauc_map_at_20_diff1 - value: 13.4178245722222 - - type: nauc_map_at_20_max - value: 18.40988771210718 - - type: nauc_map_at_20_std - value: 10.216685163366282 - - type: nauc_map_at_3_diff1 - value: 13.38370761663418 - - type: nauc_map_at_3_max - value: 17.760962555456537 - - type: nauc_map_at_3_std - value: 7.15741965624388 - - type: nauc_map_at_5_diff1 - value: 13.138133309724855 - - type: nauc_map_at_5_max - value: 17.871761295251044 - - type: nauc_map_at_5_std - value: 8.475147426940074 - - type: nauc_mrr_at_1000_diff1 - value: 75.82650818891959 - - type: nauc_mrr_at_1000_max - value: 53.6736100668434 - - type: nauc_mrr_at_1000_std - value: 1.8025016349213916 - - type: nauc_mrr_at_100_diff1 - value: 75.82530574210111 - - type: nauc_mrr_at_100_max - value: 53.68067545829002 - - type: nauc_mrr_at_100_std - value: 1.8147470536495791 - - type: nauc_mrr_at_10_diff1 - value: 75.8330135686799 - - type: nauc_mrr_at_10_max - value: 53.78626885349077 - - type: nauc_mrr_at_10_std - value: 1.7975782717226636 - - type: nauc_mrr_at_1_diff1 - value: 76.51940497836482 - - type: nauc_mrr_at_1_max - value: 51.251419487235474 - - type: nauc_mrr_at_1_std - value: 0.16714896857146574 - - type: nauc_mrr_at_20_diff1 - value: 75.82783382464166 - - type: nauc_mrr_at_20_max - value: 53.68364567043885 - - type: nauc_mrr_at_20_std - value: 1.742037904463963 - - type: nauc_mrr_at_3_diff1 - value: 75.6944609768663 - - type: nauc_mrr_at_3_max - value: 53.803941340341666 - - type: nauc_mrr_at_3_std - value: 1.1849945458077804 - - type: nauc_mrr_at_5_diff1 - value: 75.73006960604903 - - type: nauc_mrr_at_5_max - value: 53.62223096420106 - - type: nauc_mrr_at_5_std - value: 1.6144067563410909 - - type: nauc_ndcg_at_1000_diff1 - value: 21.58025241642726 - - type: nauc_ndcg_at_1000_max - value: 24.675747527001153 - - type: nauc_ndcg_at_1000_std - value: 13.075943547492718 - - type: nauc_ndcg_at_100_diff1 - value: 20.30260137544846 - - type: nauc_ndcg_at_100_max - value: 23.757528813872018 - - type: nauc_ndcg_at_100_std - value: 13.648994687574062 - - type: nauc_ndcg_at_10_diff1 - value: 18.995052360997818 - - type: nauc_ndcg_at_10_max - value: 22.254260808196037 - - type: nauc_ndcg_at_10_std - value: 11.27212390633054 - - type: nauc_ndcg_at_1_diff1 - value: 76.51940497836482 - - type: nauc_ndcg_at_1_max - value: 51.251419487235474 - - type: nauc_ndcg_at_1_std - value: 0.16714896857146574 - - type: nauc_ndcg_at_20_diff1 - value: 19.333742380695757 - - type: nauc_ndcg_at_20_max - value: 22.527779834633364 - - type: nauc_ndcg_at_20_std - value: 12.161009000707917 - - type: nauc_ndcg_at_3_diff1 - value: 20.013329040965534 - - type: nauc_ndcg_at_3_max - value: 21.99692460311921 - - type: nauc_ndcg_at_3_std - value: 6.8076290638386165 - - type: nauc_ndcg_at_5_diff1 - value: 19.08226315942471 - - type: nauc_ndcg_at_5_max - value: 21.71185964294168 - - type: nauc_ndcg_at_5_std - value: 8.671911269518214 - - type: nauc_precision_at_1000_diff1 - value: 2.4462475489446764 - - type: nauc_precision_at_1000_max - value: 29.145662064268578 - - type: nauc_precision_at_1000_std - value: 49.20704909525856 - - type: nauc_precision_at_100_diff1 - value: 0.11271196725540299 - - type: nauc_precision_at_100_max - value: 17.37584606388067 - - type: nauc_precision_at_100_std - value: 34.66099346244071 - - type: nauc_precision_at_10_diff1 - value: 2.9923183951227825 - - type: nauc_precision_at_10_max - value: 14.261884731124264 - - type: nauc_precision_at_10_std - value: 18.084188795498378 - - type: nauc_precision_at_1_diff1 - value: 76.51940497836482 - - type: nauc_precision_at_1_max - value: 51.251419487235474 - - type: nauc_precision_at_1_std - value: 0.16714896857146574 - - type: nauc_precision_at_20_diff1 - value: 1.9180293008303761 - - type: nauc_precision_at_20_max - value: 13.832269193468512 - - type: nauc_precision_at_20_std - value: 21.65284406055607 - - type: nauc_precision_at_3_diff1 - value: 7.226609484731811 - - type: nauc_precision_at_3_max - value: 15.162908526977272 - - type: nauc_precision_at_3_std - value: 8.451859972962776 - - type: nauc_precision_at_5_diff1 - value: 4.705236845538159 - - type: nauc_precision_at_5_max - value: 14.022910843582666 - - type: nauc_precision_at_5_std - value: 11.777269322821605 - - type: nauc_recall_at_1000_diff1 - value: 2.446247548945172 - - type: nauc_recall_at_1000_max - value: 29.14566206426889 - - type: nauc_recall_at_1000_std - value: 49.20704909525879 - - type: nauc_recall_at_100_diff1 - value: 0.1127119672553316 - - type: nauc_recall_at_100_max - value: 17.37584606388062 - - type: nauc_recall_at_100_std - value: 34.660993462440686 - - type: nauc_recall_at_10_diff1 - value: 2.9923183951227927 - - type: nauc_recall_at_10_max - value: 14.261884731124299 - - type: nauc_recall_at_10_std - value: 18.08418879549837 - - type: nauc_recall_at_1_diff1 - value: 76.51940497836482 - - type: nauc_recall_at_1_max - value: 51.251419487235474 - - type: nauc_recall_at_1_std - value: 0.16714896857146574 - - type: nauc_recall_at_20_diff1 - value: 1.918029300830432 - - type: nauc_recall_at_20_max - value: 13.832269193468566 - - type: nauc_recall_at_20_std - value: 21.65284406055605 - - type: nauc_recall_at_3_diff1 - value: 7.226609484731802 - - type: nauc_recall_at_3_max - value: 15.162908526977182 - - type: nauc_recall_at_3_std - value: 8.451859972962634 - - type: nauc_recall_at_5_diff1 - value: 4.705236845538197 - - type: nauc_recall_at_5_max - value: 14.02291084358265 - - type: nauc_recall_at_5_std - value: 11.777269322821638 - - type: ndcg_at_1 - value: 83.45700000000001 - - type: ndcg_at_10 - value: 71.74199999999999 - - type: ndcg_at_100 - value: 75.008 - - type: ndcg_at_1000 - value: 76.242 - - type: ndcg_at_20 - value: 73.114 - - type: ndcg_at_3 - value: 67.128 - - type: ndcg_at_5 - value: 69.645 - - type: precision_at_1 - value: 83.45700000000001 - - type: precision_at_10 - value: 14.747 - - type: precision_at_100 - value: 1.73 - - type: precision_at_1000 - value: 0.189 - - type: precision_at_20 - value: 7.8149999999999995 - - type: precision_at_3 - value: 42.323 - - type: precision_at_5 - value: 27.381 - - type: recall_at_1 - value: 41.729 - - type: recall_at_10 - value: 73.734 - - type: recall_at_100 - value: 86.502 - - type: recall_at_1000 - value: 94.60499999999999 - - type: recall_at_20 - value: 78.14999999999999 - - type: recall_at_3 - value: 63.483999999999995 - - type: recall_at_5 - value: 68.45400000000001 - task: - type: Retrieval - - dataset: - config: default - name: MTEB ImdbClassification - revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 - split: test - type: mteb/imdb - metrics: - - type: accuracy - value: 96.4904 - - type: ap - value: 94.85481918794709 - - type: ap_weighted - value: 94.85481918794709 - - type: f1 - value: 96.4898592305707 - - type: f1_weighted - value: 96.4898592305707 - - type: main_score - value: 96.4904 - task: - type: Classification - - dataset: - config: default - name: MTEB MSMARCO - revision: c5a29a104738b98a9e76336939199e264163d4a0 - split: dev - type: mteb/msmarco - metrics: - - type: main_score - value: 43.692 - - type: map_at_1 - value: 23.751 - - type: map_at_10 - value: 36.553999999999995 - - type: map_at_100 - value: 37.721 - - type: map_at_1000 - value: 37.763999999999996 - - type: map_at_20 - value: 37.289 - - type: map_at_3 - value: 32.643 - - type: map_at_5 - value: 34.851 - - type: mrr_at_1 - value: 24.455587392550143 - - type: mrr_at_10 - value: 37.18388706963206 - - type: mrr_at_100 - value: 38.28330737932916 - - type: mrr_at_1000 - value: 38.32054399710817 - - type: mrr_at_20 - value: 37.8818001216278 - - type: mrr_at_3 - value: 33.35721107927405 - - type: mrr_at_5 - value: 35.52483285577843 - - type: nauc_map_at_1000_diff1 - value: 36.3576177260684 - - type: nauc_map_at_1000_max - value: 7.854511605962703 - - type: nauc_map_at_1000_std - value: -17.701121059746878 - - type: nauc_map_at_100_diff1 - value: 36.356075649230505 - - type: nauc_map_at_100_max - value: 7.862168042999533 - - type: nauc_map_at_100_std - value: -17.670102459097233 - - type: nauc_map_at_10_diff1 - value: 36.22122978875574 - - type: nauc_map_at_10_max - value: 7.80848606967416 - - type: nauc_map_at_10_std - value: -18.3265151386167 - - type: nauc_map_at_1_diff1 - value: 39.28605466408357 - - type: nauc_map_at_1_max - value: 6.20202977590459 - - type: nauc_map_at_1_std - value: -15.734334090045026 - - type: nauc_map_at_20_diff1 - value: 36.33637880909657 - - type: nauc_map_at_20_max - value: 7.843437969476022 - - type: nauc_map_at_20_std - value: -17.917533363025996 - - type: nauc_map_at_3_diff1 - value: 36.24864976076741 - - type: nauc_map_at_3_max - value: 7.420345251835957 - - type: nauc_map_at_3_std - value: -18.71678497722944 - - type: nauc_map_at_5_diff1 - value: 36.0789619291824 - - type: nauc_map_at_5_max - value: 7.7314285669514495 - - type: nauc_map_at_5_std - value: -18.748688764538706 - - type: nauc_mrr_at_1000_diff1 - value: 36.23912675623378 - - type: nauc_mrr_at_1000_max - value: 7.690553436255147 - - type: nauc_mrr_at_1000_std - value: -17.609526070212304 - - type: nauc_mrr_at_100_diff1 - value: 36.23782651189002 - - type: nauc_mrr_at_100_max - value: 7.70075095171647 - - type: nauc_mrr_at_100_std - value: -17.575714144960184 - - type: nauc_mrr_at_10_diff1 - value: 36.125229472534215 - - type: nauc_mrr_at_10_max - value: 7.635472248755658 - - type: nauc_mrr_at_10_std - value: -18.208166616511086 - - type: nauc_mrr_at_1_diff1 - value: 39.20986875554532 - - type: nauc_mrr_at_1_max - value: 6.062668487561363 - - type: nauc_mrr_at_1_std - value: -16.04130340817602 - - type: nauc_mrr_at_20_diff1 - value: 36.21207088739667 - - type: nauc_mrr_at_20_max - value: 7.699610250145951 - - type: nauc_mrr_at_20_std - value: -17.778245221724028 - - type: nauc_mrr_at_3_diff1 - value: 36.03957583885305 - - type: nauc_mrr_at_3_max - value: 7.225515576504581 - - type: nauc_mrr_at_3_std - value: -18.74478742943741 - - type: nauc_mrr_at_5_diff1 - value: 35.969152496648974 - - type: nauc_mrr_at_5_max - value: 7.584059789018233 - - type: nauc_mrr_at_5_std - value: -18.569374723129332 - - type: nauc_ndcg_at_1000_diff1 - value: 35.894655529841806 - - type: nauc_ndcg_at_1000_max - value: 8.579327424366236 - - type: nauc_ndcg_at_1000_std - value: -16.359677367747896 - - type: nauc_ndcg_at_100_diff1 - value: 35.89861902483983 - - type: nauc_ndcg_at_100_max - value: 8.830873623962242 - - type: nauc_ndcg_at_100_std - value: -15.173125564722978 - - type: nauc_ndcg_at_10_diff1 - value: 35.36499811105169 - - type: nauc_ndcg_at_10_max - value: 8.449267180956992 - - type: nauc_ndcg_at_10_std - value: -18.41978802362402 - - type: nauc_ndcg_at_1_diff1 - value: 39.15422481210622 - - type: nauc_ndcg_at_1_max - value: 6.055515791928331 - - type: nauc_ndcg_at_1_std - value: -16.042779610876252 - - type: nauc_ndcg_at_20_diff1 - value: 35.73402868264468 - - type: nauc_ndcg_at_20_max - value: 8.695705518210847 - - type: nauc_ndcg_at_20_std - value: -16.7735829470466 - - type: nauc_ndcg_at_3_diff1 - value: 35.31358242856231 - - type: nauc_ndcg_at_3_max - value: 7.645692789058997 - - type: nauc_ndcg_at_3_std - value: -19.460003734786874 - - type: nauc_ndcg_at_5_diff1 - value: 35.05216588927143 - - type: nauc_ndcg_at_5_max - value: 8.216690520604715 - - type: nauc_ndcg_at_5_std - value: -19.3982054492159 - - type: nauc_precision_at_1000_diff1 - value: -4.440002625111349 - - type: nauc_precision_at_1000_max - value: 7.886988951901723 - - type: nauc_precision_at_1000_std - value: 9.88111187048247 - - type: nauc_precision_at_100_diff1 - value: 15.728286119463325 - - type: nauc_precision_at_100_max - value: 13.218650824470654 - - type: nauc_precision_at_100_std - value: 16.113245895522553 - - type: nauc_precision_at_10_diff1 - value: 29.51218489610567 - - type: nauc_precision_at_10_max - value: 10.197432401942912 - - type: nauc_precision_at_10_std - value: -16.950603431359493 - - type: nauc_precision_at_1_diff1 - value: 39.15422481210622 - - type: nauc_precision_at_1_max - value: 6.055515791928331 - - type: nauc_precision_at_1_std - value: -16.042779610876252 - - type: nauc_precision_at_20_diff1 - value: 27.825993070397338 - - type: nauc_precision_at_20_max - value: 11.437632287846007 - - type: nauc_precision_at_20_std - value: -7.450353566405601 - - type: nauc_precision_at_3_diff1 - value: 32.14135556796588 - - type: nauc_precision_at_3_max - value: 7.989252443574163 - - type: nauc_precision_at_3_std - value: -21.566254595671055 - - type: nauc_precision_at_5_diff1 - value: 30.68778685307082 - - type: nauc_precision_at_5_max - value: 9.332160758499892 - - type: nauc_precision_at_5_std - value: -20.928554713448914 - - type: nauc_recall_at_1000_diff1 - value: 25.00810478716878 - - type: nauc_recall_at_1000_max - value: 46.518165765201644 - - type: nauc_recall_at_1000_std - value: 61.4734635576085 - - type: nauc_recall_at_100_diff1 - value: 33.895581318261726 - - type: nauc_recall_at_100_max - value: 20.10706035872801 - - type: nauc_recall_at_100_std - value: 24.204226584457047 - - type: nauc_recall_at_10_diff1 - value: 32.363127359576296 - - type: nauc_recall_at_10_max - value: 10.729923804989545 - - type: nauc_recall_at_10_std - value: -18.1335370184202 - - type: nauc_recall_at_1_diff1 - value: 39.28605466408357 - - type: nauc_recall_at_1_max - value: 6.20202977590459 - - type: nauc_recall_at_1_std - value: -15.734334090045026 - - type: nauc_recall_at_20_diff1 - value: 33.47804003169795 - - type: nauc_recall_at_20_max - value: 12.781494765263382 - - type: nauc_recall_at_20_std - value: -9.263970132202658 - - type: nauc_recall_at_3_diff1 - value: 32.71001429428999 - - type: nauc_recall_at_3_max - value: 8.353439197382693 - - type: nauc_recall_at_3_std - value: -21.235097744366954 - - type: nauc_recall_at_5_diff1 - value: 31.87451464963415 - - type: nauc_recall_at_5_max - value: 9.635051450907305 - - type: nauc_recall_at_5_std - value: -21.113235357132794 - - type: ndcg_at_1 - value: 24.47 - - type: ndcg_at_10 - value: 43.692 - - type: ndcg_at_100 - value: 49.211 - - type: ndcg_at_1000 - value: 50.244 - - type: ndcg_at_20 - value: 46.278000000000006 - - type: ndcg_at_3 - value: 35.719 - - type: ndcg_at_5 - value: 39.652 - - type: precision_at_1 - value: 24.47 - - type: precision_at_10 - value: 6.857 - - type: precision_at_100 - value: 0.9610000000000001 - - type: precision_at_1000 - value: 0.105 - - type: precision_at_20 - value: 3.968 - - type: precision_at_3 - value: 15.181000000000001 - - type: precision_at_5 - value: 11.117 - - type: recall_at_1 - value: 23.751 - - type: recall_at_10 - value: 65.64 - - type: recall_at_100 - value: 90.967 - - type: recall_at_1000 - value: 98.738 - - type: recall_at_20 - value: 75.639 - - type: recall_at_3 - value: 43.927 - - type: recall_at_5 - value: 53.366 - task: - type: Retrieval - - dataset: - config: en - name: MTEB MTOPDomainClassification (en) - revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf - split: test - type: mteb/mtop_domain - metrics: - - type: accuracy - value: 98.82580939352485 - - type: f1 - value: 98.75201754333801 - - type: f1_weighted - value: 98.82795205108245 - - type: main_score - value: 98.82580939352485 - task: - type: Classification - - dataset: - config: en - name: MTEB MTOPIntentClassification (en) - revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba - split: test - type: mteb/mtop_intent - metrics: - - type: accuracy - value: 92.29822161422709 - - type: f1 - value: 77.75210224871594 - - type: f1_weighted - value: 93.58661422540348 - - type: main_score - value: 92.29822161422709 - task: - type: Classification - - dataset: - config: en - name: MTEB MassiveIntentClassification (en) - revision: 4672e20407010da34463acc759c162ca9734bca6 - split: test - type: mteb/amazon_massive_intent - metrics: - - type: accuracy - value: 85.17484868863484 - - type: f1 - value: 81.94484244487094 - - type: f1_weighted - value: 85.21022593423332 - - type: main_score - value: 85.17484868863484 - task: - type: Classification - - dataset: - config: en - name: MTEB MassiveScenarioClassification (en) - revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 - split: test - type: mteb/amazon_massive_scenario - metrics: - - type: accuracy - value: 89.61667787491594 - - type: f1 - value: 89.02701927621264 - - type: f1_weighted - value: 89.56306982022801 - - type: main_score - value: 89.61667787491594 - task: - type: Classification - - dataset: - config: default - name: MTEB MedrxivClusteringP2P - revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 - split: test - type: mteb/medrxiv-clustering-p2p - metrics: - - type: main_score - value: 46.318282423948574 - - type: v_measure - value: 46.318282423948574 - - type: v_measure_std - value: 0.9729055662461538 - task: - type: Clustering - - dataset: - config: default - name: MTEB MedrxivClusteringS2S - revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 - split: test - type: mteb/medrxiv-clustering-s2s - metrics: - - type: main_score - value: 44.29033625273981 - - type: v_measure - value: 44.29033625273981 - - type: v_measure_std - value: 1.0596383629128594 - task: - type: Clustering - - dataset: - config: default - name: MTEB MindSmallReranking - revision: 59042f120c80e8afa9cdbb224f67076cec0fc9a7 - split: test - type: mteb/mind_small - metrics: - - type: main_score - value: 33.0526129239962 - - type: map - value: 33.0526129239962 - - type: mrr - value: 34.29260046890935 - - type: nAUC_map_diff1 - value: 12.579738077238032 - - type: nAUC_map_max - value: -20.936629344962 - - type: nAUC_map_std - value: -1.6096805784945216 - - type: nAUC_mrr_diff1 - value: 11.597584463580807 - - type: nAUC_mrr_max - value: -15.723702838537504 - - type: nAUC_mrr_std - value: 0.2719172965777737 - task: - type: Reranking - - dataset: - config: default - name: MTEB NFCorpus - revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 - split: test - type: mteb/nfcorpus - metrics: - - type: main_score - value: 41.486000000000004 - - type: map_at_1 - value: 6.866 - - type: map_at_10 - value: 15.895999999999999 - - type: map_at_100 - value: 21.093 - - type: map_at_1000 - value: 23.067 - - type: map_at_20 - value: 18.125 - - type: map_at_3 - value: 11.421000000000001 - - type: map_at_5 - value: 13.415 - - type: mrr_at_1 - value: 52.63157894736842 - - type: mrr_at_10 - value: 61.486805248415166 - - type: mrr_at_100 - value: 62.08211009182091 - - type: mrr_at_1000 - value: 62.10828701365016 - - type: mrr_at_20 - value: 61.904411187915784 - - type: mrr_at_3 - value: 59.90712074303407 - - type: mrr_at_5 - value: 60.91331269349847 - - type: nauc_map_at_1000_diff1 - value: 25.484625278529403 - - type: nauc_map_at_1000_max - value: 31.206600396418853 - - type: nauc_map_at_1000_std - value: 15.569448072357156 - - type: nauc_map_at_100_diff1 - value: 27.636750226316764 - - type: nauc_map_at_100_max - value: 29.66992681250722 - - type: nauc_map_at_100_std - value: 10.570600484002671 - - type: nauc_map_at_10_diff1 - value: 32.76642525548697 - - type: nauc_map_at_10_max - value: 21.459225397237663 - - type: nauc_map_at_10_std - value: -3.546494734209264 - - type: nauc_map_at_1_diff1 - value: 48.8002894871328 - - type: nauc_map_at_1_max - value: 5.7236722609868815 - - type: nauc_map_at_1_std - value: -13.283554044471352 - - type: nauc_map_at_20_diff1 - value: 30.57169701502308 - - type: nauc_map_at_20_max - value: 25.79666139518404 - - type: nauc_map_at_20_std - value: 1.781732492989651 - - type: nauc_map_at_3_diff1 - value: 40.076315947201095 - - type: nauc_map_at_3_max - value: 12.862524429140054 - - type: nauc_map_at_3_std - value: -9.188349777126817 - - type: nauc_map_at_5_diff1 - value: 36.9918718052938 - - type: nauc_map_at_5_max - value: 16.74234374361876 - - type: nauc_map_at_5_std - value: -7.818523349307494 - - type: nauc_mrr_at_1000_diff1 - value: 26.88183002609805 - - type: nauc_mrr_at_1000_max - value: 47.10209348428658 - - type: nauc_mrr_at_1000_std - value: 32.067825924992924 - - type: nauc_mrr_at_100_diff1 - value: 26.871482491566745 - - type: nauc_mrr_at_100_max - value: 47.11303868498556 - - type: nauc_mrr_at_100_std - value: 32.08961428818868 - - type: nauc_mrr_at_10_diff1 - value: 26.6356914977722 - - type: nauc_mrr_at_10_max - value: 47.091624558810366 - - type: nauc_mrr_at_10_std - value: 31.942424120660164 - - type: nauc_mrr_at_1_diff1 - value: 28.19774198483673 - - type: nauc_mrr_at_1_max - value: 41.44380927834253 - - type: nauc_mrr_at_1_std - value: 25.18222691885917 - - type: nauc_mrr_at_20_diff1 - value: 26.86487347109452 - - type: nauc_mrr_at_20_max - value: 47.1987778214726 - - type: nauc_mrr_at_20_std - value: 32.143517921610034 - - type: nauc_mrr_at_3_diff1 - value: 27.34340373236422 - - type: nauc_mrr_at_3_max - value: 46.358726506276646 - - type: nauc_mrr_at_3_std - value: 31.74924155572593 - - type: nauc_mrr_at_5_diff1 - value: 27.209667205060672 - - type: nauc_mrr_at_5_max - value: 46.79883369072009 - - type: nauc_mrr_at_5_std - value: 31.655605306670758 - - type: nauc_ndcg_at_1000_diff1 - value: 18.940195769769687 - - type: nauc_ndcg_at_1000_max - value: 46.48551313937331 - - type: nauc_ndcg_at_1000_std - value: 33.64819502089232 - - type: nauc_ndcg_at_100_diff1 - value: 19.50885253809146 - - type: nauc_ndcg_at_100_max - value: 40.53174462354878 - - type: nauc_ndcg_at_100_std - value: 28.516152877751118 - - type: nauc_ndcg_at_10_diff1 - value: 16.01699218096564 - - type: nauc_ndcg_at_10_max - value: 41.17322878314514 - - type: nauc_ndcg_at_10_std - value: 29.002233224832196 - - type: nauc_ndcg_at_1_diff1 - value: 27.443547710102205 - - type: nauc_ndcg_at_1_max - value: 40.66529763309582 - - type: nauc_ndcg_at_1_std - value: 24.15016766225869 - - type: nauc_ndcg_at_20_diff1 - value: 17.541197675685062 - - type: nauc_ndcg_at_20_max - value: 40.53231266973844 - - type: nauc_ndcg_at_20_std - value: 29.54096347876548 - - type: nauc_ndcg_at_3_diff1 - value: 18.649628357473716 - - type: nauc_ndcg_at_3_max - value: 41.18603570171764 - - type: nauc_ndcg_at_3_std - value: 27.125524188420396 - - type: nauc_ndcg_at_5_diff1 - value: 17.519593751448483 - - type: nauc_ndcg_at_5_max - value: 42.715997890377345 - - type: nauc_ndcg_at_5_std - value: 27.902627839899868 - - type: nauc_precision_at_1000_diff1 - value: -15.528797630565155 - - type: nauc_precision_at_1000_max - value: 13.741640921778671 - - type: nauc_precision_at_1000_std - value: 44.50896053788372 - - type: nauc_precision_at_100_diff1 - value: -14.491464489721887 - - type: nauc_precision_at_100_max - value: 23.136434418999457 - - type: nauc_precision_at_100_std - value: 49.73145147863128 - - type: nauc_precision_at_10_diff1 - value: -4.829188942994277 - - type: nauc_precision_at_10_max - value: 40.327612559528866 - - type: nauc_precision_at_10_std - value: 39.34919529635044 - - type: nauc_precision_at_1_diff1 - value: 28.19774198483673 - - type: nauc_precision_at_1_max - value: 41.44380927834253 - - type: nauc_precision_at_1_std - value: 25.18222691885917 - - type: nauc_precision_at_20_diff1 - value: -7.210726293112847 - - type: nauc_precision_at_20_max - value: 37.195679576636984 - - type: nauc_precision_at_20_std - value: 45.4597096418357 - - type: nauc_precision_at_3_diff1 - value: 7.578219537774854 - - type: nauc_precision_at_3_max - value: 41.59775233475654 - - type: nauc_precision_at_3_std - value: 30.764584790895118 - - type: nauc_precision_at_5_diff1 - value: 1.655451789039598 - - type: nauc_precision_at_5_max - value: 43.435739407610455 - - type: nauc_precision_at_5_std - value: 33.42552263325999 - - type: nauc_recall_at_1000_diff1 - value: 5.030705700690516 - - type: nauc_recall_at_1000_max - value: 19.108072570815583 - - type: nauc_recall_at_1000_std - value: 14.697734974217308 - - type: nauc_recall_at_100_diff1 - value: 14.746540318132407 - - type: nauc_recall_at_100_max - value: 21.798705033854795 - - type: nauc_recall_at_100_std - value: 11.416195108842587 - - type: nauc_recall_at_10_diff1 - value: 25.548642427860486 - - type: nauc_recall_at_10_max - value: 18.711677681987474 - - type: nauc_recall_at_10_std - value: -5.988904818971677 - - type: nauc_recall_at_1_diff1 - value: 48.8002894871328 - - type: nauc_recall_at_1_max - value: 5.7236722609868815 - - type: nauc_recall_at_1_std - value: -13.283554044471352 - - type: nauc_recall_at_20_diff1 - value: 23.39140739154809 - - type: nauc_recall_at_20_max - value: 19.351150636155474 - - type: nauc_recall_at_20_std - value: -2.757280266915132 - - type: nauc_recall_at_3_diff1 - value: 38.17453576012812 - - type: nauc_recall_at_3_max - value: 13.47003839643972 - - type: nauc_recall_at_3_std - value: -8.75780163862688 - - type: nauc_recall_at_5_diff1 - value: 33.02812855226899 - - type: nauc_recall_at_5_max - value: 15.477626408978477 - - type: nauc_recall_at_5_std - value: -9.072206441070708 - - type: ndcg_at_1 - value: 50.773999999999994 - - type: ndcg_at_10 - value: 41.486000000000004 - - type: ndcg_at_100 - value: 39.051 - - type: ndcg_at_1000 - value: 48.106 - - type: ndcg_at_20 - value: 39.432 - - type: ndcg_at_3 - value: 47.428 - - type: ndcg_at_5 - value: 45.227000000000004 - - type: precision_at_1 - value: 52.632 - - type: precision_at_10 - value: 31.146 - - type: precision_at_100 - value: 10.328 - - type: precision_at_1000 - value: 2.432 - - type: precision_at_20 - value: 23.793 - - type: precision_at_3 - value: 45.201 - - type: precision_at_5 - value: 39.876 - - type: recall_at_1 - value: 6.866 - - type: recall_at_10 - value: 20.447000000000003 - - type: recall_at_100 - value: 40.607 - - type: recall_at_1000 - value: 73.411 - - type: recall_at_20 - value: 26.082 - - type: recall_at_3 - value: 12.484 - - type: recall_at_5 - value: 15.847 - task: - type: Retrieval - - dataset: - config: default - name: MTEB NQ - revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 - split: test - type: mteb/nq - metrics: - - type: main_score - value: 69.072 - - type: map_at_1 - value: 45.483000000000004 - - type: map_at_10 - value: 62.050000000000004 - - type: map_at_100 - value: 62.693 - - type: map_at_1000 - value: 62.702999999999996 - - type: map_at_20 - value: 62.498 - - type: map_at_3 - value: 58.285 - - type: map_at_5 - value: 60.711000000000006 - - type: mrr_at_1 - value: 50.840092699884124 - - type: mrr_at_10 - value: 64.54635224116673 - - type: mrr_at_100 - value: 64.9526548702289 - - type: mrr_at_1000 - value: 64.95908460752281 - - type: mrr_at_20 - value: 64.82949565799959 - - type: mrr_at_3 - value: 61.89165701042856 - - type: mrr_at_5 - value: 63.632676709154026 - - type: nauc_map_at_1000_diff1 - value: 43.187285304185224 - - type: nauc_map_at_1000_max - value: 32.39921659632756 - - type: nauc_map_at_1000_std - value: -5.780901333066553 - - type: nauc_map_at_100_diff1 - value: 43.184487221204456 - - type: nauc_map_at_100_max - value: 32.41176116347982 - - type: nauc_map_at_100_std - value: -5.76422606662383 - - type: nauc_map_at_10_diff1 - value: 42.967066814031746 - - type: nauc_map_at_10_max - value: 32.489617364418514 - - type: nauc_map_at_10_std - value: -6.029045531102664 - - type: nauc_map_at_1_diff1 - value: 46.16376563218624 - - type: nauc_map_at_1_max - value: 26.342624776802232 - - type: nauc_map_at_1_std - value: -7.142171388751972 - - type: nauc_map_at_20_diff1 - value: 43.15894358608328 - - type: nauc_map_at_20_max - value: 32.46492198956245 - - type: nauc_map_at_20_std - value: -5.788373305449195 - - type: nauc_map_at_3_diff1 - value: 43.231752344608545 - - type: nauc_map_at_3_max - value: 31.68003009949564 - - type: nauc_map_at_3_std - value: -8.015235132765458 - - type: nauc_map_at_5_diff1 - value: 42.86197608819917 - - type: nauc_map_at_5_max - value: 32.363857571094485 - - type: nauc_map_at_5_std - value: -6.780487416387977 - - type: nauc_mrr_at_1000_diff1 - value: 43.40542912045782 - - type: nauc_mrr_at_1000_max - value: 32.8461770324533 - - type: nauc_mrr_at_1000_std - value: -3.6505425530008204 - - type: nauc_mrr_at_100_diff1 - value: 43.40233508014468 - - type: nauc_mrr_at_100_max - value: 32.85598538385942 - - type: nauc_mrr_at_100_std - value: -3.637477352635459 - - type: nauc_mrr_at_10_diff1 - value: 43.260179162806054 - - type: nauc_mrr_at_10_max - value: 32.942643527040474 - - type: nauc_mrr_at_10_std - value: -3.712052825320437 - - type: nauc_mrr_at_1_diff1 - value: 46.354919460881206 - - type: nauc_mrr_at_1_max - value: 29.1760258591106 - - type: nauc_mrr_at_1_std - value: -4.107225031227406 - - type: nauc_mrr_at_20_diff1 - value: 43.37092385434311 - - type: nauc_mrr_at_20_max - value: 32.93390254712846 - - type: nauc_mrr_at_20_std - value: -3.5719056112132006 - - type: nauc_mrr_at_3_diff1 - value: 43.1744474040527 - - type: nauc_mrr_at_3_max - value: 32.741290559777994 - - type: nauc_mrr_at_3_std - value: -4.72677925120697 - - type: nauc_mrr_at_5_diff1 - value: 43.108396819975674 - - type: nauc_mrr_at_5_max - value: 32.970519514893084 - - type: nauc_mrr_at_5_std - value: -4.090906158975974 - - type: nauc_ndcg_at_1000_diff1 - value: 42.786664193638714 - - type: nauc_ndcg_at_1000_max - value: 33.65554095609296 - - type: nauc_ndcg_at_1000_std - value: -4.024030130584482 - - type: nauc_ndcg_at_100_diff1 - value: 42.691246775210814 - - type: nauc_ndcg_at_100_max - value: 34.063232335110875 - - type: nauc_ndcg_at_100_std - value: -3.477813807415248 - - type: nauc_ndcg_at_10_diff1 - value: 41.90988990571757 - - type: nauc_ndcg_at_10_max - value: 34.58934812881633 - - type: nauc_ndcg_at_10_std - value: -4.3295110195497655 - - type: nauc_ndcg_at_1_diff1 - value: 46.354919460881206 - - type: nauc_ndcg_at_1_max - value: 29.1760258591106 - - type: nauc_ndcg_at_1_std - value: -4.107225031227406 - - type: nauc_ndcg_at_20_diff1 - value: 42.493206675867114 - - type: nauc_ndcg_at_20_max - value: 34.562441307459544 - - type: nauc_ndcg_at_20_std - value: -3.4456116866749107 - - type: nauc_ndcg_at_3_diff1 - value: 42.24180336502808 - - type: nauc_ndcg_at_3_max - value: 33.064267018100594 - - type: nauc_ndcg_at_3_std - value: -7.786248093572142 - - type: nauc_ndcg_at_5_diff1 - value: 41.692714787779565 - - type: nauc_ndcg_at_5_max - value: 34.20502498949156 - - type: nauc_ndcg_at_5_std - value: -5.979557859282785 - - type: nauc_precision_at_1000_diff1 - value: -13.779832506640702 - - type: nauc_precision_at_1000_max - value: 1.243001688631421 - - type: nauc_precision_at_1000_std - value: 17.351623398622323 - - type: nauc_precision_at_100_diff1 - value: -11.310526816290297 - - type: nauc_precision_at_100_max - value: 5.771669506192959 - - type: nauc_precision_at_100_std - value: 19.917795079540113 - - type: nauc_precision_at_10_diff1 - value: 2.163699384635286 - - type: nauc_precision_at_10_max - value: 19.66440698458386 - - type: nauc_precision_at_10_std - value: 13.689876348315726 - - type: nauc_precision_at_1_diff1 - value: 46.354919460881206 - - type: nauc_precision_at_1_max - value: 29.1760258591106 - - type: nauc_precision_at_1_std - value: -4.107225031227406 - - type: nauc_precision_at_20_diff1 - value: -3.038735879584471 - - type: nauc_precision_at_20_max - value: 14.132968299701695 - - type: nauc_precision_at_20_std - value: 17.78069734664346 - - type: nauc_precision_at_3_diff1 - value: 21.783760758070095 - - type: nauc_precision_at_3_max - value: 30.244127986404497 - - type: nauc_precision_at_3_std - value: -0.12411163467738723 - - type: nauc_precision_at_5_diff1 - value: 10.980635723302418 - - type: nauc_precision_at_5_max - value: 25.302293738975575 - - type: nauc_precision_at_5_std - value: 6.4740817488722024 - - type: nauc_recall_at_1000_diff1 - value: 34.10343772356593 - - type: nauc_recall_at_1000_max - value: 80.72497340357538 - - type: nauc_recall_at_1000_std - value: 69.54564103264093 - - type: nauc_recall_at_100_diff1 - value: 33.427719956774126 - - type: nauc_recall_at_100_max - value: 71.54086768335449 - - type: nauc_recall_at_100_std - value: 49.66157377654885 - - type: nauc_recall_at_10_diff1 - value: 33.70139560054039 - - type: nauc_recall_at_10_max - value: 45.47878072860151 - - type: nauc_recall_at_10_std - value: 1.4188516615716378 - - type: nauc_recall_at_1_diff1 - value: 46.16376563218624 - - type: nauc_recall_at_1_max - value: 26.342624776802232 - - type: nauc_recall_at_1_std - value: -7.142171388751972 - - type: nauc_recall_at_20_diff1 - value: 35.805379874970086 - - type: nauc_recall_at_20_max - value: 51.80479822253392 - - type: nauc_recall_at_20_std - value: 13.531467576460143 - - type: nauc_recall_at_3_diff1 - value: 37.288500141631616 - - type: nauc_recall_at_3_max - value: 35.07078243516728 - - type: nauc_recall_at_3_std - value: -10.452926441410405 - - type: nauc_recall_at_5_diff1 - value: 34.83186104526897 - - type: nauc_recall_at_5_max - value: 39.58488976496973 - - type: nauc_recall_at_5_std - value: -6.3049292065708835 - - type: ndcg_at_1 - value: 50.839999999999996 - - type: ndcg_at_10 - value: 69.072 - - type: ndcg_at_100 - value: 71.538 - - type: ndcg_at_1000 - value: 71.77799999999999 - - type: ndcg_at_20 - value: 70.41 - - type: ndcg_at_3 - value: 62.544999999999995 - - type: ndcg_at_5 - value: 66.33099999999999 - - type: precision_at_1 - value: 50.839999999999996 - - type: precision_at_10 - value: 10.495000000000001 - - type: precision_at_100 - value: 1.1900000000000002 - - type: precision_at_1000 - value: 0.121 - - type: precision_at_20 - value: 5.5809999999999995 - - type: precision_at_3 - value: 27.636 - - type: precision_at_5 - value: 18.864 - - type: recall_at_1 - value: 45.483000000000004 - - type: recall_at_10 - value: 87.483 - - type: recall_at_100 - value: 97.844 - - type: recall_at_1000 - value: 99.66199999999999 - - type: recall_at_20 - value: 92.294 - - type: recall_at_3 - value: 71.2 - - type: recall_at_5 - value: 79.753 - task: - type: Retrieval - - dataset: - config: default - name: MTEB QuoraRetrieval - revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 - split: test - type: mteb/quora - metrics: - - type: main_score - value: 89.58 - - type: map_at_1 - value: 71.819 - - type: map_at_10 - value: 86.04899999999999 - - type: map_at_100 - value: 86.648 - - type: map_at_1000 - value: 86.66199999999999 - - type: map_at_20 - value: 86.441 - - type: map_at_3 - value: 83.114 - - type: map_at_5 - value: 84.981 - - type: mrr_at_1 - value: 82.62 - - type: mrr_at_10 - value: 88.62899999999979 - - type: mrr_at_100 - value: 88.70918591324215 - - type: mrr_at_1000 - value: 88.70973091492397 - - type: mrr_at_20 - value: 88.68914765317221 - - type: mrr_at_3 - value: 87.74999999999979 - - type: mrr_at_5 - value: 88.36799999999974 - - type: nauc_map_at_1000_diff1 - value: 77.89207709760448 - - type: nauc_map_at_1000_max - value: 29.63371361495422 - - type: nauc_map_at_1000_std - value: -48.628180385874344 - - type: nauc_map_at_100_diff1 - value: 77.89592179104915 - - type: nauc_map_at_100_max - value: 29.617171506130756 - - type: nauc_map_at_100_std - value: -48.66057170774648 - - type: nauc_map_at_10_diff1 - value: 78.0618161228185 - - type: nauc_map_at_10_max - value: 29.178490609366737 - - type: nauc_map_at_10_std - value: -50.74755004592002 - - type: nauc_map_at_1_diff1 - value: 81.64335579973574 - - type: nauc_map_at_1_max - value: 21.813832226652174 - - type: nauc_map_at_1_std - value: -42.57570978190876 - - type: nauc_map_at_20_diff1 - value: 77.9299081005938 - - type: nauc_map_at_20_max - value: 29.458718470003888 - - type: nauc_map_at_20_std - value: -49.63337236763102 - - type: nauc_map_at_3_diff1 - value: 78.72941448509229 - - type: nauc_map_at_3_max - value: 26.600997896960056 - - type: nauc_map_at_3_std - value: -51.889002227479885 - - type: nauc_map_at_5_diff1 - value: 78.31466610917171 - - type: nauc_map_at_5_max - value: 28.09863984582896 - - type: nauc_map_at_5_std - value: -52.14058096096497 - - type: nauc_mrr_at_1000_diff1 - value: 78.42667263739992 - - type: nauc_mrr_at_1000_max - value: 31.98996235127974 - - type: nauc_mrr_at_1000_std - value: -44.380439148429296 - - type: nauc_mrr_at_100_diff1 - value: 78.42661032698115 - - type: nauc_mrr_at_100_max - value: 31.991652631740102 - - type: nauc_mrr_at_100_std - value: -44.37854108460535 - - type: nauc_mrr_at_10_diff1 - value: 78.39126022544136 - - type: nauc_mrr_at_10_max - value: 32.02023484451197 - - type: nauc_mrr_at_10_std - value: -44.561252349176954 - - type: nauc_mrr_at_1_diff1 - value: 79.21630894647448 - - type: nauc_mrr_at_1_max - value: 31.526303156060177 - - type: nauc_mrr_at_1_std - value: -41.887504422443136 - - type: nauc_mrr_at_20_diff1 - value: 78.42548039170424 - - type: nauc_mrr_at_20_max - value: 31.99588275070137 - - type: nauc_mrr_at_20_std - value: -44.44957722627042 - - type: nauc_mrr_at_3_diff1 - value: 78.26165151833735 - - type: nauc_mrr_at_3_max - value: 32.18028826126801 - - type: nauc_mrr_at_3_std - value: -44.6998237213182 - - type: nauc_mrr_at_5_diff1 - value: 78.34786430903962 - - type: nauc_mrr_at_5_max - value: 32.168476272879566 - - type: nauc_mrr_at_5_std - value: -44.7915919956712 - - type: nauc_ndcg_at_1000_diff1 - value: 77.79198355957816 - - type: nauc_ndcg_at_1000_max - value: 31.14363511518406 - - type: nauc_ndcg_at_1000_std - value: -46.69335151274275 - - type: nauc_ndcg_at_100_diff1 - value: 77.79898090286419 - - type: nauc_ndcg_at_100_max - value: 31.115103811629215 - - type: nauc_ndcg_at_100_std - value: -46.73078913421965 - - type: nauc_ndcg_at_10_diff1 - value: 77.74856635461343 - - type: nauc_ndcg_at_10_max - value: 30.279584686212747 - - type: nauc_ndcg_at_10_std - value: -50.23514662356807 - - type: nauc_ndcg_at_1_diff1 - value: 79.17833000040999 - - type: nauc_ndcg_at_1_max - value: 31.703788144510746 - - type: nauc_ndcg_at_1_std - value: -41.854817402870715 - - type: nauc_ndcg_at_20_diff1 - value: 77.7380353804671 - - type: nauc_ndcg_at_20_max - value: 30.622294129001553 - - type: nauc_ndcg_at_20_std - value: -49.035794761065254 - - type: nauc_ndcg_at_3_diff1 - value: 77.41476880573593 - - type: nauc_ndcg_at_3_max - value: 29.015949978243032 - - type: nauc_ndcg_at_3_std - value: -49.78627087622648 - - type: nauc_ndcg_at_5_diff1 - value: 77.64439137502896 - - type: nauc_ndcg_at_5_max - value: 29.444684897492206 - - type: nauc_ndcg_at_5_std - value: -51.21908400252501 - - type: nauc_precision_at_1000_diff1 - value: -44.92396459446822 - - type: nauc_precision_at_1000_max - value: -3.674153720989045 - - type: nauc_precision_at_1000_std - value: 39.56552468277785 - - type: nauc_precision_at_100_diff1 - value: -44.75143023259094 - - type: nauc_precision_at_100_max - value: -3.705280025140011 - - type: nauc_precision_at_100_std - value: 39.433619999113326 - - type: nauc_precision_at_10_diff1 - value: -41.0651074726579 - - type: nauc_precision_at_10_max - value: -0.21097985601783667 - - type: nauc_precision_at_10_std - value: 26.24652824589493 - - type: nauc_precision_at_1_diff1 - value: 79.17833000040999 - - type: nauc_precision_at_1_max - value: 31.703788144510746 - - type: nauc_precision_at_1_std - value: -41.854817402870715 - - type: nauc_precision_at_20_diff1 - value: -43.368001340920294 - - type: nauc_precision_at_20_max - value: -2.036990010399129 - - type: nauc_precision_at_20_std - value: 32.37747041406297 - - type: nauc_precision_at_3_diff1 - value: -22.089307548346877 - - type: nauc_precision_at_3_max - value: 6.2280973175296 - - type: nauc_precision_at_3_std - value: 5.323992514036145 - - type: nauc_precision_at_5_diff1 - value: -34.07115055244003 - - type: nauc_precision_at_5_max - value: 2.5955315789198834 - - type: nauc_precision_at_5_std - value: 16.26096689407332 - - type: nauc_recall_at_1000_diff1 - value: 58.27703860947467 - - type: nauc_recall_at_1000_max - value: 68.59835835315768 - - type: nauc_recall_at_1000_std - value: 77.96687006056064 - - type: nauc_recall_at_100_diff1 - value: 73.24371223081737 - - type: nauc_recall_at_100_max - value: 39.55925344664591 - - type: nauc_recall_at_100_std - value: -32.25605030215798 - - type: nauc_recall_at_10_diff1 - value: 73.41261201339202 - - type: nauc_recall_at_10_max - value: 26.822979434062926 - - type: nauc_recall_at_10_std - value: -74.2909332592806 - - type: nauc_recall_at_1_diff1 - value: 81.64335579973574 - - type: nauc_recall_at_1_max - value: 21.813832226652174 - - type: nauc_recall_at_1_std - value: -42.57570978190876 - - type: nauc_recall_at_20_diff1 - value: 72.7621297920656 - - type: nauc_recall_at_20_max - value: 26.02492304096079 - - type: nauc_recall_at_20_std - value: -77.8724532438279 - - type: nauc_recall_at_3_diff1 - value: 75.25149312810714 - - type: nauc_recall_at_3_max - value: 23.20545662481487 - - type: nauc_recall_at_3_std - value: -59.69689982140521 - - type: nauc_recall_at_5_diff1 - value: 73.69807273001406 - - type: nauc_recall_at_5_max - value: 24.073666798066057 - - type: nauc_recall_at_5_std - value: -67.91121268130719 - - type: ndcg_at_1 - value: 82.64 - - type: ndcg_at_10 - value: 89.58 - - type: ndcg_at_100 - value: 90.606 - - type: ndcg_at_1000 - value: 90.676 - - type: ndcg_at_20 - value: 90.132 - - type: ndcg_at_3 - value: 86.88 - - type: ndcg_at_5 - value: 88.40299999999999 - - type: precision_at_1 - value: 82.64 - - type: precision_at_10 - value: 13.604 - - type: precision_at_100 - value: 1.539 - - type: precision_at_1000 - value: 0.157 - - type: precision_at_20 - value: 7.188 - - type: precision_at_3 - value: 38.083 - - type: precision_at_5 - value: 25.018 - - type: recall_at_1 - value: 71.819 - - type: recall_at_10 - value: 96.34700000000001 - - type: recall_at_100 - value: 99.715 - - type: recall_at_1000 - value: 99.995 - - type: recall_at_20 - value: 98.073 - - type: recall_at_3 - value: 88.57300000000001 - - type: recall_at_5 - value: 92.908 - task: - type: Retrieval - - dataset: - config: default - name: MTEB RedditClustering - revision: 24640382cdbf8abc73003fb0fa6d111a705499eb - split: test - type: mteb/reddit-clustering - metrics: - - type: main_score - value: 71.18966762070158 - - type: v_measure - value: 71.18966762070158 - - type: v_measure_std - value: 2.7498969054457048 - task: - type: Clustering - - dataset: - config: default - name: MTEB RedditClusteringP2P - revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 - split: test - type: mteb/reddit-clustering-p2p - metrics: - - type: main_score - value: 74.42014716862516 - - type: v_measure - value: 74.42014716862516 - - type: v_measure_std - value: 9.909739891410648 - task: - type: Clustering - - dataset: - config: default - name: MTEB SCIDOCS - revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 - split: test - type: mteb/scidocs - metrics: - - type: main_score - value: 25.041999999999998 - - type: map_at_1 - value: 5.893000000000001 - - type: map_at_10 - value: 15.260000000000002 - - type: map_at_100 - value: 18.084 - - type: map_at_1000 - value: 18.467 - - type: map_at_20 - value: 16.675 - - type: map_at_3 - value: 10.526 - - type: map_at_5 - value: 12.775 - - type: mrr_at_1 - value: 28.999999999999996 - - type: mrr_at_10 - value: 41.03575396825395 - - type: mrr_at_100 - value: 42.136771862785835 - - type: mrr_at_1000 - value: 42.16698555415099 - - type: mrr_at_20 - value: 41.707493696104315 - - type: mrr_at_3 - value: 37.34999999999998 - - type: mrr_at_5 - value: 39.59999999999995 - - type: nauc_map_at_1000_diff1 - value: 12.080002654911883 - - type: nauc_map_at_1000_max - value: 29.813563682286276 - - type: nauc_map_at_1000_std - value: 20.36659817908673 - - type: nauc_map_at_100_diff1 - value: 12.108735517749706 - - type: nauc_map_at_100_max - value: 29.76830671710955 - - type: nauc_map_at_100_std - value: 20.3433621032846 - - type: nauc_map_at_10_diff1 - value: 12.91575031185637 - - type: nauc_map_at_10_max - value: 29.427600958386318 - - type: nauc_map_at_10_std - value: 16.89867275177153 - - type: nauc_map_at_1_diff1 - value: 19.353069488987916 - - type: nauc_map_at_1_max - value: 17.093914951159693 - - type: nauc_map_at_1_std - value: 8.19886078055046 - - type: nauc_map_at_20_diff1 - value: 11.977233457943113 - - type: nauc_map_at_20_max - value: 29.171812822948805 - - type: nauc_map_at_20_std - value: 18.780517506173965 - - type: nauc_map_at_3_diff1 - value: 14.453129464176092 - - type: nauc_map_at_3_max - value: 25.801958649112077 - - type: nauc_map_at_3_std - value: 11.572823684429643 - - type: nauc_map_at_5_diff1 - value: 13.167155808104997 - - type: nauc_map_at_5_max - value: 27.355626948365792 - - type: nauc_map_at_5_std - value: 14.414151839192183 - - type: nauc_mrr_at_1000_diff1 - value: 17.262104643988636 - - type: nauc_mrr_at_1000_max - value: 23.991373837217058 - - type: nauc_mrr_at_1000_std - value: 12.44755488671623 - - type: nauc_mrr_at_100_diff1 - value: 17.267280132318703 - - type: nauc_mrr_at_100_max - value: 24.022189287889294 - - type: nauc_mrr_at_100_std - value: 12.480695500214788 - - type: nauc_mrr_at_10_diff1 - value: 17.012383998246268 - - type: nauc_mrr_at_10_max - value: 24.192637911171722 - - type: nauc_mrr_at_10_std - value: 12.524608847408917 - - type: nauc_mrr_at_1_diff1 - value: 19.43518811038007 - - type: nauc_mrr_at_1_max - value: 17.747482933395602 - - type: nauc_mrr_at_1_std - value: 8.410779775558684 - - type: nauc_mrr_at_20_diff1 - value: 17.202663281407446 - - type: nauc_mrr_at_20_max - value: 24.091991130543118 - - type: nauc_mrr_at_20_std - value: 12.503814263019908 - - type: nauc_mrr_at_3_diff1 - value: 17.52733013432995 - - type: nauc_mrr_at_3_max - value: 23.569459518780214 - - type: nauc_mrr_at_3_std - value: 11.770846827520726 - - type: nauc_mrr_at_5_diff1 - value: 17.10817561975543 - - type: nauc_mrr_at_5_max - value: 23.945141435234678 - - type: nauc_mrr_at_5_std - value: 12.034468615317719 - - type: nauc_ndcg_at_1000_diff1 - value: 12.317811393346936 - - type: nauc_ndcg_at_1000_max - value: 30.809991350156103 - - type: nauc_ndcg_at_1000_std - value: 24.517501065205067 - - type: nauc_ndcg_at_100_diff1 - value: 12.824804203182936 - - type: nauc_ndcg_at_100_max - value: 30.895499817010748 - - type: nauc_ndcg_at_100_std - value: 25.424376279745402 - - type: nauc_ndcg_at_10_diff1 - value: 13.32724552457439 - - type: nauc_ndcg_at_10_max - value: 30.409088666807456 - - type: nauc_ndcg_at_10_std - value: 18.216330475714113 - - type: nauc_ndcg_at_1_diff1 - value: 19.43518811038007 - - type: nauc_ndcg_at_1_max - value: 17.747482933395602 - - type: nauc_ndcg_at_1_std - value: 8.410779775558684 - - type: nauc_ndcg_at_20_diff1 - value: 12.224399111852902 - - type: nauc_ndcg_at_20_max - value: 29.86352330445272 - - type: nauc_ndcg_at_20_std - value: 21.196937851331807 - - type: nauc_ndcg_at_3_diff1 - value: 15.367489533734027 - - type: nauc_ndcg_at_3_max - value: 26.76486390741532 - - type: nauc_ndcg_at_3_std - value: 12.606077508789923 - - type: nauc_ndcg_at_5_diff1 - value: 13.831157482390935 - - type: nauc_ndcg_at_5_max - value: 28.070226983968904 - - type: nauc_ndcg_at_5_std - value: 15.236787943125435 - - type: nauc_precision_at_1000_diff1 - value: 0.016122957101357048 - - type: nauc_precision_at_1000_max - value: 24.380929903557334 - - type: nauc_precision_at_1000_std - value: 34.54045112720052 - - type: nauc_precision_at_100_diff1 - value: 7.255224788507301 - - type: nauc_precision_at_100_max - value: 27.98453788447542 - - type: nauc_precision_at_100_std - value: 35.38999555441665 - - type: nauc_precision_at_10_diff1 - value: 9.69185099834181 - - type: nauc_precision_at_10_max - value: 32.532315522580454 - - type: nauc_precision_at_10_std - value: 21.48948348473612 - - type: nauc_precision_at_1_diff1 - value: 19.43518811038007 - - type: nauc_precision_at_1_max - value: 17.747482933395602 - - type: nauc_precision_at_1_std - value: 8.410779775558684 - - type: nauc_precision_at_20_diff1 - value: 6.964076536695672 - - type: nauc_precision_at_20_max - value: 29.30087236410044 - - type: nauc_precision_at_20_std - value: 26.413625895571986 - - type: nauc_precision_at_3_diff1 - value: 14.145134359925155 - - type: nauc_precision_at_3_max - value: 29.915650960808303 - - type: nauc_precision_at_3_std - value: 14.095370019867797 - - type: nauc_precision_at_5_diff1 - value: 11.043933558522692 - - type: nauc_precision_at_5_max - value: 30.93016505807111 - - type: nauc_precision_at_5_std - value: 17.749256196062603 - - type: nauc_recall_at_1000_diff1 - value: -0.7776817772090345 - - type: nauc_recall_at_1000_max - value: 23.094717340324518 - - type: nauc_recall_at_1000_std - value: 37.189908681396425 - - type: nauc_recall_at_100_diff1 - value: 6.887748742013364 - - type: nauc_recall_at_100_max - value: 27.00798435230277 - - type: nauc_recall_at_100_std - value: 35.908147807345344 - - type: nauc_recall_at_10_diff1 - value: 9.605632017480751 - - type: nauc_recall_at_10_max - value: 31.845202901168655 - - type: nauc_recall_at_10_std - value: 21.497414586634683 - - type: nauc_recall_at_1_diff1 - value: 19.353069488987916 - - type: nauc_recall_at_1_max - value: 17.093914951159693 - - type: nauc_recall_at_1_std - value: 8.19886078055046 - - type: nauc_recall_at_20_diff1 - value: 6.927503731844782 - - type: nauc_recall_at_20_max - value: 28.611698183338202 - - type: nauc_recall_at_20_std - value: 26.69018660149911 - - type: nauc_recall_at_3_diff1 - value: 14.043724087062268 - - type: nauc_recall_at_3_max - value: 29.269835821380465 - - type: nauc_recall_at_3_std - value: 14.104419605998094 - - type: nauc_recall_at_5_diff1 - value: 11.017319452873336 - - type: nauc_recall_at_5_max - value: 30.295720628306228 - - type: nauc_recall_at_5_std - value: 17.758048545573825 - - type: ndcg_at_1 - value: 28.999999999999996 - - type: ndcg_at_10 - value: 25.041999999999998 - - type: ndcg_at_100 - value: 35.045 - - type: ndcg_at_1000 - value: 40.803 - - type: ndcg_at_20 - value: 28.584 - - type: ndcg_at_3 - value: 23.249 - - type: ndcg_at_5 - value: 20.533 - - type: precision_at_1 - value: 28.999999999999996 - - type: precision_at_10 - value: 13.120000000000001 - - type: precision_at_100 - value: 2.7470000000000003 - - type: precision_at_1000 - value: 0.41200000000000003 - - type: precision_at_20 - value: 8.584999999999999 - - type: precision_at_3 - value: 21.633 - - type: precision_at_5 - value: 18.099999999999998 - - type: recall_at_1 - value: 5.893000000000001 - - type: recall_at_10 - value: 26.567 - - type: recall_at_100 - value: 55.800000000000004 - - type: recall_at_1000 - value: 83.608 - - type: recall_at_20 - value: 34.86 - - type: recall_at_3 - value: 13.153 - - type: recall_at_5 - value: 18.323 - task: - type: Retrieval - - dataset: - config: default - name: MTEB SICK-R - revision: 20a6d6f312dd54037fe07a32d58e5e168867909d - split: test - type: mteb/sickr-sts - metrics: - - type: cosine_pearson - value: 86.57284584320382 - - type: cosine_spearman - value: 82.20531642680812 - - type: euclidean_pearson - value: 83.94261758556554 - - type: euclidean_spearman - value: 82.20721497738559 - - type: main_score - value: 82.20531642680812 - - type: manhattan_pearson - value: 84.15902154703083 - - type: manhattan_spearman - value: 82.19506027155957 - - type: pearson - value: 86.57284584320382 - - type: spearman - value: 82.20531642680812 - task: - type: STS - - dataset: - config: default - name: MTEB STS12 - revision: a0d554a64d88156834ff5ae9920b964011b16384 - split: test - type: mteb/sts12-sts - metrics: - - type: cosine_pearson - value: 86.28047602146931 - - type: cosine_spearman - value: 79.51504881448884 - - type: euclidean_pearson - value: 83.10545189967856 - - type: euclidean_spearman - value: 79.50586960492797 - - type: main_score - value: 79.51504881448884 - - type: manhattan_pearson - value: 83.44244457500889 - - type: manhattan_spearman - value: 79.730303339846 - - type: pearson - value: 86.28047602146931 - - type: spearman - value: 79.51504881448884 - task: - type: STS - - dataset: - config: default - name: MTEB STS13 - revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca - split: test - type: mteb/sts13-sts - metrics: - - type: cosine_pearson - value: 88.74723553048702 - - type: cosine_spearman - value: 89.18936052329725 - - type: euclidean_pearson - value: 88.90400878928668 - - type: euclidean_spearman - value: 89.19174821431281 - - type: main_score - value: 89.18936052329725 - - type: manhattan_pearson - value: 88.81504628424054 - - type: manhattan_spearman - value: 89.18063294142597 - - type: pearson - value: 88.74723553048702 - - type: spearman - value: 89.18936052329725 - task: - type: STS - - dataset: - config: default - name: MTEB STS14 - revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 - split: test - type: mteb/sts14-sts - metrics: - - type: cosine_pearson - value: 86.45403437836023 - - type: cosine_spearman - value: 85.14654611519086 - - type: euclidean_pearson - value: 85.87509624462743 - - type: euclidean_spearman - value: 85.1391108856681 - - type: main_score - value: 85.14654611519086 - - type: manhattan_pearson - value: 85.96635794953866 - - type: manhattan_spearman - value: 85.3271371527667 - - type: pearson - value: 86.45403437836023 - - type: spearman - value: 85.14654611519086 - task: - type: STS - - dataset: - config: default - name: MTEB STS15 - revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 - split: test - type: mteb/sts15-sts - metrics: - - type: cosine_pearson - value: 87.84742260009705 - - type: cosine_spearman - value: 89.10215217191254 - - type: euclidean_pearson - value: 88.97393286325477 - - type: euclidean_spearman - value: 89.1014105509662 - - type: main_score - value: 89.10215217191254 - - type: manhattan_pearson - value: 89.31698781090151 - - type: manhattan_spearman - value: 89.53000001764433 - - type: pearson - value: 87.84742260009705 - - type: spearman - value: 89.10215217191254 - task: - type: STS - - dataset: - config: default - name: MTEB STS16 - revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 - split: test - type: mteb/sts16-sts - metrics: - - type: cosine_pearson - value: 85.22397535461835 - - type: cosine_spearman - value: 87.14066355879785 - - type: euclidean_pearson - value: 86.31393364087295 - - type: euclidean_spearman - value: 87.14018892702765 - - type: main_score - value: 87.14066355879785 - - type: manhattan_pearson - value: 86.36366855248434 - - type: manhattan_spearman - value: 87.20858630423012 - - type: pearson - value: 85.22397535461835 - - type: spearman - value: 87.14066355879785 - task: - type: STS - - dataset: - config: en-en - name: MTEB STS17 (en-en) - revision: faeb762787bd10488a50c8b5be4a3b82e411949c - split: test - type: mteb/sts17-crosslingual-sts - metrics: - - type: cosine_pearson - value: 90.66131612061355 - - type: cosine_spearman - value: 90.97082650129164 - - type: euclidean_pearson - value: 90.98181906744969 - - type: euclidean_spearman - value: 90.99008476850047 - - type: main_score - value: 90.97082650129164 - - type: manhattan_pearson - value: 90.75245040709021 - - type: manhattan_spearman - value: 90.6199877691265 - - type: pearson - value: 90.66131612061355 - - type: spearman - value: 90.97082650129164 - task: - type: STS - - dataset: - config: en - name: MTEB STS22 (en) - revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 - split: test - type: mteb/sts22-crosslingual-sts - metrics: - - type: cosine_pearson - value: 67.270656447085 - - type: cosine_spearman - value: 67.82870469746828 - - type: euclidean_pearson - value: 69.03857775285664 - - type: euclidean_spearman - value: 67.74455108773341 - - type: main_score - value: 67.82870469746828 - - type: manhattan_pearson - value: 69.25304172245812 - - type: manhattan_spearman - value: 68.00987097916055 - - type: pearson - value: 67.270656447085 - - type: spearman - value: 67.82870469746828 - task: - type: STS - - dataset: - config: default - name: MTEB STSBenchmark - revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 - split: test - type: mteb/stsbenchmark-sts - metrics: - - type: cosine_pearson - value: 87.17245205384889 - - type: cosine_spearman - value: 87.7360146030987 - - type: euclidean_pearson - value: 87.48919412794656 - - type: euclidean_spearman - value: 87.7312047878383 - - type: main_score - value: 87.7360146030987 - - type: manhattan_pearson - value: 87.61476224354806 - - type: manhattan_spearman - value: 87.95220889254693 - - type: pearson - value: 87.17245205384889 - - type: spearman - value: 87.7360146030987 - task: - type: STS - - dataset: - config: default - name: MTEB SciDocsRR - revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab - split: test - type: mteb/scidocs-reranking - metrics: - - type: main_score - value: 88.43547871921146 - - type: map - value: 88.43547871921146 - - type: mrr - value: 96.5564473652709 - - type: nAUC_map_diff1 - value: -13.66029392579231 - - type: nAUC_map_max - value: 50.325613574053506 - - type: nAUC_map_std - value: 60.02986231275796 - - type: nAUC_mrr_diff1 - value: 23.83821476411125 - - type: nAUC_mrr_max - value: 86.72643311769906 - - type: nAUC_mrr_std - value: 72.12741063469213 - task: - type: Reranking - - dataset: - config: default - name: MTEB SciFact - revision: 0228b52cf27578f30900b9e5271d331663a030d7 - split: test - type: mteb/scifact - metrics: - - type: main_score - value: 78.233 - - type: map_at_1 - value: 61.49400000000001 - - type: map_at_10 - value: 73.30600000000001 - - type: map_at_100 - value: 73.719 - - type: map_at_1000 - value: 73.724 - - type: map_at_20 - value: 73.611 - - type: map_at_3 - value: 70.626 - - type: map_at_5 - value: 72.417 - - type: mrr_at_1 - value: 64.66666666666666 - - type: mrr_at_10 - value: 74.30357142857143 - - type: mrr_at_100 - value: 74.56950898079988 - - type: mrr_at_1000 - value: 74.57295833098681 - - type: mrr_at_20 - value: 74.46165223665226 - - type: mrr_at_3 - value: 72.3888888888889 - - type: mrr_at_5 - value: 73.60555555555557 - - type: nauc_map_at_1000_diff1 - value: 76.51524604780636 - - type: nauc_map_at_1000_max - value: 53.48521938401881 - - type: nauc_map_at_1000_std - value: -7.347799382158861 - - type: nauc_map_at_100_diff1 - value: 76.5122888096236 - - type: nauc_map_at_100_max - value: 53.49221847471618 - - type: nauc_map_at_100_std - value: -7.329683735681086 - - type: nauc_map_at_10_diff1 - value: 76.30928630674504 - - type: nauc_map_at_10_max - value: 53.00102977185941 - - type: nauc_map_at_10_std - value: -7.7467740085108705 - - type: nauc_map_at_1_diff1 - value: 79.54189281784247 - - type: nauc_map_at_1_max - value: 46.630071622109526 - - type: nauc_map_at_1_std - value: -14.395943134644112 - - type: nauc_map_at_20_diff1 - value: 76.41604361947962 - - type: nauc_map_at_20_max - value: 53.578883876146875 - - type: nauc_map_at_20_std - value: -7.403103451288041 - - type: nauc_map_at_3_diff1 - value: 76.25911617571941 - - type: nauc_map_at_3_max - value: 49.140287380513605 - - type: nauc_map_at_3_std - value: -11.35992449218983 - - type: nauc_map_at_5_diff1 - value: 76.35122077770336 - - type: nauc_map_at_5_max - value: 52.1744367901208 - - type: nauc_map_at_5_std - value: -7.85753955055384 - - type: nauc_mrr_at_1000_diff1 - value: 76.97223309515867 - - type: nauc_mrr_at_1000_max - value: 57.263787498613326 - - type: nauc_mrr_at_1000_std - value: -4.884090708840035 - - type: nauc_mrr_at_100_diff1 - value: 76.97312970894603 - - type: nauc_mrr_at_100_max - value: 57.26850730446478 - - type: nauc_mrr_at_100_std - value: -4.875200894216617 - - type: nauc_mrr_at_10_diff1 - value: 76.65927674223613 - - type: nauc_mrr_at_10_max - value: 57.30979763941454 - - type: nauc_mrr_at_10_std - value: -4.863331094022142 - - type: nauc_mrr_at_1_diff1 - value: 80.0454932568644 - - type: nauc_mrr_at_1_max - value: 56.76038421319305 - - type: nauc_mrr_at_1_std - value: -4.101939392632653 - - type: nauc_mrr_at_20_diff1 - value: 76.87237970440503 - - type: nauc_mrr_at_20_max - value: 57.33843605225869 - - type: nauc_mrr_at_20_std - value: -4.96248984417978 - - type: nauc_mrr_at_3_diff1 - value: 76.74130186666727 - - type: nauc_mrr_at_3_max - value: 56.19313244846155 - - type: nauc_mrr_at_3_std - value: -5.684365934009136 - - type: nauc_mrr_at_5_diff1 - value: 76.66406918799962 - - type: nauc_mrr_at_5_max - value: 57.56110093228628 - - type: nauc_mrr_at_5_std - value: -3.7464413085588073 - - type: nauc_ndcg_at_1000_diff1 - value: 76.19194173971773 - - type: nauc_ndcg_at_1000_max - value: 55.57464600170693 - - type: nauc_ndcg_at_1000_std - value: -6.0761689532372625 - - type: nauc_ndcg_at_100_diff1 - value: 76.14631273843654 - - type: nauc_ndcg_at_100_max - value: 55.72246565373382 - - type: nauc_ndcg_at_100_std - value: -5.595160698860595 - - type: nauc_ndcg_at_10_diff1 - value: 75.0108223611192 - - type: nauc_ndcg_at_10_max - value: 55.27894212877493 - - type: nauc_ndcg_at_10_std - value: -6.968331740214591 - - type: nauc_ndcg_at_1_diff1 - value: 80.0454932568644 - - type: nauc_ndcg_at_1_max - value: 56.76038421319305 - - type: nauc_ndcg_at_1_std - value: -4.101939392632653 - - type: nauc_ndcg_at_20_diff1 - value: 75.54887755702472 - - type: nauc_ndcg_at_20_max - value: 56.406879417251496 - - type: nauc_ndcg_at_20_std - value: -6.495231061329629 - - type: nauc_ndcg_at_3_diff1 - value: 75.03620356688509 - - type: nauc_ndcg_at_3_max - value: 52.147381077773424 - - type: nauc_ndcg_at_3_std - value: -8.448005688956199 - - type: nauc_ndcg_at_5_diff1 - value: 75.1195898074229 - - type: nauc_ndcg_at_5_max - value: 54.2321033861173 - - type: nauc_ndcg_at_5_std - value: -5.882690780895338 - - type: nauc_precision_at_1000_diff1 - value: -28.081979732100532 - - type: nauc_precision_at_1000_max - value: 35.055348014832916 - - type: nauc_precision_at_1000_std - value: 59.61280468927384 - - type: nauc_precision_at_100_diff1 - value: -25.112740730587458 - - type: nauc_precision_at_100_max - value: 38.26331300116496 - - type: nauc_precision_at_100_std - value: 62.46316222328831 - - type: nauc_precision_at_10_diff1 - value: -2.6766206473658833 - - type: nauc_precision_at_10_max - value: 45.95321867204845 - - type: nauc_precision_at_10_std - value: 45.07212468670564 - - type: nauc_precision_at_1_diff1 - value: 80.0454932568644 - - type: nauc_precision_at_1_max - value: 56.76038421319305 - - type: nauc_precision_at_1_std - value: -4.101939392632653 - - type: nauc_precision_at_20_diff1 - value: -10.698911116738385 - - type: nauc_precision_at_20_max - value: 43.467275950182994 - - type: nauc_precision_at_20_std - value: 48.00467321991766 - - type: nauc_precision_at_3_diff1 - value: 33.6344708541193 - - type: nauc_precision_at_3_max - value: 49.309242331670504 - - type: nauc_precision_at_3_std - value: 21.02940391379915 - - type: nauc_precision_at_5_diff1 - value: 13.560415600596318 - - type: nauc_precision_at_5_max - value: 48.918726500100085 - - type: nauc_precision_at_5_std - value: 39.940930429172184 - - type: nauc_recall_at_1000_diff1 - value: .nan - - type: nauc_recall_at_1000_max - value: .nan - - type: nauc_recall_at_1000_std - value: .nan - - type: nauc_recall_at_100_diff1 - value: 70.82166199813196 - - type: nauc_recall_at_100_max - value: 76.6106442577042 - - type: nauc_recall_at_100_std - value: 66.47992530345513 - - type: nauc_recall_at_10_diff1 - value: 62.68908885556092 - - type: nauc_recall_at_10_max - value: 58.14262437741839 - - type: nauc_recall_at_10_std - value: -12.946717875063369 - - type: nauc_recall_at_1_diff1 - value: 79.54189281784247 - - type: nauc_recall_at_1_max - value: 46.630071622109526 - - type: nauc_recall_at_1_std - value: -14.395943134644112 - - type: nauc_recall_at_20_diff1 - value: 65.79470497876567 - - type: nauc_recall_at_20_max - value: 71.68308183488456 - - type: nauc_recall_at_20_std - value: -12.556850697268453 - - type: nauc_recall_at_3_diff1 - value: 68.3240211318129 - - type: nauc_recall_at_3_max - value: 45.05998217275036 - - type: nauc_recall_at_3_std - value: -14.23179772593869 - - type: nauc_recall_at_5_diff1 - value: 67.53366869904056 - - type: nauc_recall_at_5_max - value: 53.57935627081027 - - type: nauc_recall_at_5_std - value: -3.3271112904853393 - - type: ndcg_at_1 - value: 64.667 - - type: ndcg_at_10 - value: 78.233 - - type: ndcg_at_100 - value: 79.806 - - type: ndcg_at_1000 - value: 79.92099999999999 - - type: ndcg_at_20 - value: 79.006 - - type: ndcg_at_3 - value: 74.018 - - type: ndcg_at_5 - value: 76.334 - - type: precision_at_1 - value: 64.667 - - type: precision_at_10 - value: 10.4 - - type: precision_at_100 - value: 1.1199999999999999 - - type: precision_at_1000 - value: 0.11299999999999999 - - type: precision_at_20 - value: 5.383 - - type: precision_at_3 - value: 29.444 - - type: precision_at_5 - value: 19.467000000000002 - - type: recall_at_1 - value: 61.49400000000001 - - type: recall_at_10 - value: 92.156 - - type: recall_at_100 - value: 99.167 - - type: recall_at_1000 - value: 100.0 - - type: recall_at_20 - value: 94.833 - - type: recall_at_3 - value: 80.833 - - type: recall_at_5 - value: 86.6 - task: - type: Retrieval - - dataset: - config: default - name: MTEB SprintDuplicateQuestions - revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 - split: test - type: mteb/sprintduplicatequestions-pairclassification - metrics: - - type: cosine_accuracy - value: 99.8039603960396 - - type: cosine_accuracy_threshold - value: 84.54211950302124 - - type: cosine_ap - value: 95.59056372734358 - - type: cosine_f1 - value: 90.1394422310757 - - type: cosine_f1_threshold - value: 84.54211950302124 - - type: cosine_precision - value: 89.78174603174604 - - type: cosine_recall - value: 90.5 - - type: dot_accuracy - value: 99.80594059405941 - - type: dot_accuracy_threshold - value: 85.57180166244507 - - type: dot_ap - value: 95.53453431914399 - - type: dot_f1 - value: 90.10442565887618 - - type: dot_f1_threshold - value: 84.59715843200684 - - type: dot_precision - value: 89.61424332344214 - - type: dot_recall - value: 90.60000000000001 - - type: euclidean_accuracy - value: 99.8039603960396 - - type: euclidean_accuracy_threshold - value: 53.253382444381714 - - type: euclidean_ap - value: 95.5850992402159 - - type: euclidean_f1 - value: 90.09457441513192 - - type: euclidean_f1_threshold - value: 55.725520849227905 - - type: euclidean_precision - value: 89.69276511397423 - - type: euclidean_recall - value: 90.5 - - type: main_score - value: 95.7485189884476 - - type: manhattan_accuracy - value: 99.81485148514851 - - type: manhattan_accuracy_threshold - value: 3491.29638671875 - - type: manhattan_ap - value: 95.7485189884476 - - type: manhattan_f1 - value: 90.464048954615 - - type: manhattan_f1_threshold - value: 3491.29638671875 - - type: manhattan_precision - value: 92.2996878251821 - - type: manhattan_recall - value: 88.7 - - type: max_ap - value: 95.7485189884476 - - type: max_f1 - value: 90.464048954615 - - type: max_precision - value: 92.2996878251821 - - type: max_recall - value: 90.60000000000001 - - type: similarity_accuracy - value: 99.8039603960396 - - type: similarity_accuracy_threshold - value: 84.54211950302124 - - type: similarity_ap - value: 95.59056372734358 - - type: similarity_f1 - value: 90.1394422310757 - - type: similarity_f1_threshold - value: 84.54211950302124 - - type: similarity_precision - value: 89.78174603174604 - - type: similarity_recall - value: 90.5 - task: - type: PairClassification - - dataset: - config: default - name: MTEB StackExchangeClustering - revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 - split: test - type: mteb/stackexchange-clustering - metrics: - - type: main_score - value: 78.49205191950675 - - type: v_measure - value: 78.49205191950675 - - type: v_measure_std - value: 2.84869550699959 - task: - type: Clustering - - dataset: - config: default - name: MTEB StackExchangeClusteringP2P - revision: 815ca46b2622cec33ccafc3735d572c266efdb44 - split: test - type: mteb/stackexchange-clustering-p2p - metrics: - - type: main_score - value: 48.90421736513028 - - type: v_measure - value: 48.90421736513028 - - type: v_measure_std - value: 1.6875865714471023 - task: - type: Clustering - - dataset: - config: default - name: MTEB StackOverflowDupQuestions - revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 - split: test - type: mteb/stackoverflowdupquestions-reranking - metrics: - - type: main_score - value: 52.9874730481696 - - type: map - value: 52.9874730481696 - - type: mrr - value: 53.85867604617604 - - type: nAUC_map_diff1 - value: 39.633429293407616 - - type: nAUC_map_max - value: 10.236807988858546 - - type: nAUC_map_std - value: 10.276522217929674 - - type: nAUC_mrr_diff1 - value: 40.0543079218377 - - type: nAUC_mrr_max - value: 10.96209807382042 - - type: nAUC_mrr_std - value: 10.524400196109918 - task: - type: Reranking - - dataset: - config: default - name: MTEB SummEval - revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c - split: test - type: mteb/summeval - metrics: - - type: cosine_pearson - value: 30.727801109114232 - - type: cosine_spearman - value: 31.66058223980157 - - type: dot_pearson - value: 30.78818248622866 - - type: dot_spearman - value: 31.525158776890265 - - type: main_score - value: 31.66058223980157 - - type: pearson - value: 30.727801109114232 - - type: spearman - value: 31.66058223980157 - task: - type: Summarization - - dataset: - config: default - name: MTEB TRECCOVID - revision: bb9466bac8153a0349341eb1b22e06409e78ef4e - split: test - type: mteb/trec-covid - metrics: - - type: main_score - value: 85.206 - - type: map_at_1 - value: 0.246 - - type: map_at_10 - value: 2.1950000000000003 - - type: map_at_100 - value: 14.179 - - type: map_at_1000 - value: 35.037 - - type: map_at_20 - value: 4.143 - - type: map_at_3 - value: 0.7100000000000001 - - type: map_at_5 - value: 1.135 - - type: mrr_at_1 - value: 94.0 - - type: mrr_at_10 - value: 96.66666666666666 - - type: mrr_at_100 - value: 96.66666666666666 - - type: mrr_at_1000 - value: 96.66666666666666 - - type: mrr_at_20 - value: 96.66666666666666 - - type: mrr_at_3 - value: 96.66666666666666 - - type: mrr_at_5 - value: 96.66666666666666 - - type: nauc_map_at_1000_diff1 - value: -4.6264497624527525 - - type: nauc_map_at_1000_max - value: 44.594457564749355 - - type: nauc_map_at_1000_std - value: 73.17642341400133 - - type: nauc_map_at_100_diff1 - value: 23.451335157405726 - - type: nauc_map_at_100_max - value: 25.426398857299525 - - type: nauc_map_at_100_std - value: 64.07416694472633 - - type: nauc_map_at_10_diff1 - value: 46.57568738568346 - - type: nauc_map_at_10_max - value: 9.693233249079238 - - type: nauc_map_at_10_std - value: 28.549530265164357 - - type: nauc_map_at_1_diff1 - value: 53.48238396620123 - - type: nauc_map_at_1_max - value: 0.33476619393733076 - - type: nauc_map_at_1_std - value: 8.906362219128463 - - type: nauc_map_at_20_diff1 - value: 39.40719602207749 - - type: nauc_map_at_20_max - value: 9.635915072074045 - - type: nauc_map_at_20_std - value: 35.15634791346394 - - type: nauc_map_at_3_diff1 - value: 53.11784737840137 - - type: nauc_map_at_3_max - value: 3.059682761072153 - - type: nauc_map_at_3_std - value: 21.310633086556617 - - type: nauc_map_at_5_diff1 - value: 49.91570701185436 - - type: nauc_map_at_5_max - value: 8.045082896244576 - - type: nauc_map_at_5_std - value: 20.597686235051647 - - type: nauc_mrr_at_1000_diff1 - value: 41.98412698412726 - - type: nauc_mrr_at_1000_max - value: 78.24463118580779 - - type: nauc_mrr_at_1000_std - value: 0.30812324930028195 - - type: nauc_mrr_at_100_diff1 - value: 41.98412698412726 - - type: nauc_mrr_at_100_max - value: 78.24463118580779 - - type: nauc_mrr_at_100_std - value: 0.30812324930028195 - - type: nauc_mrr_at_10_diff1 - value: 41.98412698412726 - - type: nauc_mrr_at_10_max - value: 78.24463118580779 - - type: nauc_mrr_at_10_std - value: 0.30812324930028195 - - type: nauc_mrr_at_1_diff1 - value: 38.62433862433873 - - type: nauc_mrr_at_1_max - value: 80.78120136943666 - - type: nauc_mrr_at_1_std - value: -10.768751945222197 - - type: nauc_mrr_at_20_diff1 - value: 41.98412698412726 - - type: nauc_mrr_at_20_max - value: 78.24463118580779 - - type: nauc_mrr_at_20_std - value: 0.30812324930028195 - - type: nauc_mrr_at_3_diff1 - value: 41.98412698412726 - - type: nauc_mrr_at_3_max - value: 78.24463118580779 - - type: nauc_mrr_at_3_std - value: 0.30812324930028195 - - type: nauc_mrr_at_5_diff1 - value: 41.98412698412726 - - type: nauc_mrr_at_5_max - value: 78.24463118580779 - - type: nauc_mrr_at_5_std - value: 0.30812324930028195 - - type: nauc_ndcg_at_1000_diff1 - value: 0.5174948602880207 - - type: nauc_ndcg_at_1000_max - value: 48.60686602077053 - - type: nauc_ndcg_at_1000_std - value: 75.72456343175277 - - type: nauc_ndcg_at_100_diff1 - value: -20.747252137999254 - - type: nauc_ndcg_at_100_max - value: 49.985132618254994 - - type: nauc_ndcg_at_100_std - value: 61.096383293836574 - - type: nauc_ndcg_at_10_diff1 - value: 6.791377920463332 - - type: nauc_ndcg_at_10_max - value: 57.50019332833286 - - type: nauc_ndcg_at_10_std - value: 49.201028841219426 - - type: nauc_ndcg_at_1_diff1 - value: 54.92683440362145 - - type: nauc_ndcg_at_1_max - value: 83.8667228129276 - - type: nauc_ndcg_at_1_std - value: 1.6738604063586122 - - type: nauc_ndcg_at_20_diff1 - value: -5.1948699196314925 - - type: nauc_ndcg_at_20_max - value: 54.483087684806556 - - type: nauc_ndcg_at_20_std - value: 50.54823818118781 - - type: nauc_ndcg_at_3_diff1 - value: 26.267246500164372 - - type: nauc_ndcg_at_3_max - value: 63.0173212926611 - - type: nauc_ndcg_at_3_std - value: 41.025597406368256 - - type: nauc_ndcg_at_5_diff1 - value: 16.910185454343036 - - type: nauc_ndcg_at_5_max - value: 60.9328683868778 - - type: nauc_ndcg_at_5_std - value: 36.70169905857712 - - type: nauc_precision_at_1000_diff1 - value: -46.374447765983525 - - type: nauc_precision_at_1000_max - value: 35.36052337813863 - - type: nauc_precision_at_1000_std - value: 14.219220668161018 - - type: nauc_precision_at_100_diff1 - value: -29.7838083657744 - - type: nauc_precision_at_100_max - value: 43.93589400385112 - - type: nauc_precision_at_100_std - value: 55.425045718579945 - - type: nauc_precision_at_10_diff1 - value: -12.016613405227687 - - type: nauc_precision_at_10_max - value: 57.79924427743131 - - type: nauc_precision_at_10_std - value: 49.022036703550675 - - type: nauc_precision_at_1_diff1 - value: 38.62433862433873 - - type: nauc_precision_at_1_max - value: 80.78120136943666 - - type: nauc_precision_at_1_std - value: -10.768751945222197 - - type: nauc_precision_at_20_diff1 - value: -23.95633847880195 - - type: nauc_precision_at_20_max - value: 48.34715917258276 - - type: nauc_precision_at_20_std - value: 48.82198285255887 - - type: nauc_precision_at_3_diff1 - value: 6.871296905858807 - - type: nauc_precision_at_3_max - value: 70.54805793285054 - - type: nauc_precision_at_3_std - value: 44.65108624094803 - - type: nauc_precision_at_5_diff1 - value: -9.074932448759695 - - type: nauc_precision_at_5_max - value: 67.41284242437573 - - type: nauc_precision_at_5_std - value: 23.876891983919577 - - type: nauc_recall_at_1000_diff1 - value: 8.142288830293255 - - type: nauc_recall_at_1000_max - value: 38.85182826835104 - - type: nauc_recall_at_1000_std - value: 68.60783819217335 - - type: nauc_recall_at_100_diff1 - value: 34.262914076287466 - - type: nauc_recall_at_100_max - value: 12.87009658528838 - - type: nauc_recall_at_100_std - value: 56.21330603762995 - - type: nauc_recall_at_10_diff1 - value: 49.33830945338758 - - type: nauc_recall_at_10_max - value: 0.3539875530671406 - - type: nauc_recall_at_10_std - value: 26.85864465557644 - - type: nauc_recall_at_1_diff1 - value: 53.48238396620123 - - type: nauc_recall_at_1_max - value: 0.33476619393733076 - - type: nauc_recall_at_1_std - value: 8.906362219128463 - - type: nauc_recall_at_20_diff1 - value: 44.21928181266254 - - type: nauc_recall_at_20_max - value: -0.9198356057088594 - - type: nauc_recall_at_20_std - value: 31.484376992896784 - - type: nauc_recall_at_3_diff1 - value: 53.038093080990876 - - type: nauc_recall_at_3_max - value: -1.4170895916973003 - - type: nauc_recall_at_3_std - value: 21.890202855574497 - - type: nauc_recall_at_5_diff1 - value: 49.39742214825278 - - type: nauc_recall_at_5_max - value: 2.8412267611894517 - - type: nauc_recall_at_5_std - value: 18.01598921859512 - - type: ndcg_at_1 - value: 91.0 - - type: ndcg_at_10 - value: 85.206 - - type: ndcg_at_100 - value: 67.29 - - type: ndcg_at_1000 - value: 60.584 - - type: ndcg_at_20 - value: 82.321 - - type: ndcg_at_3 - value: 88.642 - - type: ndcg_at_5 - value: 87.063 - - type: precision_at_1 - value: 94.0 - - type: precision_at_10 - value: 89.8 - - type: precision_at_100 - value: 69.78 - - type: precision_at_1000 - value: 26.738 - - type: precision_at_20 - value: 87.2 - - type: precision_at_3 - value: 92.0 - - type: precision_at_5 - value: 90.8 - - type: recall_at_1 - value: 0.246 - - type: recall_at_10 - value: 2.344 - - type: recall_at_100 - value: 16.962 - - type: recall_at_1000 - value: 57.325 - - type: recall_at_20 - value: 4.517 - - type: recall_at_3 - value: 0.731 - - type: recall_at_5 - value: 1.1780000000000002 - task: - type: Retrieval - - dataset: - config: default - name: MTEB Touche2020 - revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f - split: test - type: mteb/touche2020 - metrics: - - type: main_score - value: 31.455 - - type: map_at_1 - value: 2.9739999999999998 - - type: map_at_10 - value: 12.183 - - type: map_at_100 - value: 18.772 - - type: map_at_1000 - value: 20.415 - - type: map_at_20 - value: 14.451 - - type: map_at_3 - value: 6.507000000000001 - - type: map_at_5 - value: 8.66 - - type: mrr_at_1 - value: 40.816326530612244 - - type: mrr_at_10 - value: 57.70975056689341 - - type: mrr_at_100 - value: 58.18379126542391 - - type: mrr_at_1000 - value: 58.18379126542391 - - type: mrr_at_20 - value: 57.85552316164561 - - type: mrr_at_3 - value: 54.08163265306123 - - type: mrr_at_5 - value: 56.42857142857143 - - type: nauc_map_at_1000_diff1 - value: 3.1567471051481437 - - type: nauc_map_at_1000_max - value: -1.5882060729791523 - - type: nauc_map_at_1000_std - value: 18.69622198722074 - - type: nauc_map_at_100_diff1 - value: 3.3449677678147536 - - type: nauc_map_at_100_max - value: -2.8928606866168405 - - type: nauc_map_at_100_std - value: 15.789984947653412 - - type: nauc_map_at_10_diff1 - value: 2.9696743570444264 - - type: nauc_map_at_10_max - value: -9.096749212011876 - - type: nauc_map_at_10_std - value: -5.38545817258353 - - type: nauc_map_at_1_diff1 - value: 20.680780404542546 - - type: nauc_map_at_1_max - value: -7.04722927447817 - - type: nauc_map_at_1_std - value: -7.062494733973898 - - type: nauc_map_at_20_diff1 - value: 4.070437790119271 - - type: nauc_map_at_20_max - value: -4.84491434686032 - - type: nauc_map_at_20_std - value: 0.5846341109021014 - - type: nauc_map_at_3_diff1 - value: 11.9634978045925 - - type: nauc_map_at_3_max - value: -8.27834591046608 - - type: nauc_map_at_3_std - value: -8.687615453381065 - - type: nauc_map_at_5_diff1 - value: 0.9195191526009436 - - type: nauc_map_at_5_max - value: -1.673813362719489 - - type: nauc_map_at_5_std - value: -6.67549753473631 - - type: nauc_mrr_at_1000_diff1 - value: 19.877993208719573 - - type: nauc_mrr_at_1000_max - value: -10.37776706406218 - - type: nauc_mrr_at_1000_std - value: 7.132169578056367 - - type: nauc_mrr_at_100_diff1 - value: 19.877993208719573 - - type: nauc_mrr_at_100_max - value: -10.37776706406218 - - type: nauc_mrr_at_100_std - value: 7.132169578056367 - - type: nauc_mrr_at_10_diff1 - value: 20.414285568401457 - - type: nauc_mrr_at_10_max - value: -9.677800295687861 - - type: nauc_mrr_at_10_std - value: 8.001103690180859 - - type: nauc_mrr_at_1_diff1 - value: 22.393284073955723 - - type: nauc_mrr_at_1_max - value: -5.889370191243167 - - type: nauc_mrr_at_1_std - value: -1.5183536173658247 - - type: nauc_mrr_at_20_diff1 - value: 20.455564720604055 - - type: nauc_mrr_at_20_max - value: -10.230642830103074 - - type: nauc_mrr_at_20_std - value: 7.863582453266621 - - type: nauc_mrr_at_3_diff1 - value: 17.554895390732618 - - type: nauc_mrr_at_3_max - value: -15.618463505555052 - - type: nauc_mrr_at_3_std - value: 5.913231577966864 - - type: nauc_mrr_at_5_diff1 - value: 18.393678507779914 - - type: nauc_mrr_at_5_max - value: -11.903593353147762 - - type: nauc_mrr_at_5_std - value: 7.580745996262831 - - type: nauc_ndcg_at_1000_diff1 - value: 13.746937095530473 - - type: nauc_ndcg_at_1000_max - value: -0.9319249687895838 - - type: nauc_ndcg_at_1000_std - value: 38.56328031451904 - - type: nauc_ndcg_at_100_diff1 - value: 13.854865944415895 - - type: nauc_ndcg_at_100_max - value: -7.142142012591404 - - type: nauc_ndcg_at_100_std - value: 35.61341954818848 - - type: nauc_ndcg_at_10_diff1 - value: 9.010144273248759 - - type: nauc_ndcg_at_10_max - value: -15.320014897424574 - - type: nauc_ndcg_at_10_std - value: 2.84883880489144 - - type: nauc_ndcg_at_1_diff1 - value: 20.939533945592967 - - type: nauc_ndcg_at_1_max - value: -6.387319972188946 - - type: nauc_ndcg_at_1_std - value: -0.5258673122126726 - - type: nauc_ndcg_at_20_diff1 - value: 14.660827309009496 - - type: nauc_ndcg_at_20_max - value: -13.476196120145994 - - type: nauc_ndcg_at_20_std - value: 8.22391881710838 - - type: nauc_ndcg_at_3_diff1 - value: 13.429985227235935 - - type: nauc_ndcg_at_3_max - value: -14.904544592570247 - - type: nauc_ndcg_at_3_std - value: 1.599779998183342 - - type: nauc_ndcg_at_5_diff1 - value: 8.085466231900622 - - type: nauc_ndcg_at_5_max - value: -9.09591969526831 - - type: nauc_ndcg_at_5_std - value: 3.5794092637248505 - - type: nauc_precision_at_1000_diff1 - value: -9.31941215946743 - - type: nauc_precision_at_1000_max - value: 31.52913520470716 - - type: nauc_precision_at_1000_std - value: 22.720784312185856 - - type: nauc_precision_at_100_diff1 - value: 8.958548406995279 - - type: nauc_precision_at_100_max - value: 15.100597910674104 - - type: nauc_precision_at_100_std - value: 71.04548238175113 - - type: nauc_precision_at_10_diff1 - value: 12.4698194690008 - - type: nauc_precision_at_10_max - value: -15.84870544871496 - - type: nauc_precision_at_10_std - value: 7.575297622501928 - - type: nauc_precision_at_1_diff1 - value: 22.393284073955723 - - type: nauc_precision_at_1_max - value: -5.889370191243167 - - type: nauc_precision_at_1_std - value: -1.5183536173658247 - - type: nauc_precision_at_20_diff1 - value: 15.393505718138758 - - type: nauc_precision_at_20_max - value: -3.70684298539384 - - type: nauc_precision_at_20_std - value: 29.426137824970304 - - type: nauc_precision_at_3_diff1 - value: 9.997768085465394 - - type: nauc_precision_at_3_max - value: -17.12224314347674 - - type: nauc_precision_at_3_std - value: -1.343018166772313 - - type: nauc_precision_at_5_diff1 - value: 3.8936997437913554 - - type: nauc_precision_at_5_max - value: -5.689104289687632 - - type: nauc_precision_at_5_std - value: 3.181098051304285 - - type: nauc_recall_at_1000_diff1 - value: 9.908303508158387 - - type: nauc_recall_at_1000_max - value: 6.174506592699848 - - type: nauc_recall_at_1000_std - value: 77.41931114780012 - - type: nauc_recall_at_100_diff1 - value: 10.286839241876192 - - type: nauc_recall_at_100_max - value: -6.6138697026666815 - - type: nauc_recall_at_100_std - value: 49.608313692633224 - - type: nauc_recall_at_10_diff1 - value: 2.215545846659851 - - type: nauc_recall_at_10_max - value: -17.83025802478445 - - type: nauc_recall_at_10_std - value: -3.3784768673705465 - - type: nauc_recall_at_1_diff1 - value: 20.680780404542546 - - type: nauc_recall_at_1_max - value: -7.04722927447817 - - type: nauc_recall_at_1_std - value: -7.062494733973898 - - type: nauc_recall_at_20_diff1 - value: 6.974410239251615 - - type: nauc_recall_at_20_max - value: -14.161147924731646 - - type: nauc_recall_at_20_std - value: 9.328412057721454 - - type: nauc_recall_at_3_diff1 - value: 7.904589805754212 - - type: nauc_recall_at_3_max - value: -12.1912388648593 - - type: nauc_recall_at_3_std - value: -9.221542013385555 - - type: nauc_recall_at_5_diff1 - value: -3.2604132752706914 - - type: nauc_recall_at_5_max - value: -6.886351441658915 - - type: nauc_recall_at_5_std - value: -7.014252851712789 - - type: ndcg_at_1 - value: 39.796 - - type: ndcg_at_10 - value: 31.455 - - type: ndcg_at_100 - value: 42.388999999999996 - - type: ndcg_at_1000 - value: 53.556000000000004 - - type: ndcg_at_20 - value: 30.808000000000003 - - type: ndcg_at_3 - value: 35.831 - - type: ndcg_at_5 - value: 32.845 - - type: precision_at_1 - value: 40.816 - - type: precision_at_10 - value: 27.143 - - type: precision_at_100 - value: 8.449 - - type: precision_at_1000 - value: 1.6179999999999999 - - type: precision_at_20 - value: 19.387999999999998 - - type: precision_at_3 - value: 35.374 - - type: precision_at_5 - value: 31.019999999999996 - - type: recall_at_1 - value: 2.9739999999999998 - - type: recall_at_10 - value: 19.39 - - type: recall_at_100 - value: 51.636 - - type: recall_at_1000 - value: 86.99900000000001 - - type: recall_at_20 - value: 26.478 - - type: recall_at_3 - value: 7.703 - - type: recall_at_5 - value: 11.42 - task: - type: Retrieval - - dataset: - config: default - name: MTEB ToxicConversationsClassification - revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de - split: test - type: mteb/toxic_conversations_50k - metrics: - - type: accuracy - value: 86.9384765625 - - type: ap - value: 31.737513704141552 - - type: ap_weighted - value: 31.737513704141552 - - type: f1 - value: 71.5490757306975 - - type: f1_weighted - value: 89.14632533489856 - - type: main_score - value: 86.9384765625 - task: - type: Classification - - dataset: - config: default - name: MTEB TweetSentimentExtractionClassification - revision: d604517c81ca91fe16a244d1248fc021f9ecee7a - split: test - type: mteb/tweet_sentiment_extraction - metrics: - - type: accuracy - value: 73.57668364459535 - - type: f1 - value: 73.90467103648074 - - type: f1_weighted - value: 73.42158415034704 - - type: main_score - value: 73.57668364459535 - task: - type: Classification - - dataset: - config: default - name: MTEB TwentyNewsgroupsClustering - revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 - split: test - type: mteb/twentynewsgroups-clustering - metrics: - - type: main_score - value: 58.574148097494685 - - type: v_measure - value: 58.574148097494685 - - type: v_measure_std - value: 0.9443161637490822 - task: - type: Clustering - - dataset: - config: default - name: MTEB TwitterSemEval2015 - revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 - split: test - type: mteb/twittersemeval2015-pairclassification - metrics: - - type: cosine_accuracy - value: 88.1385229778864 - - type: cosine_accuracy_threshold - value: 83.86307954788208 - - type: cosine_ap - value: 80.17965893449055 - - type: cosine_f1 - value: 73.0614300100705 - - type: cosine_f1_threshold - value: 80.7942807674408 - - type: cosine_precision - value: 69.8603755416466 - - type: cosine_recall - value: 76.56992084432717 - - type: dot_accuracy - value: 88.2100494724921 - - type: dot_accuracy_threshold - value: 83.84793996810913 - - type: dot_ap - value: 80.18603932881858 - - type: dot_f1 - value: 73.07643714466204 - - type: dot_f1_threshold - value: 80.87586164474487 - - type: dot_precision - value: 70.10909090909091 - - type: dot_recall - value: 76.3060686015831 - - type: euclidean_accuracy - value: 88.1385229778864 - - type: euclidean_accuracy_threshold - value: 56.77661895751953 - - type: euclidean_ap - value: 80.1784070881624 - - type: euclidean_f1 - value: 73.04830369529574 - - type: euclidean_f1_threshold - value: 61.91838979721069 - - type: euclidean_precision - value: 69.96859144720948 - - type: euclidean_recall - value: 76.41160949868075 - - type: main_score - value: 80.18603932881858 - - type: manhattan_accuracy - value: 88.0431543184121 - - type: manhattan_accuracy_threshold - value: 3755.6137084960938 - - type: manhattan_ap - value: 79.98270453664578 - - type: manhattan_f1 - value: 72.68242015061023 - - type: manhattan_f1_threshold - value: 3892.494583129883 - - type: manhattan_precision - value: 71.54907975460122 - - type: manhattan_recall - value: 73.85224274406332 - - type: max_ap - value: 80.18603932881858 - - type: max_f1 - value: 73.07643714466204 - - type: max_precision - value: 71.54907975460122 - - type: max_recall - value: 76.56992084432717 - - type: similarity_accuracy - value: 88.1385229778864 - - type: similarity_accuracy_threshold - value: 83.86307954788208 - - type: similarity_ap - value: 80.17965893449055 - - type: similarity_f1 - value: 73.0614300100705 - - type: similarity_f1_threshold - value: 80.7942807674408 - - type: similarity_precision - value: 69.8603755416466 - - type: similarity_recall - value: 76.56992084432717 - task: - type: PairClassification - - dataset: - config: default - name: MTEB TwitterURLCorpus - revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf - split: test - type: mteb/twitterurlcorpus-pairclassification - metrics: - - type: cosine_accuracy - value: 89.7892653393876 - - type: cosine_accuracy_threshold - value: 79.69566583633423 - - type: cosine_ap - value: 87.4579867302024 - - type: cosine_f1 - value: 79.91620843152658 - - type: cosine_f1_threshold - value: 78.53609323501587 - - type: cosine_precision - value: 77.7155329210622 - - type: cosine_recall - value: 82.24514936864799 - - type: dot_accuracy - value: 89.78732487289945 - - type: dot_accuracy_threshold - value: 80.05315661430359 - - type: dot_ap - value: 87.44916182456272 - - type: dot_f1 - value: 79.90419878751591 - - type: dot_f1_threshold - value: 78.57890725135803 - - type: dot_precision - value: 77.73409057812728 - - type: dot_recall - value: 82.19895287958116 - - type: euclidean_accuracy - value: 89.78538440641131 - - type: euclidean_accuracy_threshold - value: 62.29925751686096 - - type: euclidean_ap - value: 87.45904868911386 - - type: euclidean_f1 - value: 79.93127404474657 - - type: euclidean_f1_threshold - value: 65.61101078987122 - - type: euclidean_precision - value: 77.62060210373595 - - type: euclidean_recall - value: 82.38373883584848 - - type: main_score - value: 87.46554314325058 - - type: manhattan_accuracy - value: 89.76597974152986 - - type: manhattan_accuracy_threshold - value: 3988.5299682617188 - - type: manhattan_ap - value: 87.46554314325058 - - type: manhattan_f1 - value: 79.97181740645973 - - type: manhattan_f1_threshold - value: 4235.905838012695 - - type: manhattan_precision - value: 77.13713427283783 - - type: manhattan_recall - value: 83.02279026793964 - - type: max_ap - value: 87.46554314325058 - - type: max_f1 - value: 79.97181740645973 - - type: max_precision - value: 77.73409057812728 - - type: max_recall - value: 83.02279026793964 - - type: similarity_accuracy - value: 89.7892653393876 - - type: similarity_accuracy_threshold - value: 79.69566583633423 - - type: similarity_ap - value: 87.4579867302024 - - type: similarity_f1 - value: 79.91620843152658 - - type: similarity_f1_threshold - value: 78.53609323501587 - - type: similarity_precision - value: 77.7155329210622 - - type: similarity_recall - value: 82.24514936864799 - task: - type: PairClassification -tags: -- mteb -- sentence-transformers -- transformers -- sentence-similarity -license: mit ---- - - - -# Updates - -New open-source models and ToDoList will be listed on https://github.com/DunZhang/Stella/blob/main/news_and_todo.md. - -You can also find these models on my [homepage](https://huggingface.co/infgrad). - -# Introduction - -The models are trained based on `Alibaba-NLP/gte-large-en-v1.5` and `Alibaba-NLP/gte-Qwen2-1.5B-instruct`. Thanks for -their contributions! - -**We simplify usage of prompts, providing two prompts for most general tasks, one is for s2p, another one is for s2s.** - -Prompt of s2p task(e.g. retrieve task): - -```text -Instruct: Given a web search query, retrieve relevant passages that answer the query.\nQuery: {query} -``` - -Prompt of s2s task(e.g. semantic textual similarity task): - -```text -Instruct: Retrieve semantically similar text.\nQuery: {query} -``` - -The models are finally trained by [MRL]((https://arxiv.org/abs/2205.13147)), so they have multiple dimensions: 512, 768, -1024, 2048, 4096, 6144 and 8192. - -The higher the dimension, the better the performance. -**Generally speaking, 1024d is good enough.** The MTEB score of 1024d is only 0.001 lower than 8192d. - -# Model directory structure - -The model directory structure is very simple, it is a standard SentenceTransformer directory **with a series -of `2_Dense_{dims}` -folders**, where `dims` represents the final vector dimension. - -For example, the `2_Dense_256` folder stores Linear weights that convert vector dimensions to 256 dimensions. -Please refer to the following chapters for specific instructions on how to use them. - -# Usage - -You can use `SentenceTransformers` or `transformers` library to encode text. - -## Sentence Transformers - -```python -from sentence_transformers import SentenceTransformer - -# This model supports two prompts: "s2p_query" and "s2s_query" for sentence-to-passage and sentence-to-sentence tasks, respectively. -# They are defined in `config_sentence_transformers.json` -query_prompt_name = "s2p_query" -queries = [ - "What are some ways to reduce stress?", - "What are the benefits of drinking green tea?", -] -# docs do not need any prompts -docs = [ - "There are many effective ways to reduce stress. Some common techniques include deep breathing, meditation, and physical activity. Engaging in hobbies, spending time in nature, and connecting with loved ones can also help alleviate stress. Additionally, setting boundaries, practicing self-care, and learning to say no can prevent stress from building up.", - "Green tea has been consumed for centuries and is known for its potential health benefits. It contains antioxidants that may help protect the body against damage caused by free radicals. Regular consumption of green tea has been associated with improved heart health, enhanced cognitive function, and a reduced risk of certain types of cancer. The polyphenols in green tea may also have anti-inflammatory and weight loss properties.", -] - -# ！The default dimension is 1024, if you need other dimensions, please clone the model and modify `modules.json` to replace `2_Dense_1024` with another dimension, e.g. `2_Dense_256` or `2_Dense_8192` ! -model = SentenceTransformer("dunzhang/stella_en_400M_v5", trust_remote_code=True).cuda() -query_embeddings = model.encode(queries, prompt_name=query_prompt_name) -doc_embeddings = model.encode(docs) -print(query_embeddings.shape, doc_embeddings.shape) -# (2, 1024) (2, 1024) - -similarities = model.similarity(query_embeddings, doc_embeddings) -print(similarities) -# tensor([[0.8398, 0.2990], -# [0.3282, 0.8095]]) -``` - -## Transformers - -```python -import os -import torch -from transformers import AutoModel, AutoTokenizer -from sklearn.preprocessing import normalize - -query_prompt = "Instruct: Given a web search query, retrieve relevant passages that answer the query.\nQuery: " -queries = [ - "What are some ways to reduce stress?", - "What are the benefits of drinking green tea?", -] -queries = [query_prompt + query for query in queries] -# docs do not need any prompts -docs = [ - "There are many effective ways to reduce stress. Some common techniques include deep breathing, meditation, and physical activity. Engaging in hobbies, spending time in nature, and connecting with loved ones can also help alleviate stress. Additionally, setting boundaries, practicing self-care, and learning to say no can prevent stress from building up.", - "Green tea has been consumed for centuries and is known for its potential health benefits. It contains antioxidants that may help protect the body against damage caused by free radicals. Regular consumption of green tea has been associated with improved heart health, enhanced cognitive function, and a reduced risk of certain types of cancer. The polyphenols in green tea may also have anti-inflammatory and weight loss properties.", -] - -# The path of your model after cloning it -model_dir = "{Your MODEL_PATH}" - -vector_dim = 1024 -vector_linear_directory = f"2_Dense_{vector_dim}" -model = AutoModel.from_pretrained(model_dir, trust_remote_code=True).cuda().eval() -tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) -vector_linear = torch.nn.Linear(in_features=model.config.hidden_size, out_features=vector_dim) -vector_linear_dict = { - k.replace("linear.", ""): v for k, v in - torch.load(os.path.join(model_dir, f"{vector_linear_directory}/pytorch_model.bin")).items() -} -vector_linear.load_state_dict(vector_linear_dict) -vector_linear.cuda() - -# Embed the queries -with torch.no_grad(): - input_data = tokenizer(queries, padding="longest", truncation=True, max_length=512, return_tensors="pt") - input_data = {k: v.cuda() for k, v in input_data.items()} - attention_mask = input_data["attention_mask"] - last_hidden_state = model(**input_data)[0] - last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0) - query_vectors = last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] - query_vectors = normalize(vector_linear(query_vectors).cpu().numpy()) - -# Embed the documents -with torch.no_grad(): - input_data = tokenizer(docs, padding="longest", truncation=True, max_length=512, return_tensors="pt") - input_data = {k: v.cuda() for k, v in input_data.items()} - attention_mask = input_data["attention_mask"] - last_hidden_state = model(**input_data)[0] - last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0) - docs_vectors = last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] - docs_vectors = normalize(vector_linear(docs_vectors).cpu().numpy()) - -print(query_vectors.shape, docs_vectors.shape) -# (2, 1024) (2, 1024) - -similarities = query_vectors @ docs_vectors.T -print(similarities) -# [[0.8397531 0.29900077] -# [0.32818374 0.80954516]] -``` - -# FAQ - -Q: The details of training? - -A: The training method and datasets will be released in the future. (specific time unknown, may be provided in a paper) - -Q: How to choose a suitable prompt for my own task? - -A: In most cases, please use the s2p and s2s prompts. These two prompts account for the vast majority of the training -data. - -Q: How to reproduce MTEB results? - -A: Please use evaluation scripts in `Alibaba-NLP/gte-Qwen2-1.5B-instruct` or `intfloat/e5-mistral-7b-instruct` - -Q: Why each dimension has a linear weight? - -A: MRL has multiple training methods, we choose this method which has the best performance. - -Q: What is the sequence length of models? - -A: 512 is recommended, in our experiments, almost all models perform poorly on specialized long text retrieval datasets. Besides, the -model is trained on datasets of 512 length. This may be an optimization term. - +--- +model-index: +- name: stella_en_400M_v5 + results: + - dataset: + config: en + name: MTEB AmazonCounterfactualClassification (en) + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + split: test + type: mteb/amazon_counterfactual + metrics: + - type: accuracy + value: 92.35820895522387 + - type: ap + value: 70.81322736988783 + - type: ap_weighted + value: 70.81322736988783 + - type: f1 + value: 88.9505466159595 + - type: f1_weighted + value: 92.68630932872613 + - type: main_score + value: 92.35820895522387 + task: + type: Classification + - dataset: + config: default + name: MTEB AmazonPolarityClassification + revision: e2d317d38cd51312af73b3d32a06d1a08b442046 + split: test + type: mteb/amazon_polarity + metrics: + - type: accuracy + value: 97.1945 + - type: ap + value: 96.08192192244094 + - type: ap_weighted + value: 96.08192192244094 + - type: f1 + value: 97.1936887167346 + - type: f1_weighted + value: 97.1936887167346 + - type: main_score + value: 97.1945 + task: + type: Classification + - dataset: + config: en + name: MTEB AmazonReviewsClassification (en) + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + split: test + type: mteb/amazon_reviews_multi + metrics: + - type: accuracy + value: 59.528000000000006 + - type: f1 + value: 59.21016819840188 + - type: f1_weighted + value: 59.21016819840188 + - type: main_score + value: 59.528000000000006 + task: + type: Classification + - dataset: + config: default + name: MTEB ArguAna + revision: c22ab2a51041ffd869aaddef7af8d8215647e41a + split: test + type: mteb/arguana + metrics: + - type: main_score + value: 64.24 + - type: map_at_1 + value: 40.398 + - type: map_at_10 + value: 56.215 + - type: map_at_100 + value: 56.833999999999996 + - type: map_at_1000 + value: 56.835 + - type: map_at_20 + value: 56.747 + - type: map_at_3 + value: 52.181 + - type: map_at_5 + value: 54.628 + - type: mrr_at_1 + value: 41.25177809388336 + - type: mrr_at_10 + value: 56.570762491815216 + - type: mrr_at_100 + value: 57.17548614361504 + - type: mrr_at_1000 + value: 57.176650626377466 + - type: mrr_at_20 + value: 57.08916253512566 + - type: mrr_at_3 + value: 52.47747747747754 + - type: mrr_at_5 + value: 54.94547178757718 + - type: nauc_map_at_1000_diff1 + value: 22.408086887100158 + - type: nauc_map_at_1000_max + value: -8.730419096847543 + - type: nauc_map_at_1000_std + value: -17.789262741255737 + - type: nauc_map_at_100_diff1 + value: 22.407371684274025 + - type: nauc_map_at_100_max + value: -8.732263549026266 + - type: nauc_map_at_100_std + value: -17.79550515579994 + - type: nauc_map_at_10_diff1 + value: 21.925005073301246 + - type: nauc_map_at_10_max + value: -8.990323944492134 + - type: nauc_map_at_10_std + value: -18.199246301671458 + - type: nauc_map_at_1_diff1 + value: 26.23276644969203 + - type: nauc_map_at_1_max + value: -12.376511389571245 + - type: nauc_map_at_1_std + value: -18.11411715207284 + - type: nauc_map_at_20_diff1 + value: 22.32455790850922 + - type: nauc_map_at_20_max + value: -8.664671547236034 + - type: nauc_map_at_20_std + value: -17.8290016125137 + - type: nauc_map_at_3_diff1 + value: 22.395462147465064 + - type: nauc_map_at_3_max + value: -8.206580750918844 + - type: nauc_map_at_3_std + value: -17.604490446911484 + - type: nauc_map_at_5_diff1 + value: 21.95307379904799 + - type: nauc_map_at_5_max + value: -8.03958102978443 + - type: nauc_map_at_5_std + value: -17.36578866595004 + - type: nauc_mrr_at_1000_diff1 + value: 20.124236798365587 + - type: nauc_mrr_at_1000_max + value: -9.587376069575898 + - type: nauc_mrr_at_1000_std + value: -17.79191612151833 + - type: nauc_mrr_at_100_diff1 + value: 20.123612603474033 + - type: nauc_mrr_at_100_max + value: -9.589187218607831 + - type: nauc_mrr_at_100_std + value: -17.7981617777748 + - type: nauc_mrr_at_10_diff1 + value: 19.723683875738075 + - type: nauc_mrr_at_10_max + value: -9.774151729178815 + - type: nauc_mrr_at_10_std + value: -18.168668675495162 + - type: nauc_mrr_at_1_diff1 + value: 23.945332059908132 + - type: nauc_mrr_at_1_max + value: -12.260461466152819 + - type: nauc_mrr_at_1_std + value: -18.007194922921148 + - type: nauc_mrr_at_20_diff1 + value: 20.04819461810257 + - type: nauc_mrr_at_20_max + value: -9.518368283588936 + - type: nauc_mrr_at_20_std + value: -17.831608149836136 + - type: nauc_mrr_at_3_diff1 + value: 19.8571785245832 + - type: nauc_mrr_at_3_max + value: -9.464375021240478 + - type: nauc_mrr_at_3_std + value: -17.728533927330453 + - type: nauc_mrr_at_5_diff1 + value: 19.670313652167827 + - type: nauc_mrr_at_5_max + value: -8.966372585728434 + - type: nauc_mrr_at_5_std + value: -17.468955834324817 + - type: nauc_ndcg_at_1000_diff1 + value: 21.863049281767417 + - type: nauc_ndcg_at_1000_max + value: -8.18698520924057 + - type: nauc_ndcg_at_1000_std + value: -17.634483364794804 + - type: nauc_ndcg_at_100_diff1 + value: 21.849924385738586 + - type: nauc_ndcg_at_100_max + value: -8.226437560889345 + - type: nauc_ndcg_at_100_std + value: -17.774648478087002 + - type: nauc_ndcg_at_10_diff1 + value: 19.888395590413573 + - type: nauc_ndcg_at_10_max + value: -8.968706085632382 + - type: nauc_ndcg_at_10_std + value: -19.31386964628115 + - type: nauc_ndcg_at_1_diff1 + value: 26.23276644969203 + - type: nauc_ndcg_at_1_max + value: -12.376511389571245 + - type: nauc_ndcg_at_1_std + value: -18.11411715207284 + - type: nauc_ndcg_at_20_diff1 + value: 21.38413342416933 + - type: nauc_ndcg_at_20_max + value: -7.636238194084164 + - type: nauc_ndcg_at_20_std + value: -17.946390844693028 + - type: nauc_ndcg_at_3_diff1 + value: 21.29169165029195 + - type: nauc_ndcg_at_3_max + value: -6.793840499730093 + - type: nauc_ndcg_at_3_std + value: -17.52359001586737 + - type: nauc_ndcg_at_5_diff1 + value: 20.238297656671364 + - type: nauc_ndcg_at_5_max + value: -6.424992706950072 + - type: nauc_ndcg_at_5_std + value: -17.082391132291356 + - type: nauc_precision_at_1000_diff1 + value: -7.05195108528572 + - type: nauc_precision_at_1000_max + value: 34.439879624882145 + - type: nauc_precision_at_1000_std + value: 68.72436351659353 + - type: nauc_precision_at_100_diff1 + value: -2.769464113932605 + - type: nauc_precision_at_100_max + value: 9.89562961226698 + - type: nauc_precision_at_100_std + value: -0.5880967482224028 + - type: nauc_precision_at_10_diff1 + value: 2.1371544726832323 + - type: nauc_precision_at_10_max + value: -11.93051325147756 + - type: nauc_precision_at_10_std + value: -30.83144187392059 + - type: nauc_precision_at_1_diff1 + value: 26.23276644969203 + - type: nauc_precision_at_1_max + value: -12.376511389571245 + - type: nauc_precision_at_1_std + value: -18.11411715207284 + - type: nauc_precision_at_20_diff1 + value: 3.780146814257504 + - type: nauc_precision_at_20_max + value: 17.06527540214615 + - type: nauc_precision_at_20_std + value: -20.36832563035565 + - type: nauc_precision_at_3_diff1 + value: 17.63894384012077 + - type: nauc_precision_at_3_max + value: -2.0220490624638887 + - type: nauc_precision_at_3_std + value: -17.285601413493918 + - type: nauc_precision_at_5_diff1 + value: 12.557855071944601 + - type: nauc_precision_at_5_max + value: 0.5840236463956658 + - type: nauc_precision_at_5_std + value: -15.827224420217846 + - type: nauc_recall_at_1000_diff1 + value: -7.051951085286463 + - type: nauc_recall_at_1000_max + value: 34.43987962487738 + - type: nauc_recall_at_1000_std + value: 68.724363516591 + - type: nauc_recall_at_100_diff1 + value: -2.769464113930314 + - type: nauc_recall_at_100_max + value: 9.895629612270017 + - type: nauc_recall_at_100_std + value: -0.58809674821745 + - type: nauc_recall_at_10_diff1 + value: 2.1371544726834495 + - type: nauc_recall_at_10_max + value: -11.930513251477253 + - type: nauc_recall_at_10_std + value: -30.83144187392047 + - type: nauc_recall_at_1_diff1 + value: 26.23276644969203 + - type: nauc_recall_at_1_max + value: -12.376511389571245 + - type: nauc_recall_at_1_std + value: -18.11411715207284 + - type: nauc_recall_at_20_diff1 + value: 3.7801468142575922 + - type: nauc_recall_at_20_max + value: 17.0652754021456 + - type: nauc_recall_at_20_std + value: -20.36832563035559 + - type: nauc_recall_at_3_diff1 + value: 17.63894384012074 + - type: nauc_recall_at_3_max + value: -2.02204906246383 + - type: nauc_recall_at_3_std + value: -17.28560141349386 + - type: nauc_recall_at_5_diff1 + value: 12.55785507194463 + - type: nauc_recall_at_5_max + value: 0.5840236463957296 + - type: nauc_recall_at_5_std + value: -15.827224420217856 + - type: ndcg_at_1 + value: 40.398 + - type: ndcg_at_10 + value: 64.24 + - type: ndcg_at_100 + value: 66.631 + - type: ndcg_at_1000 + value: 66.65100000000001 + - type: ndcg_at_20 + value: 66.086 + - type: ndcg_at_3 + value: 55.938 + - type: ndcg_at_5 + value: 60.370000000000005 + - type: precision_at_1 + value: 40.398 + - type: precision_at_10 + value: 8.962 + - type: precision_at_100 + value: 0.9950000000000001 + - type: precision_at_1000 + value: 0.1 + - type: precision_at_20 + value: 4.836 + - type: precision_at_3 + value: 22.262 + - type: precision_at_5 + value: 15.519 + - type: recall_at_1 + value: 40.398 + - type: recall_at_10 + value: 89.616 + - type: recall_at_100 + value: 99.502 + - type: recall_at_1000 + value: 99.644 + - type: recall_at_20 + value: 96.72800000000001 + - type: recall_at_3 + value: 66.78500000000001 + - type: recall_at_5 + value: 77.596 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ArxivClusteringP2P + revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d + split: test + type: mteb/arxiv-clustering-p2p + metrics: + - type: main_score + value: 55.1564333205451 + - type: v_measure + value: 55.1564333205451 + - type: v_measure_std + value: 14.696883012214512 + task: + type: Clustering + - dataset: + config: default + name: MTEB ArxivClusteringS2S + revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 + split: test + type: mteb/arxiv-clustering-s2s + metrics: + - type: main_score + value: 49.823698316694795 + - type: v_measure + value: 49.823698316694795 + - type: v_measure_std + value: 14.951660654298186 + task: + type: Clustering + - dataset: + config: default + name: MTEB AskUbuntuDupQuestions + revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 + split: test + type: mteb/askubuntudupquestions-reranking + metrics: + - type: main_score + value: 66.15294503553424 + - type: map + value: 66.15294503553424 + - type: mrr + value: 78.53438420612935 + - type: nAUC_map_diff1 + value: 12.569697092717997 + - type: nAUC_map_max + value: 21.50670312412572 + - type: nAUC_map_std + value: 16.943786429229064 + - type: nAUC_mrr_diff1 + value: 15.590272897361238 + - type: nAUC_mrr_max + value: 34.96072022474653 + - type: nAUC_mrr_std + value: 21.649217605241045 + task: + type: Reranking + - dataset: + config: default + name: MTEB BIOSSES + revision: d3fb88f8f02e40887cd149695127462bbcf29b4a + split: test + type: mteb/biosses-sts + metrics: + - type: cosine_pearson + value: 85.7824546319275 + - type: cosine_spearman + value: 83.29587385660628 + - type: euclidean_pearson + value: 84.58764190565167 + - type: euclidean_spearman + value: 83.30069324352772 + - type: main_score + value: 83.29587385660628 + - type: manhattan_pearson + value: 84.95996839947179 + - type: manhattan_spearman + value: 83.87480271054358 + - type: pearson + value: 85.7824546319275 + - type: spearman + value: 83.29587385660628 + task: + type: STS + - dataset: + config: default + name: MTEB Banking77Classification + revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 + split: test + type: mteb/banking77 + metrics: + - type: accuracy + value: 89.30194805194806 + - type: f1 + value: 89.26182507266391 + - type: f1_weighted + value: 89.26182507266391 + - type: main_score + value: 89.30194805194806 + task: + type: Classification + - dataset: + config: default + name: MTEB BiorxivClusteringP2P + revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 + split: test + type: mteb/biorxiv-clustering-p2p + metrics: + - type: main_score + value: 50.67972171889736 + - type: v_measure + value: 50.67972171889736 + - type: v_measure_std + value: 0.7687409980036303 + task: + type: Clustering + - dataset: + config: default + name: MTEB BiorxivClusteringS2S + revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 + split: test + type: mteb/biorxiv-clustering-s2s + metrics: + - type: main_score + value: 45.80539715556144 + - type: v_measure + value: 45.80539715556144 + - type: v_measure_std + value: 0.9601346216579142 + task: + type: Clustering + - dataset: + config: default + name: MTEB CQADupstackRetrieval + revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 + split: test + type: mteb/cqadupstack + metrics: + - type: main_score + value: 44.361250000000005 + - type: map_at_1 + value: 28.304499999999997 + - type: map_at_10 + value: 38.54841666666666 + - type: map_at_100 + value: 39.83141666666667 + - type: map_at_1000 + value: 39.944750000000006 + - type: map_at_20 + value: 39.25341666666667 + - type: map_at_3 + value: 35.406749999999995 + - type: map_at_5 + value: 37.15558333333333 + - type: mrr_at_1 + value: 34.09077232860122 + - type: mrr_at_10 + value: 43.15445393211421 + - type: mrr_at_100 + value: 43.98645286848257 + - type: mrr_at_1000 + value: 44.037631313469404 + - type: mrr_at_20 + value: 43.64045813249614 + - type: mrr_at_3 + value: 40.674138648480486 + - type: mrr_at_5 + value: 42.106251182620255 + - type: nauc_map_at_1000_diff1 + value: 46.250011739434996 + - type: nauc_map_at_1000_max + value: 30.13664446260598 + - type: nauc_map_at_1000_std + value: 5.422301791618935 + - type: nauc_map_at_100_diff1 + value: 46.253631351999395 + - type: nauc_map_at_100_max + value: 30.12612918885181 + - type: nauc_map_at_100_std + value: 5.367077019987172 + - type: nauc_map_at_10_diff1 + value: 46.328171341741346 + - type: nauc_map_at_10_max + value: 29.80274612581464 + - type: nauc_map_at_10_std + value: 4.62996685176396 + - type: nauc_map_at_1_diff1 + value: 51.56118117729493 + - type: nauc_map_at_1_max + value: 27.94885243863768 + - type: nauc_map_at_1_std + value: 1.700366508927356 + - type: nauc_map_at_20_diff1 + value: 46.286750260299094 + - type: nauc_map_at_20_max + value: 29.979205290353278 + - type: nauc_map_at_20_std + value: 5.010588412441873 + - type: nauc_map_at_3_diff1 + value: 47.10018183619064 + - type: nauc_map_at_3_max + value: 29.062318206078753 + - type: nauc_map_at_3_std + value: 3.2235696254694197 + - type: nauc_map_at_5_diff1 + value: 46.41971733050039 + - type: nauc_map_at_5_max + value: 29.456798617695657 + - type: nauc_map_at_5_std + value: 4.0921691023077145 + - type: nauc_mrr_at_1000_diff1 + value: 45.88888977975723 + - type: nauc_mrr_at_1000_max + value: 32.162138978089544 + - type: nauc_mrr_at_1000_std + value: 6.2811943424217915 + - type: nauc_mrr_at_100_diff1 + value: 45.87480433011124 + - type: nauc_mrr_at_100_max + value: 32.16011334212834 + - type: nauc_mrr_at_100_std + value: 6.2865717772421785 + - type: nauc_mrr_at_10_diff1 + value: 45.849652904658825 + - type: nauc_mrr_at_10_max + value: 32.13847916232293 + - type: nauc_mrr_at_10_std + value: 6.105718728141999 + - type: nauc_mrr_at_1_diff1 + value: 51.013730325062156 + - type: nauc_mrr_at_1_max + value: 32.77457396492779 + - type: nauc_mrr_at_1_std + value: 4.415684893471724 + - type: nauc_mrr_at_20_diff1 + value: 45.86663046255274 + - type: nauc_mrr_at_20_max + value: 32.15219360697865 + - type: nauc_mrr_at_20_std + value: 6.19603046412763 + - type: nauc_mrr_at_3_diff1 + value: 46.522376582423185 + - type: nauc_mrr_at_3_max + value: 32.18259009733714 + - type: nauc_mrr_at_3_std + value: 5.288000648220897 + - type: nauc_mrr_at_5_diff1 + value: 45.86611481369745 + - type: nauc_mrr_at_5_max + value: 32.14261639054921 + - type: nauc_mrr_at_5_std + value: 5.8811238177073735 + - type: nauc_ndcg_at_1000_diff1 + value: 44.5055097547565 + - type: nauc_ndcg_at_1000_max + value: 31.149682057975458 + - type: nauc_ndcg_at_1000_std + value: 8.157937194901333 + - type: nauc_ndcg_at_100_diff1 + value: 44.12398363638596 + - type: nauc_ndcg_at_100_max + value: 30.878064321409994 + - type: nauc_ndcg_at_100_std + value: 8.40493441452808 + - type: nauc_ndcg_at_10_diff1 + value: 44.200093505221474 + - type: nauc_ndcg_at_10_max + value: 30.15267107733158 + - type: nauc_ndcg_at_10_std + value: 6.407495361566107 + - type: nauc_ndcg_at_1_diff1 + value: 51.013730325062156 + - type: nauc_ndcg_at_1_max + value: 32.77457396492779 + - type: nauc_ndcg_at_1_std + value: 4.415684893471724 + - type: nauc_ndcg_at_20_diff1 + value: 44.16988321564116 + - type: nauc_ndcg_at_20_max + value: 30.333532500651213 + - type: nauc_ndcg_at_20_std + value: 7.10024701386895 + - type: nauc_ndcg_at_3_diff1 + value: 45.35982873879988 + - type: nauc_ndcg_at_3_max + value: 30.288312457948702 + - type: nauc_ndcg_at_3_std + value: 4.653900898293395 + - type: nauc_ndcg_at_5_diff1 + value: 44.324558115380185 + - type: nauc_ndcg_at_5_max + value: 30.048149698941373 + - type: nauc_ndcg_at_5_std + value: 5.6684459618413205 + - type: nauc_precision_at_1000_diff1 + value: -7.282175798304458 + - type: nauc_precision_at_1000_max + value: 7.820142031765352 + - type: nauc_precision_at_1000_std + value: 11.736131836431172 + - type: nauc_precision_at_100_diff1 + value: 1.0222940256506976 + - type: nauc_precision_at_100_max + value: 16.12346497070298 + - type: nauc_precision_at_100_std + value: 18.202607395247874 + - type: nauc_precision_at_10_diff1 + value: 18.289439185857837 + - type: nauc_precision_at_10_max + value: 26.116517399154375 + - type: nauc_precision_at_10_std + value: 13.921214069982302 + - type: nauc_precision_at_1_diff1 + value: 51.013730325062156 + - type: nauc_precision_at_1_max + value: 32.77457396492779 + - type: nauc_precision_at_1_std + value: 4.415684893471724 + - type: nauc_precision_at_20_diff1 + value: 12.365165405210886 + - type: nauc_precision_at_20_max + value: 22.946297258937367 + - type: nauc_precision_at_20_std + value: 16.13862870358933 + - type: nauc_precision_at_3_diff1 + value: 32.063423642849685 + - type: nauc_precision_at_3_max + value: 30.140965811989407 + - type: nauc_precision_at_3_std + value: 8.501746262550146 + - type: nauc_precision_at_5_diff1 + value: 24.777203357717948 + - type: nauc_precision_at_5_max + value: 28.401579566848472 + - type: nauc_precision_at_5_std + value: 11.643246774390914 + - type: nauc_recall_at_1000_diff1 + value: 30.04216463401409 + - type: nauc_recall_at_1000_max + value: 34.98067760563842 + - type: nauc_recall_at_1000_std + value: 48.01453905250591 + - type: nauc_recall_at_100_diff1 + value: 31.193415507513972 + - type: nauc_recall_at_100_max + value: 28.69740149270981 + - type: nauc_recall_at_100_std + value: 25.20960758920368 + - type: nauc_recall_at_10_diff1 + value: 36.18870823636506 + - type: nauc_recall_at_10_max + value: 26.005625231341238 + - type: nauc_recall_at_10_std + value: 8.891983977041376 + - type: nauc_recall_at_1_diff1 + value: 51.56118117729493 + - type: nauc_recall_at_1_max + value: 27.94885243863768 + - type: nauc_recall_at_1_std + value: 1.700366508927356 + - type: nauc_recall_at_20_diff1 + value: 34.93996118564803 + - type: nauc_recall_at_20_max + value: 26.149961715956138 + - type: nauc_recall_at_20_std + value: 12.0657502367633 + - type: nauc_recall_at_3_diff1 + value: 40.80743946709512 + - type: nauc_recall_at_3_max + value: 26.443127773025783 + - type: nauc_recall_at_3_std + value: 3.7011448604241477 + - type: nauc_recall_at_5_diff1 + value: 37.608535157055776 + - type: nauc_recall_at_5_max + value: 26.168016189725822 + - type: nauc_recall_at_5_std + value: 6.344191564595316 + - type: ndcg_at_1 + value: 34.09083333333333 + - type: ndcg_at_10 + value: 44.361250000000005 + - type: ndcg_at_100 + value: 49.586166666666664 + - type: ndcg_at_1000 + value: 51.623583333333336 + - type: ndcg_at_20 + value: 46.40158333333333 + - type: ndcg_at_3 + value: 39.27733333333333 + - type: ndcg_at_5 + value: 41.662333333333336 + - type: precision_at_1 + value: 34.09083333333333 + - type: precision_at_10 + value: 7.957000000000002 + - type: precision_at_100 + value: 1.2521666666666669 + - type: precision_at_1000 + value: 0.16125 + - type: precision_at_20 + value: 4.6755 + - type: precision_at_3 + value: 18.402083333333334 + - type: precision_at_5 + value: 13.104333333333335 + - type: recall_at_1 + value: 28.304499999999997 + - type: recall_at_10 + value: 56.80666666666667 + - type: recall_at_100 + value: 79.66208333333334 + - type: recall_at_1000 + value: 93.6455 + - type: recall_at_20 + value: 64.2495 + - type: recall_at_3 + value: 42.431333333333335 + - type: recall_at_5 + value: 48.665416666666665 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ClimateFEVER + revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 + split: test + type: mteb/climate-fever + metrics: + - type: main_score + value: 43.525999999999996 + - type: map_at_1 + value: 19.291 + - type: map_at_10 + value: 33.471000000000004 + - type: map_at_100 + value: 35.388999999999996 + - type: map_at_1000 + value: 35.568 + - type: map_at_20 + value: 34.496 + - type: map_at_3 + value: 28.713 + - type: map_at_5 + value: 31.384 + - type: mrr_at_1 + value: 43.77850162866449 + - type: mrr_at_10 + value: 56.28576598934912 + - type: mrr_at_100 + value: 56.8588518168194 + - type: mrr_at_1000 + value: 56.878236725973544 + - type: mrr_at_20 + value: 56.6409328120183 + - type: mrr_at_3 + value: 53.56134636264935 + - type: mrr_at_5 + value: 55.27795874049956 + - type: nauc_map_at_1000_diff1 + value: 27.262513153363876 + - type: nauc_map_at_1000_max + value: 40.099398684385584 + - type: nauc_map_at_1000_std + value: 18.847812394005512 + - type: nauc_map_at_100_diff1 + value: 27.238993503030745 + - type: nauc_map_at_100_max + value: 40.07730434492169 + - type: nauc_map_at_100_std + value: 18.795349250833684 + - type: nauc_map_at_10_diff1 + value: 27.70929180366227 + - type: nauc_map_at_10_max + value: 39.55987024970173 + - type: nauc_map_at_10_std + value: 17.214881544648996 + - type: nauc_map_at_1_diff1 + value: 43.34155892182403 + - type: nauc_map_at_1_max + value: 38.23324890148018 + - type: nauc_map_at_1_std + value: 6.0781444393516075 + - type: nauc_map_at_20_diff1 + value: 27.311577477800103 + - type: nauc_map_at_20_max + value: 39.624414083413456 + - type: nauc_map_at_20_std + value: 18.149811054163287 + - type: nauc_map_at_3_diff1 + value: 30.475965062734367 + - type: nauc_map_at_3_max + value: 38.49324825043695 + - type: nauc_map_at_3_std + value: 13.357656038648487 + - type: nauc_map_at_5_diff1 + value: 28.425110095017747 + - type: nauc_map_at_5_max + value: 39.017894870747796 + - type: nauc_map_at_5_std + value: 15.543817194122564 + - type: nauc_mrr_at_1000_diff1 + value: 33.16689354701644 + - type: nauc_mrr_at_1000_max + value: 41.70755363247148 + - type: nauc_mrr_at_1000_std + value: 24.61667417463176 + - type: nauc_mrr_at_100_diff1 + value: 33.147229262917506 + - type: nauc_mrr_at_100_max + value: 41.712455697170725 + - type: nauc_mrr_at_100_std + value: 24.6418922043652 + - type: nauc_mrr_at_10_diff1 + value: 32.94185191112572 + - type: nauc_mrr_at_10_max + value: 41.64272730141954 + - type: nauc_mrr_at_10_std + value: 24.663391015702707 + - type: nauc_mrr_at_1_diff1 + value: 39.571969559016395 + - type: nauc_mrr_at_1_max + value: 39.396249211263495 + - type: nauc_mrr_at_1_std + value: 16.984149923258357 + - type: nauc_mrr_at_20_diff1 + value: 33.10040770334742 + - type: nauc_mrr_at_20_max + value: 41.807565560083034 + - type: nauc_mrr_at_20_std + value: 24.8064180365271 + - type: nauc_mrr_at_3_diff1 + value: 33.065406161485704 + - type: nauc_mrr_at_3_max + value: 41.049510969934694 + - type: nauc_mrr_at_3_std + value: 23.18371458928609 + - type: nauc_mrr_at_5_diff1 + value: 33.2389593543916 + - type: nauc_mrr_at_5_max + value: 41.629486918949915 + - type: nauc_mrr_at_5_std + value: 24.5777253036149 + - type: nauc_ndcg_at_1000_diff1 + value: 25.868840609197637 + - type: nauc_ndcg_at_1000_max + value: 42.79564910784761 + - type: nauc_ndcg_at_1000_std + value: 27.035091271680113 + - type: nauc_ndcg_at_100_diff1 + value: 25.019789319579942 + - type: nauc_ndcg_at_100_max + value: 42.482345143533735 + - type: nauc_ndcg_at_100_std + value: 26.76872010731345 + - type: nauc_ndcg_at_10_diff1 + value: 25.949464660653238 + - type: nauc_ndcg_at_10_max + value: 40.79769544643906 + - type: nauc_ndcg_at_10_std + value: 22.486116508973204 + - type: nauc_ndcg_at_1_diff1 + value: 39.571969559016395 + - type: nauc_ndcg_at_1_max + value: 39.396249211263495 + - type: nauc_ndcg_at_1_std + value: 16.984149923258357 + - type: nauc_ndcg_at_20_diff1 + value: 25.173455685962214 + - type: nauc_ndcg_at_20_max + value: 40.88873540662413 + - type: nauc_ndcg_at_20_std + value: 24.4451041955519 + - type: nauc_ndcg_at_3_diff1 + value: 28.185416070726333 + - type: nauc_ndcg_at_3_max + value: 39.10600031163912 + - type: nauc_ndcg_at_3_std + value: 18.42694044215541 + - type: nauc_ndcg_at_5_diff1 + value: 27.112647584005583 + - type: nauc_ndcg_at_5_max + value: 40.154045682322526 + - type: nauc_ndcg_at_5_std + value: 20.26822517176828 + - type: nauc_precision_at_1000_diff1 + value: -16.42087927044017 + - type: nauc_precision_at_1000_max + value: 3.5326295053913 + - type: nauc_precision_at_1000_std + value: 24.406810708493197 + - type: nauc_precision_at_100_diff1 + value: -12.17648135724982 + - type: nauc_precision_at_100_max + value: 15.895489260126183 + - type: nauc_precision_at_100_std + value: 32.48346122610907 + - type: nauc_precision_at_10_diff1 + value: -1.2493131347748072 + - type: nauc_precision_at_10_max + value: 26.409459305604376 + - type: nauc_precision_at_10_std + value: 31.115432019300016 + - type: nauc_precision_at_1_diff1 + value: 39.571969559016395 + - type: nauc_precision_at_1_max + value: 39.396249211263495 + - type: nauc_precision_at_1_std + value: 16.984149923258357 + - type: nauc_precision_at_20_diff1 + value: -6.597509397240593 + - type: nauc_precision_at_20_max + value: 21.461984620659695 + - type: nauc_precision_at_20_std + value: 32.9450259748889 + - type: nauc_precision_at_3_diff1 + value: 9.46378764865453 + - type: nauc_precision_at_3_max + value: 32.03650819375425 + - type: nauc_precision_at_3_std + value: 26.489382638510765 + - type: nauc_precision_at_5_diff1 + value: 3.5987036728169537 + - type: nauc_precision_at_5_max + value: 30.633955978579703 + - type: nauc_precision_at_5_std + value: 30.532430088014443 + - type: nauc_recall_at_1000_diff1 + value: 10.714633106872254 + - type: nauc_recall_at_1000_max + value: 43.94958623961 + - type: nauc_recall_at_1000_std + value: 51.78914468954123 + - type: nauc_recall_at_100_diff1 + value: 9.63781472255557 + - type: nauc_recall_at_100_max + value: 38.50917465255336 + - type: nauc_recall_at_100_std + value: 37.78623984642377 + - type: nauc_recall_at_10_diff1 + value: 16.480342820841688 + - type: nauc_recall_at_10_max + value: 35.982566867357406 + - type: nauc_recall_at_10_std + value: 23.30688188788895 + - type: nauc_recall_at_1_diff1 + value: 43.34155892182403 + - type: nauc_recall_at_1_max + value: 38.23324890148018 + - type: nauc_recall_at_1_std + value: 6.0781444393516075 + - type: nauc_recall_at_20_diff1 + value: 13.521048985146367 + - type: nauc_recall_at_20_max + value: 34.62462209239834 + - type: nauc_recall_at_20_std + value: 27.85924191501618 + - type: nauc_recall_at_3_diff1 + value: 23.57032748533523 + - type: nauc_recall_at_3_max + value: 36.32703197635613 + - type: nauc_recall_at_3_std + value: 15.730238734014337 + - type: nauc_recall_at_5_diff1 + value: 19.61387036368584 + - type: nauc_recall_at_5_max + value: 36.22030835529556 + - type: nauc_recall_at_5_std + value: 19.76310648649897 + - type: ndcg_at_1 + value: 43.779 + - type: ndcg_at_10 + value: 43.525999999999996 + - type: ndcg_at_100 + value: 50.138000000000005 + - type: ndcg_at_1000 + value: 52.991 + - type: ndcg_at_20 + value: 46.083 + - type: ndcg_at_3 + value: 38.002 + - type: ndcg_at_5 + value: 39.842 + - type: precision_at_1 + value: 43.779 + - type: precision_at_10 + value: 13.205 + - type: precision_at_100 + value: 2.051 + - type: precision_at_1000 + value: 0.259 + - type: precision_at_20 + value: 7.722999999999999 + - type: precision_at_3 + value: 28.903000000000002 + - type: precision_at_5 + value: 21.368000000000002 + - type: recall_at_1 + value: 19.291 + - type: recall_at_10 + value: 48.754 + - type: recall_at_100 + value: 70.97200000000001 + - type: recall_at_1000 + value: 86.611 + - type: recall_at_20 + value: 55.884 + - type: recall_at_3 + value: 34.101 + - type: recall_at_5 + value: 40.784 + task: + type: Retrieval + - dataset: + config: default + name: MTEB DBPedia + revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 + split: test + type: mteb/dbpedia + metrics: + - type: main_score + value: 49.884 + - type: map_at_1 + value: 9.913 + - type: map_at_10 + value: 23.186999999999998 + - type: map_at_100 + value: 34.207 + - type: map_at_1000 + value: 36.318 + - type: map_at_20 + value: 27.419 + - type: map_at_3 + value: 15.656 + - type: map_at_5 + value: 18.945999999999998 + - type: mrr_at_1 + value: 75.75 + - type: mrr_at_10 + value: 82.16279761904761 + - type: mrr_at_100 + value: 82.48445635330299 + - type: mrr_at_1000 + value: 82.4870246719901 + - type: mrr_at_20 + value: 82.36203632968338 + - type: mrr_at_3 + value: 81.29166666666666 + - type: mrr_at_5 + value: 82.02916666666667 + - type: nauc_map_at_1000_diff1 + value: 17.0739966990996 + - type: nauc_map_at_1000_max + value: 28.440065298437133 + - type: nauc_map_at_1000_std + value: 20.83498154003865 + - type: nauc_map_at_100_diff1 + value: 17.75982086107111 + - type: nauc_map_at_100_max + value: 26.87850835673573 + - type: nauc_map_at_100_std + value: 18.350282298599275 + - type: nauc_map_at_10_diff1 + value: 17.15984258564116 + - type: nauc_map_at_10_max + value: 10.846179132675553 + - type: nauc_map_at_10_std + value: -6.263534464094614 + - type: nauc_map_at_1_diff1 + value: 24.014897777973694 + - type: nauc_map_at_1_max + value: -4.556638938723358 + - type: nauc_map_at_1_std + value: -22.7844467526989 + - type: nauc_map_at_20_diff1 + value: 16.3179372493187 + - type: nauc_map_at_20_max + value: 17.176378915498915 + - type: nauc_map_at_20_std + value: 1.9378637630340372 + - type: nauc_map_at_3_diff1 + value: 19.12786794046792 + - type: nauc_map_at_3_max + value: 0.09063919305677291 + - type: nauc_map_at_3_std + value: -16.713143158330492 + - type: nauc_map_at_5_diff1 + value: 18.76504725420023 + - type: nauc_map_at_5_max + value: 5.040867712207419 + - type: nauc_map_at_5_std + value: -12.382578318931165 + - type: nauc_mrr_at_1000_diff1 + value: 54.61266255011247 + - type: nauc_mrr_at_1000_max + value: 60.83961280977112 + - type: nauc_mrr_at_1000_std + value: 32.70429260443016 + - type: nauc_mrr_at_100_diff1 + value: 54.61346236538542 + - type: nauc_mrr_at_100_max + value: 60.8407974416647 + - type: nauc_mrr_at_100_std + value: 32.69272843993462 + - type: nauc_mrr_at_10_diff1 + value: 54.74633685810871 + - type: nauc_mrr_at_10_max + value: 61.084525933097865 + - type: nauc_mrr_at_10_std + value: 33.001220210025565 + - type: nauc_mrr_at_1_diff1 + value: 56.12708423835806 + - type: nauc_mrr_at_1_max + value: 58.9314540998289 + - type: nauc_mrr_at_1_std + value: 27.39422607651012 + - type: nauc_mrr_at_20_diff1 + value: 54.58896150245695 + - type: nauc_mrr_at_20_max + value: 60.890929983464815 + - type: nauc_mrr_at_20_std + value: 32.65559641276393 + - type: nauc_mrr_at_3_diff1 + value: 54.38229071443791 + - type: nauc_mrr_at_3_max + value: 59.987849044098596 + - type: nauc_mrr_at_3_std + value: 33.439813880719974 + - type: nauc_mrr_at_5_diff1 + value: 54.961790262449824 + - type: nauc_mrr_at_5_max + value: 61.17705173908951 + - type: nauc_mrr_at_5_std + value: 33.30939850734856 + - type: nauc_ndcg_at_1000_diff1 + value: 29.27465932507067 + - type: nauc_ndcg_at_1000_max + value: 47.952543312315214 + - type: nauc_ndcg_at_1000_std + value: 36.17132236391485 + - type: nauc_ndcg_at_100_diff1 + value: 28.63072328980134 + - type: nauc_ndcg_at_100_max + value: 41.460833419186564 + - type: nauc_ndcg_at_100_std + value: 27.157100358988135 + - type: nauc_ndcg_at_10_diff1 + value: 23.41488013023301 + - type: nauc_ndcg_at_10_max + value: 39.27798133072349 + - type: nauc_ndcg_at_10_std + value: 21.979241438928312 + - type: nauc_ndcg_at_1_diff1 + value: 46.12120543657642 + - type: nauc_ndcg_at_1_max + value: 47.28452124039853 + - type: nauc_ndcg_at_1_std + value: 19.799884708952543 + - type: nauc_ndcg_at_20_diff1 + value: 23.627669045115574 + - type: nauc_ndcg_at_20_max + value: 35.88225062457673 + - type: nauc_ndcg_at_20_std + value: 18.218628030529498 + - type: nauc_ndcg_at_3_diff1 + value: 25.37309228946118 + - type: nauc_ndcg_at_3_max + value: 40.64426332992231 + - type: nauc_ndcg_at_3_std + value: 24.608330645901482 + - type: nauc_ndcg_at_5_diff1 + value: 24.055798594999654 + - type: nauc_ndcg_at_5_max + value: 41.16180524175431 + - type: nauc_ndcg_at_5_std + value: 24.048305528761315 + - type: nauc_precision_at_1000_diff1 + value: -18.234943251015576 + - type: nauc_precision_at_1000_max + value: 0.48708502364659184 + - type: nauc_precision_at_1000_std + value: 2.4473601543134027 + - type: nauc_precision_at_100_diff1 + value: -3.0077810947381227 + - type: nauc_precision_at_100_max + value: 25.27249321108913 + - type: nauc_precision_at_100_std + value: 37.36575792126928 + - type: nauc_precision_at_10_diff1 + value: -0.2393778190297635 + - type: nauc_precision_at_10_max + value: 36.40513293547299 + - type: nauc_precision_at_10_std + value: 37.4827885766009 + - type: nauc_precision_at_1_diff1 + value: 56.12708423835806 + - type: nauc_precision_at_1_max + value: 58.9314540998289 + - type: nauc_precision_at_1_std + value: 27.39422607651012 + - type: nauc_precision_at_20_diff1 + value: -1.2010133229402933 + - type: nauc_precision_at_20_max + value: 34.117541814385966 + - type: nauc_precision_at_20_std + value: 39.13273254177449 + - type: nauc_precision_at_3_diff1 + value: 11.757378092198486 + - type: nauc_precision_at_3_max + value: 42.637962482588875 + - type: nauc_precision_at_3_std + value: 37.42465077352342 + - type: nauc_precision_at_5_diff1 + value: 7.233177203405101 + - type: nauc_precision_at_5_max + value: 43.1663582897407 + - type: nauc_precision_at_5_std + value: 38.848449220750055 + - type: nauc_recall_at_1000_diff1 + value: 27.33938551969145 + - type: nauc_recall_at_1000_max + value: 45.5614254479334 + - type: nauc_recall_at_1000_std + value: 50.58528916250458 + - type: nauc_recall_at_100_diff1 + value: 23.610383761920097 + - type: nauc_recall_at_100_max + value: 31.422168485847184 + - type: nauc_recall_at_100_std + value: 25.58649926458304 + - type: nauc_recall_at_10_diff1 + value: 14.62495111808408 + - type: nauc_recall_at_10_max + value: 7.4295041277681095 + - type: nauc_recall_at_10_std + value: -9.32297089600654 + - type: nauc_recall_at_1_diff1 + value: 24.014897777973694 + - type: nauc_recall_at_1_max + value: -4.556638938723358 + - type: nauc_recall_at_1_std + value: -22.7844467526989 + - type: nauc_recall_at_20_diff1 + value: 14.027862330014662 + - type: nauc_recall_at_20_max + value: 12.437478731690844 + - type: nauc_recall_at_20_std + value: -3.0740743798103676 + - type: nauc_recall_at_3_diff1 + value: 16.354018356566712 + - type: nauc_recall_at_3_max + value: -2.9812231240997917 + - type: nauc_recall_at_3_std + value: -18.27746460743442 + - type: nauc_recall_at_5_diff1 + value: 16.81486583473587 + - type: nauc_recall_at_5_max + value: 2.420128513974744 + - type: nauc_recall_at_5_std + value: -14.441820321214108 + - type: ndcg_at_1 + value: 63.87500000000001 + - type: ndcg_at_10 + value: 49.884 + - type: ndcg_at_100 + value: 54.738 + - type: ndcg_at_1000 + value: 61.635 + - type: ndcg_at_20 + value: 48.894999999999996 + - type: ndcg_at_3 + value: 54.287 + - type: ndcg_at_5 + value: 52.40899999999999 + - type: precision_at_1 + value: 75.75 + - type: precision_at_10 + value: 40.9 + - type: precision_at_100 + value: 13.139999999999999 + - type: precision_at_1000 + value: 2.533 + - type: precision_at_20 + value: 30.8 + - type: precision_at_3 + value: 57.667 + - type: precision_at_5 + value: 51.05 + - type: recall_at_1 + value: 9.913 + - type: recall_at_10 + value: 28.591 + - type: recall_at_100 + value: 61.017999999999994 + - type: recall_at_1000 + value: 83.383 + - type: recall_at_20 + value: 37.834 + - type: recall_at_3 + value: 17.049 + - type: recall_at_5 + value: 21.685 + task: + type: Retrieval + - dataset: + config: default + name: MTEB EmotionClassification + revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 + split: test + type: mteb/emotion + metrics: + - type: accuracy + value: 78.77499999999999 + - type: f1 + value: 73.74058240799386 + - type: f1_weighted + value: 79.78804377638227 + - type: main_score + value: 78.77499999999999 + task: + type: Classification + - dataset: + config: default + name: MTEB FEVER + revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 + split: test + type: mteb/fever + metrics: + - type: main_score + value: 90.986 + - type: map_at_1 + value: 81.601 + - type: map_at_10 + value: 88.242 + - type: map_at_100 + value: 88.46000000000001 + - type: map_at_1000 + value: 88.472 + - type: map_at_20 + value: 88.375 + - type: map_at_3 + value: 87.237 + - type: map_at_5 + value: 87.85300000000001 + - type: mrr_at_1 + value: 87.81878187818782 + - type: mrr_at_10 + value: 92.20301196786335 + - type: mrr_at_100 + value: 92.24884236673292 + - type: mrr_at_1000 + value: 92.2496338899362 + - type: mrr_at_20 + value: 92.23112073283473 + - type: mrr_at_3 + value: 91.77417741774165 + - type: mrr_at_5 + value: 92.03970397039689 + - type: nauc_map_at_1000_diff1 + value: 56.54670664910505 + - type: nauc_map_at_1000_max + value: 33.08375749975477 + - type: nauc_map_at_1000_std + value: 2.7491595418252865 + - type: nauc_map_at_100_diff1 + value: 56.50887688686924 + - type: nauc_map_at_100_max + value: 33.075487189958494 + - type: nauc_map_at_100_std + value: 2.7675869969253375 + - type: nauc_map_at_10_diff1 + value: 56.08080806610569 + - type: nauc_map_at_10_max + value: 32.776972098819066 + - type: nauc_map_at_10_std + value: 2.5904846711290097 + - type: nauc_map_at_1_diff1 + value: 60.645344065853145 + - type: nauc_map_at_1_max + value: 31.232776777514797 + - type: nauc_map_at_1_std + value: -1.1946138176109171 + - type: nauc_map_at_20_diff1 + value: 56.28378454162355 + - type: nauc_map_at_20_max + value: 32.98207150385811 + - type: nauc_map_at_20_std + value: 2.8469814040214025 + - type: nauc_map_at_3_diff1 + value: 55.81958007095375 + - type: nauc_map_at_3_max + value: 31.602707711038313 + - type: nauc_map_at_3_std + value: 0.8117019292273401 + - type: nauc_map_at_5_diff1 + value: 55.706025752316535 + - type: nauc_map_at_5_max + value: 32.16032683604737 + - type: nauc_map_at_5_std + value: 1.8853201503498669 + - type: nauc_mrr_at_1000_diff1 + value: 75.4997173366251 + - type: nauc_mrr_at_1000_max + value: 41.49117135484116 + - type: nauc_mrr_at_1000_std + value: -2.0636172883680852 + - type: nauc_mrr_at_100_diff1 + value: 75.50118860648519 + - type: nauc_mrr_at_100_max + value: 41.49490161517194 + - type: nauc_mrr_at_100_std + value: -2.057024385178682 + - type: nauc_mrr_at_10_diff1 + value: 75.47295153099428 + - type: nauc_mrr_at_10_max + value: 41.55003304042536 + - type: nauc_mrr_at_10_std + value: -2.0353663198929253 + - type: nauc_mrr_at_1_diff1 + value: 76.632058433229 + - type: nauc_mrr_at_1_max + value: 39.754483718891656 + - type: nauc_mrr_at_1_std + value: -2.962241058101701 + - type: nauc_mrr_at_20_diff1 + value: 75.47221882396194 + - type: nauc_mrr_at_20_max + value: 41.50779280480839 + - type: nauc_mrr_at_20_std + value: -1.9620212266426307 + - type: nauc_mrr_at_3_diff1 + value: 75.5682297897137 + - type: nauc_mrr_at_3_max + value: 41.53543801506081 + - type: nauc_mrr_at_3_std + value: -3.391681195945978 + - type: nauc_mrr_at_5_diff1 + value: 75.37562775183947 + - type: nauc_mrr_at_5_max + value: 41.42028509006753 + - type: nauc_mrr_at_5_std + value: -2.418698675622726 + - type: nauc_ndcg_at_1000_diff1 + value: 59.364557011624 + - type: nauc_ndcg_at_1000_max + value: 35.4112238125149 + - type: nauc_ndcg_at_1000_std + value: 3.717516193303376 + - type: nauc_ndcg_at_100_diff1 + value: 58.55706703023122 + - type: nauc_ndcg_at_100_max + value: 35.352285999934594 + - type: nauc_ndcg_at_100_std + value: 4.273437944266781 + - type: nauc_ndcg_at_10_diff1 + value: 56.77422701267037 + - type: nauc_ndcg_at_10_max + value: 34.24909893882957 + - type: nauc_ndcg_at_10_std + value: 4.178151434006727 + - type: nauc_ndcg_at_1_diff1 + value: 76.632058433229 + - type: nauc_ndcg_at_1_max + value: 39.754483718891656 + - type: nauc_ndcg_at_1_std + value: -2.962241058101701 + - type: nauc_ndcg_at_20_diff1 + value: 57.27343398231262 + - type: nauc_ndcg_at_20_max + value: 34.7416626740278 + - type: nauc_ndcg_at_20_std + value: 4.955858766014002 + - type: nauc_ndcg_at_3_diff1 + value: 57.69267803121093 + - type: nauc_ndcg_at_3_max + value: 33.13744317023105 + - type: nauc_ndcg_at_3_std + value: 0.40380284030057023 + - type: nauc_ndcg_at_5_diff1 + value: 56.57461019113917 + - type: nauc_ndcg_at_5_max + value: 33.244657840804386 + - type: nauc_ndcg_at_5_std + value: 2.5121440827702046 + - type: nauc_precision_at_1000_diff1 + value: -14.54492513449718 + - type: nauc_precision_at_1000_max + value: -5.94552147573623 + - type: nauc_precision_at_1000_std + value: 1.2446209816057374 + - type: nauc_precision_at_100_diff1 + value: -15.452676132568344 + - type: nauc_precision_at_100_max + value: -3.760241749847617 + - type: nauc_precision_at_100_std + value: 4.623534605290865 + - type: nauc_precision_at_10_diff1 + value: -12.712908026086176 + - type: nauc_precision_at_10_max + value: 0.45241316994816805 + - type: nauc_precision_at_10_std + value: 7.849478570138391 + - type: nauc_precision_at_1_diff1 + value: 76.632058433229 + - type: nauc_precision_at_1_max + value: 39.754483718891656 + - type: nauc_precision_at_1_std + value: -2.962241058101701 + - type: nauc_precision_at_20_diff1 + value: -14.514618673172041 + - type: nauc_precision_at_20_max + value: -1.113635490621818 + - type: nauc_precision_at_20_std + value: 8.599811730457576 + - type: nauc_precision_at_3_diff1 + value: 6.1367799850003815 + - type: nauc_precision_at_3_max + value: 8.466271950897857 + - type: nauc_precision_at_3_std + value: 1.7458051543195068 + - type: nauc_precision_at_5_diff1 + value: -5.804548945783379 + - type: nauc_precision_at_5_max + value: 3.4060251839074818 + - type: nauc_precision_at_5_std + value: 5.583410511782371 + - type: nauc_recall_at_1000_diff1 + value: 19.329432953574095 + - type: nauc_recall_at_1000_max + value: 43.260442595158736 + - type: nauc_recall_at_1000_std + value: 53.89644660661804 + - type: nauc_recall_at_100_diff1 + value: 21.265326296051235 + - type: nauc_recall_at_100_max + value: 38.573000195373695 + - type: nauc_recall_at_100_std + value: 42.169391082152785 + - type: nauc_recall_at_10_diff1 + value: 29.785129558987432 + - type: nauc_recall_at_10_max + value: 28.379657867558034 + - type: nauc_recall_at_10_std + value: 21.132574624091973 + - type: nauc_recall_at_1_diff1 + value: 60.645344065853145 + - type: nauc_recall_at_1_max + value: 31.232776777514797 + - type: nauc_recall_at_1_std + value: -1.1946138176109171 + - type: nauc_recall_at_20_diff1 + value: 25.88845612373954 + - type: nauc_recall_at_20_max + value: 30.24785945821152 + - type: nauc_recall_at_20_std + value: 31.73911437468067 + - type: nauc_recall_at_3_diff1 + value: 42.2968464797395 + - type: nauc_recall_at_3_max + value: 26.494318009870018 + - type: nauc_recall_at_3_std + value: 2.6045977160467544 + - type: nauc_recall_at_5_diff1 + value: 35.81340094401374 + - type: nauc_recall_at_5_max + value: 25.91082947510634 + - type: nauc_recall_at_5_std + value: 9.759404930864779 + - type: ndcg_at_1 + value: 87.819 + - type: ndcg_at_10 + value: 90.986 + - type: ndcg_at_100 + value: 91.69 + - type: ndcg_at_1000 + value: 91.863 + - type: ndcg_at_20 + value: 91.293 + - type: ndcg_at_3 + value: 89.621 + - type: ndcg_at_5 + value: 90.333 + - type: precision_at_1 + value: 87.819 + - type: precision_at_10 + value: 10.753 + - type: precision_at_100 + value: 1.138 + - type: precision_at_1000 + value: 0.117 + - type: precision_at_20 + value: 5.4879999999999995 + - type: precision_at_3 + value: 33.703 + - type: precision_at_5 + value: 20.831 + - type: recall_at_1 + value: 81.601 + - type: recall_at_10 + value: 95.44200000000001 + - type: recall_at_100 + value: 98.14399999999999 + - type: recall_at_1000 + value: 99.157 + - type: recall_at_20 + value: 96.43 + - type: recall_at_3 + value: 91.729 + - type: recall_at_5 + value: 93.552 + task: + type: Retrieval + - dataset: + config: default + name: MTEB FiQA2018 + revision: 27a168819829fe9bcd655c2df245fb19452e8e06 + split: test + type: mteb/fiqa + metrics: + - type: main_score + value: 56.056 + - type: map_at_1 + value: 28.666000000000004 + - type: map_at_10 + value: 47.437000000000005 + - type: map_at_100 + value: 49.537 + - type: map_at_1000 + value: 49.665 + - type: map_at_20 + value: 48.618 + - type: map_at_3 + value: 41.355 + - type: map_at_5 + value: 44.525 + - type: mrr_at_1 + value: 55.55555555555556 + - type: mrr_at_10 + value: 63.705173427395614 + - type: mrr_at_100 + value: 64.25449940779741 + - type: mrr_at_1000 + value: 64.27635581092147 + - type: mrr_at_20 + value: 64.03796029079103 + - type: mrr_at_3 + value: 61.49691358024688 + - type: mrr_at_5 + value: 62.73148148148143 + - type: nauc_map_at_1000_diff1 + value: 43.24282910397747 + - type: nauc_map_at_1000_max + value: 28.506093180265644 + - type: nauc_map_at_1000_std + value: -13.040508386155054 + - type: nauc_map_at_100_diff1 + value: 43.23650442904607 + - type: nauc_map_at_100_max + value: 28.470565635459156 + - type: nauc_map_at_100_std + value: -12.988098780714935 + - type: nauc_map_at_10_diff1 + value: 43.393840733087686 + - type: nauc_map_at_10_max + value: 26.637302062720153 + - type: nauc_map_at_10_std + value: -14.47500292113762 + - type: nauc_map_at_1_diff1 + value: 47.705150227211725 + - type: nauc_map_at_1_max + value: 15.354189686550129 + - type: nauc_map_at_1_std + value: -14.559819859039067 + - type: nauc_map_at_20_diff1 + value: 43.14121075706104 + - type: nauc_map_at_20_max + value: 27.811170590408395 + - type: nauc_map_at_20_std + value: -13.459413585283583 + - type: nauc_map_at_3_diff1 + value: 44.33938667720801 + - type: nauc_map_at_3_max + value: 21.785619884549398 + - type: nauc_map_at_3_std + value: -15.569980103071593 + - type: nauc_map_at_5_diff1 + value: 43.39280905665027 + - type: nauc_map_at_5_max + value: 25.021492190645017 + - type: nauc_map_at_5_std + value: -14.48856622187443 + - type: nauc_mrr_at_1000_diff1 + value: 52.971563939946286 + - type: nauc_mrr_at_1000_max + value: 38.88019486172324 + - type: nauc_mrr_at_1000_std + value: -12.412991642381616 + - type: nauc_mrr_at_100_diff1 + value: 52.978468139876945 + - type: nauc_mrr_at_100_max + value: 38.89751787948751 + - type: nauc_mrr_at_100_std + value: -12.3677876252269 + - type: nauc_mrr_at_10_diff1 + value: 52.78507148048174 + - type: nauc_mrr_at_10_max + value: 38.55079809310022 + - type: nauc_mrr_at_10_std + value: -12.944127025078755 + - type: nauc_mrr_at_1_diff1 + value: 55.52626805861546 + - type: nauc_mrr_at_1_max + value: 40.49306809164979 + - type: nauc_mrr_at_1_std + value: -12.886607701317681 + - type: nauc_mrr_at_20_diff1 + value: 52.9592152665678 + - type: nauc_mrr_at_20_max + value: 38.88514014589964 + - type: nauc_mrr_at_20_std + value: -12.434464359819444 + - type: nauc_mrr_at_3_diff1 + value: 52.73696844091174 + - type: nauc_mrr_at_3_max + value: 38.61018727252859 + - type: nauc_mrr_at_3_std + value: -13.123989867364166 + - type: nauc_mrr_at_5_diff1 + value: 53.037110010188 + - type: nauc_mrr_at_5_max + value: 38.44770729849151 + - type: nauc_mrr_at_5_std + value: -13.49318771828972 + - type: nauc_ndcg_at_1000_diff1 + value: 44.73813840091289 + - type: nauc_ndcg_at_1000_max + value: 33.70113904685389 + - type: nauc_ndcg_at_1000_std + value: -10.328687058192742 + - type: nauc_ndcg_at_100_diff1 + value: 44.595174119928835 + - type: nauc_ndcg_at_100_max + value: 33.4788285112467 + - type: nauc_ndcg_at_100_std + value: -8.695355259716946 + - type: nauc_ndcg_at_10_diff1 + value: 44.39837225263 + - type: nauc_ndcg_at_10_max + value: 29.188289725593393 + - type: nauc_ndcg_at_10_std + value: -13.67608323673103 + - type: nauc_ndcg_at_1_diff1 + value: 55.52626805861546 + - type: nauc_ndcg_at_1_max + value: 40.49306809164979 + - type: nauc_ndcg_at_1_std + value: -12.886607701317681 + - type: nauc_ndcg_at_20_diff1 + value: 44.24661739902305 + - type: nauc_ndcg_at_20_max + value: 31.667868318249965 + - type: nauc_ndcg_at_20_std + value: -10.65470780066342 + - type: nauc_ndcg_at_3_diff1 + value: 43.39857166975522 + - type: nauc_ndcg_at_3_max + value: 31.764668313577495 + - type: nauc_ndcg_at_3_std + value: -14.494866954678152 + - type: nauc_ndcg_at_5_diff1 + value: 43.16976647347281 + - type: nauc_ndcg_at_5_max + value: 29.878329062643143 + - type: nauc_ndcg_at_5_std + value: -13.987689089179739 + - type: nauc_precision_at_1000_diff1 + value: -9.807973252625484 + - type: nauc_precision_at_1000_max + value: 26.6279603849494 + - type: nauc_precision_at_1000_std + value: 7.113187103520632 + - type: nauc_precision_at_100_diff1 + value: -4.777149603323976 + - type: nauc_precision_at_100_max + value: 31.03410463692187 + - type: nauc_precision_at_100_std + value: 10.463144150275435 + - type: nauc_precision_at_10_diff1 + value: 8.691528703215962 + - type: nauc_precision_at_10_max + value: 33.329579434123374 + - type: nauc_precision_at_10_std + value: -0.8002015226329403 + - type: nauc_precision_at_1_diff1 + value: 55.52626805861546 + - type: nauc_precision_at_1_max + value: 40.49306809164979 + - type: nauc_precision_at_1_std + value: -12.886607701317681 + - type: nauc_precision_at_20_diff1 + value: 3.4564653474184284 + - type: nauc_precision_at_20_max + value: 34.401070158471136 + - type: nauc_precision_at_20_std + value: 5.813431200164549 + - type: nauc_precision_at_3_diff1 + value: 22.463219705462187 + - type: nauc_precision_at_3_max + value: 34.77413976546924 + - type: nauc_precision_at_3_std + value: -7.083890789741479 + - type: nauc_precision_at_5_diff1 + value: 14.011006004883154 + - type: nauc_precision_at_5_max + value: 35.73655466853702 + - type: nauc_precision_at_5_std + value: -2.8395172077771598 + - type: nauc_recall_at_1000_diff1 + value: 16.478046357391555 + - type: nauc_recall_at_1000_max + value: 43.231704288282344 + - type: nauc_recall_at_1000_std + value: 38.430684937573645 + - type: nauc_recall_at_100_diff1 + value: 30.764718344602436 + - type: nauc_recall_at_100_max + value: 31.769050487166655 + - type: nauc_recall_at_100_std + value: 23.48468311677149 + - type: nauc_recall_at_10_diff1 + value: 34.47339565324045 + - type: nauc_recall_at_10_max + value: 19.054212335800454 + - type: nauc_recall_at_10_std + value: -11.039734015330437 + - type: nauc_recall_at_1_diff1 + value: 47.705150227211725 + - type: nauc_recall_at_1_max + value: 15.354189686550129 + - type: nauc_recall_at_1_std + value: -14.559819859039067 + - type: nauc_recall_at_20_diff1 + value: 32.1011474016873 + - type: nauc_recall_at_20_max + value: 25.546372988304423 + - type: nauc_recall_at_20_std + value: -0.007233471152482897 + - type: nauc_recall_at_3_diff1 + value: 37.5708138019065 + - type: nauc_recall_at_3_max + value: 16.66410785756736 + - type: nauc_recall_at_3_std + value: -15.404817020108966 + - type: nauc_recall_at_5_diff1 + value: 35.714519648479595 + - type: nauc_recall_at_5_max + value: 19.02075233009296 + - type: nauc_recall_at_5_std + value: -13.180963359760725 + - type: ndcg_at_1 + value: 55.556000000000004 + - type: ndcg_at_10 + value: 56.056 + - type: ndcg_at_100 + value: 62.44 + - type: ndcg_at_1000 + value: 64.263 + - type: ndcg_at_20 + value: 58.638999999999996 + - type: ndcg_at_3 + value: 51.722 + - type: ndcg_at_5 + value: 52.701 + - type: precision_at_1 + value: 55.556000000000004 + - type: precision_at_10 + value: 15.679000000000002 + - type: precision_at_100 + value: 2.252 + - type: precision_at_1000 + value: 0.257 + - type: precision_at_20 + value: 9.02 + - type: precision_at_3 + value: 34.619 + - type: precision_at_5 + value: 25.093 + - type: recall_at_1 + value: 28.666000000000004 + - type: recall_at_10 + value: 63.717999999999996 + - type: recall_at_100 + value: 86.938 + - type: recall_at_1000 + value: 97.603 + - type: recall_at_20 + value: 71.649 + - type: recall_at_3 + value: 46.663 + - type: recall_at_5 + value: 53.313 + task: + type: Retrieval + - dataset: + config: default + name: MTEB HotpotQA + revision: ab518f4d6fcca38d87c25209f94beba119d02014 + split: test + type: mteb/hotpotqa + metrics: + - type: main_score + value: 71.74199999999999 + - type: map_at_1 + value: 41.729 + - type: map_at_10 + value: 63.168 + - type: map_at_100 + value: 64.132 + - type: map_at_1000 + value: 64.199 + - type: map_at_20 + value: 63.736000000000004 + - type: map_at_3 + value: 59.826 + - type: map_at_5 + value: 61.882000000000005 + - type: mrr_at_1 + value: 83.45712356515868 + - type: mrr_at_10 + value: 87.850342432719 + - type: mrr_at_100 + value: 88.0016320691113 + - type: mrr_at_1000 + value: 88.00576596968136 + - type: mrr_at_20 + value: 87.94463253190389 + - type: mrr_at_3 + value: 87.13706954760278 + - type: mrr_at_5 + value: 87.59419311276136 + - type: nauc_map_at_1000_diff1 + value: 13.635446621095054 + - type: nauc_map_at_1000_max + value: 18.670632529445633 + - type: nauc_map_at_1000_std + value: 10.444842636150575 + - type: nauc_map_at_100_diff1 + value: 13.599262398010783 + - type: nauc_map_at_100_max + value: 18.636389405484806 + - type: nauc_map_at_100_std + value: 10.460027483576043 + - type: nauc_map_at_10_diff1 + value: 13.235053919323942 + - type: nauc_map_at_10_max + value: 18.252140477080047 + - type: nauc_map_at_10_std + value: 9.9075337042203 + - type: nauc_map_at_1_diff1 + value: 76.51940497836482 + - type: nauc_map_at_1_max + value: 51.251419487235474 + - type: nauc_map_at_1_std + value: 0.16714896857146574 + - type: nauc_map_at_20_diff1 + value: 13.4178245722222 + - type: nauc_map_at_20_max + value: 18.40988771210718 + - type: nauc_map_at_20_std + value: 10.216685163366282 + - type: nauc_map_at_3_diff1 + value: 13.38370761663418 + - type: nauc_map_at_3_max + value: 17.760962555456537 + - type: nauc_map_at_3_std + value: 7.15741965624388 + - type: nauc_map_at_5_diff1 + value: 13.138133309724855 + - type: nauc_map_at_5_max + value: 17.871761295251044 + - type: nauc_map_at_5_std + value: 8.475147426940074 + - type: nauc_mrr_at_1000_diff1 + value: 75.82650818891959 + - type: nauc_mrr_at_1000_max + value: 53.6736100668434 + - type: nauc_mrr_at_1000_std + value: 1.8025016349213916 + - type: nauc_mrr_at_100_diff1 + value: 75.82530574210111 + - type: nauc_mrr_at_100_max + value: 53.68067545829002 + - type: nauc_mrr_at_100_std + value: 1.8147470536495791 + - type: nauc_mrr_at_10_diff1 + value: 75.8330135686799 + - type: nauc_mrr_at_10_max + value: 53.78626885349077 + - type: nauc_mrr_at_10_std + value: 1.7975782717226636 + - type: nauc_mrr_at_1_diff1 + value: 76.51940497836482 + - type: nauc_mrr_at_1_max + value: 51.251419487235474 + - type: nauc_mrr_at_1_std + value: 0.16714896857146574 + - type: nauc_mrr_at_20_diff1 + value: 75.82783382464166 + - type: nauc_mrr_at_20_max + value: 53.68364567043885 + - type: nauc_mrr_at_20_std + value: 1.742037904463963 + - type: nauc_mrr_at_3_diff1 + value: 75.6944609768663 + - type: nauc_mrr_at_3_max + value: 53.803941340341666 + - type: nauc_mrr_at_3_std + value: 1.1849945458077804 + - type: nauc_mrr_at_5_diff1 + value: 75.73006960604903 + - type: nauc_mrr_at_5_max + value: 53.62223096420106 + - type: nauc_mrr_at_5_std + value: 1.6144067563410909 + - type: nauc_ndcg_at_1000_diff1 + value: 21.58025241642726 + - type: nauc_ndcg_at_1000_max + value: 24.675747527001153 + - type: nauc_ndcg_at_1000_std + value: 13.075943547492718 + - type: nauc_ndcg_at_100_diff1 + value: 20.30260137544846 + - type: nauc_ndcg_at_100_max + value: 23.757528813872018 + - type: nauc_ndcg_at_100_std + value: 13.648994687574062 + - type: nauc_ndcg_at_10_diff1 + value: 18.995052360997818 + - type: nauc_ndcg_at_10_max + value: 22.254260808196037 + - type: nauc_ndcg_at_10_std + value: 11.27212390633054 + - type: nauc_ndcg_at_1_diff1 + value: 76.51940497836482 + - type: nauc_ndcg_at_1_max + value: 51.251419487235474 + - type: nauc_ndcg_at_1_std + value: 0.16714896857146574 + - type: nauc_ndcg_at_20_diff1 + value: 19.333742380695757 + - type: nauc_ndcg_at_20_max + value: 22.527779834633364 + - type: nauc_ndcg_at_20_std + value: 12.161009000707917 + - type: nauc_ndcg_at_3_diff1 + value: 20.013329040965534 + - type: nauc_ndcg_at_3_max + value: 21.99692460311921 + - type: nauc_ndcg_at_3_std + value: 6.8076290638386165 + - type: nauc_ndcg_at_5_diff1 + value: 19.08226315942471 + - type: nauc_ndcg_at_5_max + value: 21.71185964294168 + - type: nauc_ndcg_at_5_std + value: 8.671911269518214 + - type: nauc_precision_at_1000_diff1 + value: 2.4462475489446764 + - type: nauc_precision_at_1000_max + value: 29.145662064268578 + - type: nauc_precision_at_1000_std + value: 49.20704909525856 + - type: nauc_precision_at_100_diff1 + value: 0.11271196725540299 + - type: nauc_precision_at_100_max + value: 17.37584606388067 + - type: nauc_precision_at_100_std + value: 34.66099346244071 + - type: nauc_precision_at_10_diff1 + value: 2.9923183951227825 + - type: nauc_precision_at_10_max + value: 14.261884731124264 + - type: nauc_precision_at_10_std + value: 18.084188795498378 + - type: nauc_precision_at_1_diff1 + value: 76.51940497836482 + - type: nauc_precision_at_1_max + value: 51.251419487235474 + - type: nauc_precision_at_1_std + value: 0.16714896857146574 + - type: nauc_precision_at_20_diff1 + value: 1.9180293008303761 + - type: nauc_precision_at_20_max + value: 13.832269193468512 + - type: nauc_precision_at_20_std + value: 21.65284406055607 + - type: nauc_precision_at_3_diff1 + value: 7.226609484731811 + - type: nauc_precision_at_3_max + value: 15.162908526977272 + - type: nauc_precision_at_3_std + value: 8.451859972962776 + - type: nauc_precision_at_5_diff1 + value: 4.705236845538159 + - type: nauc_precision_at_5_max + value: 14.022910843582666 + - type: nauc_precision_at_5_std + value: 11.777269322821605 + - type: nauc_recall_at_1000_diff1 + value: 2.446247548945172 + - type: nauc_recall_at_1000_max + value: 29.14566206426889 + - type: nauc_recall_at_1000_std + value: 49.20704909525879 + - type: nauc_recall_at_100_diff1 + value: 0.1127119672553316 + - type: nauc_recall_at_100_max + value: 17.37584606388062 + - type: nauc_recall_at_100_std + value: 34.660993462440686 + - type: nauc_recall_at_10_diff1 + value: 2.9923183951227927 + - type: nauc_recall_at_10_max + value: 14.261884731124299 + - type: nauc_recall_at_10_std + value: 18.08418879549837 + - type: nauc_recall_at_1_diff1 + value: 76.51940497836482 + - type: nauc_recall_at_1_max + value: 51.251419487235474 + - type: nauc_recall_at_1_std + value: 0.16714896857146574 + - type: nauc_recall_at_20_diff1 + value: 1.918029300830432 + - type: nauc_recall_at_20_max + value: 13.832269193468566 + - type: nauc_recall_at_20_std + value: 21.65284406055605 + - type: nauc_recall_at_3_diff1 + value: 7.226609484731802 + - type: nauc_recall_at_3_max + value: 15.162908526977182 + - type: nauc_recall_at_3_std + value: 8.451859972962634 + - type: nauc_recall_at_5_diff1 + value: 4.705236845538197 + - type: nauc_recall_at_5_max + value: 14.02291084358265 + - type: nauc_recall_at_5_std + value: 11.777269322821638 + - type: ndcg_at_1 + value: 83.45700000000001 + - type: ndcg_at_10 + value: 71.74199999999999 + - type: ndcg_at_100 + value: 75.008 + - type: ndcg_at_1000 + value: 76.242 + - type: ndcg_at_20 + value: 73.114 + - type: ndcg_at_3 + value: 67.128 + - type: ndcg_at_5 + value: 69.645 + - type: precision_at_1 + value: 83.45700000000001 + - type: precision_at_10 + value: 14.747 + - type: precision_at_100 + value: 1.73 + - type: precision_at_1000 + value: 0.189 + - type: precision_at_20 + value: 7.8149999999999995 + - type: precision_at_3 + value: 42.323 + - type: precision_at_5 + value: 27.381 + - type: recall_at_1 + value: 41.729 + - type: recall_at_10 + value: 73.734 + - type: recall_at_100 + value: 86.502 + - type: recall_at_1000 + value: 94.60499999999999 + - type: recall_at_20 + value: 78.14999999999999 + - type: recall_at_3 + value: 63.483999999999995 + - type: recall_at_5 + value: 68.45400000000001 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ImdbClassification + revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 + split: test + type: mteb/imdb + metrics: + - type: accuracy + value: 96.4904 + - type: ap + value: 94.85481918794709 + - type: ap_weighted + value: 94.85481918794709 + - type: f1 + value: 96.4898592305707 + - type: f1_weighted + value: 96.4898592305707 + - type: main_score + value: 96.4904 + task: + type: Classification + - dataset: + config: default + name: MTEB MSMARCO + revision: c5a29a104738b98a9e76336939199e264163d4a0 + split: dev + type: mteb/msmarco + metrics: + - type: main_score + value: 43.692 + - type: map_at_1 + value: 23.751 + - type: map_at_10 + value: 36.553999999999995 + - type: map_at_100 + value: 37.721 + - type: map_at_1000 + value: 37.763999999999996 + - type: map_at_20 + value: 37.289 + - type: map_at_3 + value: 32.643 + - type: map_at_5 + value: 34.851 + - type: mrr_at_1 + value: 24.455587392550143 + - type: mrr_at_10 + value: 37.18388706963206 + - type: mrr_at_100 + value: 38.28330737932916 + - type: mrr_at_1000 + value: 38.32054399710817 + - type: mrr_at_20 + value: 37.8818001216278 + - type: mrr_at_3 + value: 33.35721107927405 + - type: mrr_at_5 + value: 35.52483285577843 + - type: nauc_map_at_1000_diff1 + value: 36.3576177260684 + - type: nauc_map_at_1000_max + value: 7.854511605962703 + - type: nauc_map_at_1000_std + value: -17.701121059746878 + - type: nauc_map_at_100_diff1 + value: 36.356075649230505 + - type: nauc_map_at_100_max + value: 7.862168042999533 + - type: nauc_map_at_100_std + value: -17.670102459097233 + - type: nauc_map_at_10_diff1 + value: 36.22122978875574 + - type: nauc_map_at_10_max + value: 7.80848606967416 + - type: nauc_map_at_10_std + value: -18.3265151386167 + - type: nauc_map_at_1_diff1 + value: 39.28605466408357 + - type: nauc_map_at_1_max + value: 6.20202977590459 + - type: nauc_map_at_1_std + value: -15.734334090045026 + - type: nauc_map_at_20_diff1 + value: 36.33637880909657 + - type: nauc_map_at_20_max + value: 7.843437969476022 + - type: nauc_map_at_20_std + value: -17.917533363025996 + - type: nauc_map_at_3_diff1 + value: 36.24864976076741 + - type: nauc_map_at_3_max + value: 7.420345251835957 + - type: nauc_map_at_3_std + value: -18.71678497722944 + - type: nauc_map_at_5_diff1 + value: 36.0789619291824 + - type: nauc_map_at_5_max + value: 7.7314285669514495 + - type: nauc_map_at_5_std + value: -18.748688764538706 + - type: nauc_mrr_at_1000_diff1 + value: 36.23912675623378 + - type: nauc_mrr_at_1000_max + value: 7.690553436255147 + - type: nauc_mrr_at_1000_std + value: -17.609526070212304 + - type: nauc_mrr_at_100_diff1 + value: 36.23782651189002 + - type: nauc_mrr_at_100_max + value: 7.70075095171647 + - type: nauc_mrr_at_100_std + value: -17.575714144960184 + - type: nauc_mrr_at_10_diff1 + value: 36.125229472534215 + - type: nauc_mrr_at_10_max + value: 7.635472248755658 + - type: nauc_mrr_at_10_std + value: -18.208166616511086 + - type: nauc_mrr_at_1_diff1 + value: 39.20986875554532 + - type: nauc_mrr_at_1_max + value: 6.062668487561363 + - type: nauc_mrr_at_1_std + value: -16.04130340817602 + - type: nauc_mrr_at_20_diff1 + value: 36.21207088739667 + - type: nauc_mrr_at_20_max + value: 7.699610250145951 + - type: nauc_mrr_at_20_std + value: -17.778245221724028 + - type: nauc_mrr_at_3_diff1 + value: 36.03957583885305 + - type: nauc_mrr_at_3_max + value: 7.225515576504581 + - type: nauc_mrr_at_3_std + value: -18.74478742943741 + - type: nauc_mrr_at_5_diff1 + value: 35.969152496648974 + - type: nauc_mrr_at_5_max + value: 7.584059789018233 + - type: nauc_mrr_at_5_std + value: -18.569374723129332 + - type: nauc_ndcg_at_1000_diff1 + value: 35.894655529841806 + - type: nauc_ndcg_at_1000_max + value: 8.579327424366236 + - type: nauc_ndcg_at_1000_std + value: -16.359677367747896 + - type: nauc_ndcg_at_100_diff1 + value: 35.89861902483983 + - type: nauc_ndcg_at_100_max + value: 8.830873623962242 + - type: nauc_ndcg_at_100_std + value: -15.173125564722978 + - type: nauc_ndcg_at_10_diff1 + value: 35.36499811105169 + - type: nauc_ndcg_at_10_max + value: 8.449267180956992 + - type: nauc_ndcg_at_10_std + value: -18.41978802362402 + - type: nauc_ndcg_at_1_diff1 + value: 39.15422481210622 + - type: nauc_ndcg_at_1_max + value: 6.055515791928331 + - type: nauc_ndcg_at_1_std + value: -16.042779610876252 + - type: nauc_ndcg_at_20_diff1 + value: 35.73402868264468 + - type: nauc_ndcg_at_20_max + value: 8.695705518210847 + - type: nauc_ndcg_at_20_std + value: -16.7735829470466 + - type: nauc_ndcg_at_3_diff1 + value: 35.31358242856231 + - type: nauc_ndcg_at_3_max + value: 7.645692789058997 + - type: nauc_ndcg_at_3_std + value: -19.460003734786874 + - type: nauc_ndcg_at_5_diff1 + value: 35.05216588927143 + - type: nauc_ndcg_at_5_max + value: 8.216690520604715 + - type: nauc_ndcg_at_5_std + value: -19.3982054492159 + - type: nauc_precision_at_1000_diff1 + value: -4.440002625111349 + - type: nauc_precision_at_1000_max + value: 7.886988951901723 + - type: nauc_precision_at_1000_std + value: 9.88111187048247 + - type: nauc_precision_at_100_diff1 + value: 15.728286119463325 + - type: nauc_precision_at_100_max + value: 13.218650824470654 + - type: nauc_precision_at_100_std + value: 16.113245895522553 + - type: nauc_precision_at_10_diff1 + value: 29.51218489610567 + - type: nauc_precision_at_10_max + value: 10.197432401942912 + - type: nauc_precision_at_10_std + value: -16.950603431359493 + - type: nauc_precision_at_1_diff1 + value: 39.15422481210622 + - type: nauc_precision_at_1_max + value: 6.055515791928331 + - type: nauc_precision_at_1_std + value: -16.042779610876252 + - type: nauc_precision_at_20_diff1 + value: 27.825993070397338 + - type: nauc_precision_at_20_max + value: 11.437632287846007 + - type: nauc_precision_at_20_std + value: -7.450353566405601 + - type: nauc_precision_at_3_diff1 + value: 32.14135556796588 + - type: nauc_precision_at_3_max + value: 7.989252443574163 + - type: nauc_precision_at_3_std + value: -21.566254595671055 + - type: nauc_precision_at_5_diff1 + value: 30.68778685307082 + - type: nauc_precision_at_5_max + value: 9.332160758499892 + - type: nauc_precision_at_5_std + value: -20.928554713448914 + - type: nauc_recall_at_1000_diff1 + value: 25.00810478716878 + - type: nauc_recall_at_1000_max + value: 46.518165765201644 + - type: nauc_recall_at_1000_std + value: 61.4734635576085 + - type: nauc_recall_at_100_diff1 + value: 33.895581318261726 + - type: nauc_recall_at_100_max + value: 20.10706035872801 + - type: nauc_recall_at_100_std + value: 24.204226584457047 + - type: nauc_recall_at_10_diff1 + value: 32.363127359576296 + - type: nauc_recall_at_10_max + value: 10.729923804989545 + - type: nauc_recall_at_10_std + value: -18.1335370184202 + - type: nauc_recall_at_1_diff1 + value: 39.28605466408357 + - type: nauc_recall_at_1_max + value: 6.20202977590459 + - type: nauc_recall_at_1_std + value: -15.734334090045026 + - type: nauc_recall_at_20_diff1 + value: 33.47804003169795 + - type: nauc_recall_at_20_max + value: 12.781494765263382 + - type: nauc_recall_at_20_std + value: -9.263970132202658 + - type: nauc_recall_at_3_diff1 + value: 32.71001429428999 + - type: nauc_recall_at_3_max + value: 8.353439197382693 + - type: nauc_recall_at_3_std + value: -21.235097744366954 + - type: nauc_recall_at_5_diff1 + value: 31.87451464963415 + - type: nauc_recall_at_5_max + value: 9.635051450907305 + - type: nauc_recall_at_5_std + value: -21.113235357132794 + - type: ndcg_at_1 + value: 24.47 + - type: ndcg_at_10 + value: 43.692 + - type: ndcg_at_100 + value: 49.211 + - type: ndcg_at_1000 + value: 50.244 + - type: ndcg_at_20 + value: 46.278000000000006 + - type: ndcg_at_3 + value: 35.719 + - type: ndcg_at_5 + value: 39.652 + - type: precision_at_1 + value: 24.47 + - type: precision_at_10 + value: 6.857 + - type: precision_at_100 + value: 0.9610000000000001 + - type: precision_at_1000 + value: 0.105 + - type: precision_at_20 + value: 3.968 + - type: precision_at_3 + value: 15.181000000000001 + - type: precision_at_5 + value: 11.117 + - type: recall_at_1 + value: 23.751 + - type: recall_at_10 + value: 65.64 + - type: recall_at_100 + value: 90.967 + - type: recall_at_1000 + value: 98.738 + - type: recall_at_20 + value: 75.639 + - type: recall_at_3 + value: 43.927 + - type: recall_at_5 + value: 53.366 + task: + type: Retrieval + - dataset: + config: en + name: MTEB MTOPDomainClassification (en) + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + split: test + type: mteb/mtop_domain + metrics: + - type: accuracy + value: 98.82580939352485 + - type: f1 + value: 98.75201754333801 + - type: f1_weighted + value: 98.82795205108245 + - type: main_score + value: 98.82580939352485 + task: + type: Classification + - dataset: + config: en + name: MTEB MTOPIntentClassification (en) + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + split: test + type: mteb/mtop_intent + metrics: + - type: accuracy + value: 92.29822161422709 + - type: f1 + value: 77.75210224871594 + - type: f1_weighted + value: 93.58661422540348 + - type: main_score + value: 92.29822161422709 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveIntentClassification (en) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 85.17484868863484 + - type: f1 + value: 81.94484244487094 + - type: f1_weighted + value: 85.21022593423332 + - type: main_score + value: 85.17484868863484 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveScenarioClassification (en) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 89.61667787491594 + - type: f1 + value: 89.02701927621264 + - type: f1_weighted + value: 89.56306982022801 + - type: main_score + value: 89.61667787491594 + task: + type: Classification + - dataset: + config: default + name: MTEB MedrxivClusteringP2P + revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 + split: test + type: mteb/medrxiv-clustering-p2p + metrics: + - type: main_score + value: 46.318282423948574 + - type: v_measure + value: 46.318282423948574 + - type: v_measure_std + value: 0.9729055662461538 + task: + type: Clustering + - dataset: + config: default + name: MTEB MedrxivClusteringS2S + revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 + split: test + type: mteb/medrxiv-clustering-s2s + metrics: + - type: main_score + value: 44.29033625273981 + - type: v_measure + value: 44.29033625273981 + - type: v_measure_std + value: 1.0596383629128594 + task: + type: Clustering + - dataset: + config: default + name: MTEB MindSmallReranking + revision: 59042f120c80e8afa9cdbb224f67076cec0fc9a7 + split: test + type: mteb/mind_small + metrics: + - type: main_score + value: 33.0526129239962 + - type: map + value: 33.0526129239962 + - type: mrr + value: 34.29260046890935 + - type: nAUC_map_diff1 + value: 12.579738077238032 + - type: nAUC_map_max + value: -20.936629344962 + - type: nAUC_map_std + value: -1.6096805784945216 + - type: nAUC_mrr_diff1 + value: 11.597584463580807 + - type: nAUC_mrr_max + value: -15.723702838537504 + - type: nAUC_mrr_std + value: 0.2719172965777737 + task: + type: Reranking + - dataset: + config: default + name: MTEB NFCorpus + revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 + split: test + type: mteb/nfcorpus + metrics: + - type: main_score + value: 41.486000000000004 + - type: map_at_1 + value: 6.866 + - type: map_at_10 + value: 15.895999999999999 + - type: map_at_100 + value: 21.093 + - type: map_at_1000 + value: 23.067 + - type: map_at_20 + value: 18.125 + - type: map_at_3 + value: 11.421000000000001 + - type: map_at_5 + value: 13.415 + - type: mrr_at_1 + value: 52.63157894736842 + - type: mrr_at_10 + value: 61.486805248415166 + - type: mrr_at_100 + value: 62.08211009182091 + - type: mrr_at_1000 + value: 62.10828701365016 + - type: mrr_at_20 + value: 61.904411187915784 + - type: mrr_at_3 + value: 59.90712074303407 + - type: mrr_at_5 + value: 60.91331269349847 + - type: nauc_map_at_1000_diff1 + value: 25.484625278529403 + - type: nauc_map_at_1000_max + value: 31.206600396418853 + - type: nauc_map_at_1000_std + value: 15.569448072357156 + - type: nauc_map_at_100_diff1 + value: 27.636750226316764 + - type: nauc_map_at_100_max + value: 29.66992681250722 + - type: nauc_map_at_100_std + value: 10.570600484002671 + - type: nauc_map_at_10_diff1 + value: 32.76642525548697 + - type: nauc_map_at_10_max + value: 21.459225397237663 + - type: nauc_map_at_10_std + value: -3.546494734209264 + - type: nauc_map_at_1_diff1 + value: 48.8002894871328 + - type: nauc_map_at_1_max + value: 5.7236722609868815 + - type: nauc_map_at_1_std + value: -13.283554044471352 + - type: nauc_map_at_20_diff1 + value: 30.57169701502308 + - type: nauc_map_at_20_max + value: 25.79666139518404 + - type: nauc_map_at_20_std + value: 1.781732492989651 + - type: nauc_map_at_3_diff1 + value: 40.076315947201095 + - type: nauc_map_at_3_max + value: 12.862524429140054 + - type: nauc_map_at_3_std + value: -9.188349777126817 + - type: nauc_map_at_5_diff1 + value: 36.9918718052938 + - type: nauc_map_at_5_max + value: 16.74234374361876 + - type: nauc_map_at_5_std + value: -7.818523349307494 + - type: nauc_mrr_at_1000_diff1 + value: 26.88183002609805 + - type: nauc_mrr_at_1000_max + value: 47.10209348428658 + - type: nauc_mrr_at_1000_std + value: 32.067825924992924 + - type: nauc_mrr_at_100_diff1 + value: 26.871482491566745 + - type: nauc_mrr_at_100_max + value: 47.11303868498556 + - type: nauc_mrr_at_100_std + value: 32.08961428818868 + - type: nauc_mrr_at_10_diff1 + value: 26.6356914977722 + - type: nauc_mrr_at_10_max + value: 47.091624558810366 + - type: nauc_mrr_at_10_std + value: 31.942424120660164 + - type: nauc_mrr_at_1_diff1 + value: 28.19774198483673 + - type: nauc_mrr_at_1_max + value: 41.44380927834253 + - type: nauc_mrr_at_1_std + value: 25.18222691885917 + - type: nauc_mrr_at_20_diff1 + value: 26.86487347109452 + - type: nauc_mrr_at_20_max + value: 47.1987778214726 + - type: nauc_mrr_at_20_std + value: 32.143517921610034 + - type: nauc_mrr_at_3_diff1 + value: 27.34340373236422 + - type: nauc_mrr_at_3_max + value: 46.358726506276646 + - type: nauc_mrr_at_3_std + value: 31.74924155572593 + - type: nauc_mrr_at_5_diff1 + value: 27.209667205060672 + - type: nauc_mrr_at_5_max + value: 46.79883369072009 + - type: nauc_mrr_at_5_std + value: 31.655605306670758 + - type: nauc_ndcg_at_1000_diff1 + value: 18.940195769769687 + - type: nauc_ndcg_at_1000_max + value: 46.48551313937331 + - type: nauc_ndcg_at_1000_std + value: 33.64819502089232 + - type: nauc_ndcg_at_100_diff1 + value: 19.50885253809146 + - type: nauc_ndcg_at_100_max + value: 40.53174462354878 + - type: nauc_ndcg_at_100_std + value: 28.516152877751118 + - type: nauc_ndcg_at_10_diff1 + value: 16.01699218096564 + - type: nauc_ndcg_at_10_max + value: 41.17322878314514 + - type: nauc_ndcg_at_10_std + value: 29.002233224832196 + - type: nauc_ndcg_at_1_diff1 + value: 27.443547710102205 + - type: nauc_ndcg_at_1_max + value: 40.66529763309582 + - type: nauc_ndcg_at_1_std + value: 24.15016766225869 + - type: nauc_ndcg_at_20_diff1 + value: 17.541197675685062 + - type: nauc_ndcg_at_20_max + value: 40.53231266973844 + - type: nauc_ndcg_at_20_std + value: 29.54096347876548 + - type: nauc_ndcg_at_3_diff1 + value: 18.649628357473716 + - type: nauc_ndcg_at_3_max + value: 41.18603570171764 + - type: nauc_ndcg_at_3_std + value: 27.125524188420396 + - type: nauc_ndcg_at_5_diff1 + value: 17.519593751448483 + - type: nauc_ndcg_at_5_max + value: 42.715997890377345 + - type: nauc_ndcg_at_5_std + value: 27.902627839899868 + - type: nauc_precision_at_1000_diff1 + value: -15.528797630565155 + - type: nauc_precision_at_1000_max + value: 13.741640921778671 + - type: nauc_precision_at_1000_std + value: 44.50896053788372 + - type: nauc_precision_at_100_diff1 + value: -14.491464489721887 + - type: nauc_precision_at_100_max + value: 23.136434418999457 + - type: nauc_precision_at_100_std + value: 49.73145147863128 + - type: nauc_precision_at_10_diff1 + value: -4.829188942994277 + - type: nauc_precision_at_10_max + value: 40.327612559528866 + - type: nauc_precision_at_10_std + value: 39.34919529635044 + - type: nauc_precision_at_1_diff1 + value: 28.19774198483673 + - type: nauc_precision_at_1_max + value: 41.44380927834253 + - type: nauc_precision_at_1_std + value: 25.18222691885917 + - type: nauc_precision_at_20_diff1 + value: -7.210726293112847 + - type: nauc_precision_at_20_max + value: 37.195679576636984 + - type: nauc_precision_at_20_std + value: 45.4597096418357 + - type: nauc_precision_at_3_diff1 + value: 7.578219537774854 + - type: nauc_precision_at_3_max + value: 41.59775233475654 + - type: nauc_precision_at_3_std + value: 30.764584790895118 + - type: nauc_precision_at_5_diff1 + value: 1.655451789039598 + - type: nauc_precision_at_5_max + value: 43.435739407610455 + - type: nauc_precision_at_5_std + value: 33.42552263325999 + - type: nauc_recall_at_1000_diff1 + value: 5.030705700690516 + - type: nauc_recall_at_1000_max + value: 19.108072570815583 + - type: nauc_recall_at_1000_std + value: 14.697734974217308 + - type: nauc_recall_at_100_diff1 + value: 14.746540318132407 + - type: nauc_recall_at_100_max + value: 21.798705033854795 + - type: nauc_recall_at_100_std + value: 11.416195108842587 + - type: nauc_recall_at_10_diff1 + value: 25.548642427860486 + - type: nauc_recall_at_10_max + value: 18.711677681987474 + - type: nauc_recall_at_10_std + value: -5.988904818971677 + - type: nauc_recall_at_1_diff1 + value: 48.8002894871328 + - type: nauc_recall_at_1_max + value: 5.7236722609868815 + - type: nauc_recall_at_1_std + value: -13.283554044471352 + - type: nauc_recall_at_20_diff1 + value: 23.39140739154809 + - type: nauc_recall_at_20_max + value: 19.351150636155474 + - type: nauc_recall_at_20_std + value: -2.757280266915132 + - type: nauc_recall_at_3_diff1 + value: 38.17453576012812 + - type: nauc_recall_at_3_max + value: 13.47003839643972 + - type: nauc_recall_at_3_std + value: -8.75780163862688 + - type: nauc_recall_at_5_diff1 + value: 33.02812855226899 + - type: nauc_recall_at_5_max + value: 15.477626408978477 + - type: nauc_recall_at_5_std + value: -9.072206441070708 + - type: ndcg_at_1 + value: 50.773999999999994 + - type: ndcg_at_10 + value: 41.486000000000004 + - type: ndcg_at_100 + value: 39.051 + - type: ndcg_at_1000 + value: 48.106 + - type: ndcg_at_20 + value: 39.432 + - type: ndcg_at_3 + value: 47.428 + - type: ndcg_at_5 + value: 45.227000000000004 + - type: precision_at_1 + value: 52.632 + - type: precision_at_10 + value: 31.146 + - type: precision_at_100 + value: 10.328 + - type: precision_at_1000 + value: 2.432 + - type: precision_at_20 + value: 23.793 + - type: precision_at_3 + value: 45.201 + - type: precision_at_5 + value: 39.876 + - type: recall_at_1 + value: 6.866 + - type: recall_at_10 + value: 20.447000000000003 + - type: recall_at_100 + value: 40.607 + - type: recall_at_1000 + value: 73.411 + - type: recall_at_20 + value: 26.082 + - type: recall_at_3 + value: 12.484 + - type: recall_at_5 + value: 15.847 + task: + type: Retrieval + - dataset: + config: default + name: MTEB NQ + revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 + split: test + type: mteb/nq + metrics: + - type: main_score + value: 69.072 + - type: map_at_1 + value: 45.483000000000004 + - type: map_at_10 + value: 62.050000000000004 + - type: map_at_100 + value: 62.693 + - type: map_at_1000 + value: 62.702999999999996 + - type: map_at_20 + value: 62.498 + - type: map_at_3 + value: 58.285 + - type: map_at_5 + value: 60.711000000000006 + - type: mrr_at_1 + value: 50.840092699884124 + - type: mrr_at_10 + value: 64.54635224116673 + - type: mrr_at_100 + value: 64.9526548702289 + - type: mrr_at_1000 + value: 64.95908460752281 + - type: mrr_at_20 + value: 64.82949565799959 + - type: mrr_at_3 + value: 61.89165701042856 + - type: mrr_at_5 + value: 63.632676709154026 + - type: nauc_map_at_1000_diff1 + value: 43.187285304185224 + - type: nauc_map_at_1000_max + value: 32.39921659632756 + - type: nauc_map_at_1000_std + value: -5.780901333066553 + - type: nauc_map_at_100_diff1 + value: 43.184487221204456 + - type: nauc_map_at_100_max + value: 32.41176116347982 + - type: nauc_map_at_100_std + value: -5.76422606662383 + - type: nauc_map_at_10_diff1 + value: 42.967066814031746 + - type: nauc_map_at_10_max + value: 32.489617364418514 + - type: nauc_map_at_10_std + value: -6.029045531102664 + - type: nauc_map_at_1_diff1 + value: 46.16376563218624 + - type: nauc_map_at_1_max + value: 26.342624776802232 + - type: nauc_map_at_1_std + value: -7.142171388751972 + - type: nauc_map_at_20_diff1 + value: 43.15894358608328 + - type: nauc_map_at_20_max + value: 32.46492198956245 + - type: nauc_map_at_20_std + value: -5.788373305449195 + - type: nauc_map_at_3_diff1 + value: 43.231752344608545 + - type: nauc_map_at_3_max + value: 31.68003009949564 + - type: nauc_map_at_3_std + value: -8.015235132765458 + - type: nauc_map_at_5_diff1 + value: 42.86197608819917 + - type: nauc_map_at_5_max + value: 32.363857571094485 + - type: nauc_map_at_5_std + value: -6.780487416387977 + - type: nauc_mrr_at_1000_diff1 + value: 43.40542912045782 + - type: nauc_mrr_at_1000_max + value: 32.8461770324533 + - type: nauc_mrr_at_1000_std + value: -3.6505425530008204 + - type: nauc_mrr_at_100_diff1 + value: 43.40233508014468 + - type: nauc_mrr_at_100_max + value: 32.85598538385942 + - type: nauc_mrr_at_100_std + value: -3.637477352635459 + - type: nauc_mrr_at_10_diff1 + value: 43.260179162806054 + - type: nauc_mrr_at_10_max + value: 32.942643527040474 + - type: nauc_mrr_at_10_std + value: -3.712052825320437 + - type: nauc_mrr_at_1_diff1 + value: 46.354919460881206 + - type: nauc_mrr_at_1_max + value: 29.1760258591106 + - type: nauc_mrr_at_1_std + value: -4.107225031227406 + - type: nauc_mrr_at_20_diff1 + value: 43.37092385434311 + - type: nauc_mrr_at_20_max + value: 32.93390254712846 + - type: nauc_mrr_at_20_std + value: -3.5719056112132006 + - type: nauc_mrr_at_3_diff1 + value: 43.1744474040527 + - type: nauc_mrr_at_3_max + value: 32.741290559777994 + - type: nauc_mrr_at_3_std + value: -4.72677925120697 + - type: nauc_mrr_at_5_diff1 + value: 43.108396819975674 + - type: nauc_mrr_at_5_max + value: 32.970519514893084 + - type: nauc_mrr_at_5_std + value: -4.090906158975974 + - type: nauc_ndcg_at_1000_diff1 + value: 42.786664193638714 + - type: nauc_ndcg_at_1000_max + value: 33.65554095609296 + - type: nauc_ndcg_at_1000_std + value: -4.024030130584482 + - type: nauc_ndcg_at_100_diff1 + value: 42.691246775210814 + - type: nauc_ndcg_at_100_max + value: 34.063232335110875 + - type: nauc_ndcg_at_100_std + value: -3.477813807415248 + - type: nauc_ndcg_at_10_diff1 + value: 41.90988990571757 + - type: nauc_ndcg_at_10_max + value: 34.58934812881633 + - type: nauc_ndcg_at_10_std + value: -4.3295110195497655 + - type: nauc_ndcg_at_1_diff1 + value: 46.354919460881206 + - type: nauc_ndcg_at_1_max + value: 29.1760258591106 + - type: nauc_ndcg_at_1_std + value: -4.107225031227406 + - type: nauc_ndcg_at_20_diff1 + value: 42.493206675867114 + - type: nauc_ndcg_at_20_max + value: 34.562441307459544 + - type: nauc_ndcg_at_20_std + value: -3.4456116866749107 + - type: nauc_ndcg_at_3_diff1 + value: 42.24180336502808 + - type: nauc_ndcg_at_3_max + value: 33.064267018100594 + - type: nauc_ndcg_at_3_std + value: -7.786248093572142 + - type: nauc_ndcg_at_5_diff1 + value: 41.692714787779565 + - type: nauc_ndcg_at_5_max + value: 34.20502498949156 + - type: nauc_ndcg_at_5_std + value: -5.979557859282785 + - type: nauc_precision_at_1000_diff1 + value: -13.779832506640702 + - type: nauc_precision_at_1000_max + value: 1.243001688631421 + - type: nauc_precision_at_1000_std + value: 17.351623398622323 + - type: nauc_precision_at_100_diff1 + value: -11.310526816290297 + - type: nauc_precision_at_100_max + value: 5.771669506192959 + - type: nauc_precision_at_100_std + value: 19.917795079540113 + - type: nauc_precision_at_10_diff1 + value: 2.163699384635286 + - type: nauc_precision_at_10_max + value: 19.66440698458386 + - type: nauc_precision_at_10_std + value: 13.689876348315726 + - type: nauc_precision_at_1_diff1 + value: 46.354919460881206 + - type: nauc_precision_at_1_max + value: 29.1760258591106 + - type: nauc_precision_at_1_std + value: -4.107225031227406 + - type: nauc_precision_at_20_diff1 + value: -3.038735879584471 + - type: nauc_precision_at_20_max + value: 14.132968299701695 + - type: nauc_precision_at_20_std + value: 17.78069734664346 + - type: nauc_precision_at_3_diff1 + value: 21.783760758070095 + - type: nauc_precision_at_3_max + value: 30.244127986404497 + - type: nauc_precision_at_3_std + value: -0.12411163467738723 + - type: nauc_precision_at_5_diff1 + value: 10.980635723302418 + - type: nauc_precision_at_5_max + value: 25.302293738975575 + - type: nauc_precision_at_5_std + value: 6.4740817488722024 + - type: nauc_recall_at_1000_diff1 + value: 34.10343772356593 + - type: nauc_recall_at_1000_max + value: 80.72497340357538 + - type: nauc_recall_at_1000_std + value: 69.54564103264093 + - type: nauc_recall_at_100_diff1 + value: 33.427719956774126 + - type: nauc_recall_at_100_max + value: 71.54086768335449 + - type: nauc_recall_at_100_std + value: 49.66157377654885 + - type: nauc_recall_at_10_diff1 + value: 33.70139560054039 + - type: nauc_recall_at_10_max + value: 45.47878072860151 + - type: nauc_recall_at_10_std + value: 1.4188516615716378 + - type: nauc_recall_at_1_diff1 + value: 46.16376563218624 + - type: nauc_recall_at_1_max + value: 26.342624776802232 + - type: nauc_recall_at_1_std + value: -7.142171388751972 + - type: nauc_recall_at_20_diff1 + value: 35.805379874970086 + - type: nauc_recall_at_20_max + value: 51.80479822253392 + - type: nauc_recall_at_20_std + value: 13.531467576460143 + - type: nauc_recall_at_3_diff1 + value: 37.288500141631616 + - type: nauc_recall_at_3_max + value: 35.07078243516728 + - type: nauc_recall_at_3_std + value: -10.452926441410405 + - type: nauc_recall_at_5_diff1 + value: 34.83186104526897 + - type: nauc_recall_at_5_max + value: 39.58488976496973 + - type: nauc_recall_at_5_std + value: -6.3049292065708835 + - type: ndcg_at_1 + value: 50.839999999999996 + - type: ndcg_at_10 + value: 69.072 + - type: ndcg_at_100 + value: 71.538 + - type: ndcg_at_1000 + value: 71.77799999999999 + - type: ndcg_at_20 + value: 70.41 + - type: ndcg_at_3 + value: 62.544999999999995 + - type: ndcg_at_5 + value: 66.33099999999999 + - type: precision_at_1 + value: 50.839999999999996 + - type: precision_at_10 + value: 10.495000000000001 + - type: precision_at_100 + value: 1.1900000000000002 + - type: precision_at_1000 + value: 0.121 + - type: precision_at_20 + value: 5.5809999999999995 + - type: precision_at_3 + value: 27.636 + - type: precision_at_5 + value: 18.864 + - type: recall_at_1 + value: 45.483000000000004 + - type: recall_at_10 + value: 87.483 + - type: recall_at_100 + value: 97.844 + - type: recall_at_1000 + value: 99.66199999999999 + - type: recall_at_20 + value: 92.294 + - type: recall_at_3 + value: 71.2 + - type: recall_at_5 + value: 79.753 + task: + type: Retrieval + - dataset: + config: default + name: MTEB QuoraRetrieval + revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 + split: test + type: mteb/quora + metrics: + - type: main_score + value: 89.58 + - type: map_at_1 + value: 71.819 + - type: map_at_10 + value: 86.04899999999999 + - type: map_at_100 + value: 86.648 + - type: map_at_1000 + value: 86.66199999999999 + - type: map_at_20 + value: 86.441 + - type: map_at_3 + value: 83.114 + - type: map_at_5 + value: 84.981 + - type: mrr_at_1 + value: 82.62 + - type: mrr_at_10 + value: 88.62899999999979 + - type: mrr_at_100 + value: 88.70918591324215 + - type: mrr_at_1000 + value: 88.70973091492397 + - type: mrr_at_20 + value: 88.68914765317221 + - type: mrr_at_3 + value: 87.74999999999979 + - type: mrr_at_5 + value: 88.36799999999974 + - type: nauc_map_at_1000_diff1 + value: 77.89207709760448 + - type: nauc_map_at_1000_max + value: 29.63371361495422 + - type: nauc_map_at_1000_std + value: -48.628180385874344 + - type: nauc_map_at_100_diff1 + value: 77.89592179104915 + - type: nauc_map_at_100_max + value: 29.617171506130756 + - type: nauc_map_at_100_std + value: -48.66057170774648 + - type: nauc_map_at_10_diff1 + value: 78.0618161228185 + - type: nauc_map_at_10_max + value: 29.178490609366737 + - type: nauc_map_at_10_std + value: -50.74755004592002 + - type: nauc_map_at_1_diff1 + value: 81.64335579973574 + - type: nauc_map_at_1_max + value: 21.813832226652174 + - type: nauc_map_at_1_std + value: -42.57570978190876 + - type: nauc_map_at_20_diff1 + value: 77.9299081005938 + - type: nauc_map_at_20_max + value: 29.458718470003888 + - type: nauc_map_at_20_std + value: -49.63337236763102 + - type: nauc_map_at_3_diff1 + value: 78.72941448509229 + - type: nauc_map_at_3_max + value: 26.600997896960056 + - type: nauc_map_at_3_std + value: -51.889002227479885 + - type: nauc_map_at_5_diff1 + value: 78.31466610917171 + - type: nauc_map_at_5_max + value: 28.09863984582896 + - type: nauc_map_at_5_std + value: -52.14058096096497 + - type: nauc_mrr_at_1000_diff1 + value: 78.42667263739992 + - type: nauc_mrr_at_1000_max + value: 31.98996235127974 + - type: nauc_mrr_at_1000_std + value: -44.380439148429296 + - type: nauc_mrr_at_100_diff1 + value: 78.42661032698115 + - type: nauc_mrr_at_100_max + value: 31.991652631740102 + - type: nauc_mrr_at_100_std + value: -44.37854108460535 + - type: nauc_mrr_at_10_diff1 + value: 78.39126022544136 + - type: nauc_mrr_at_10_max + value: 32.02023484451197 + - type: nauc_mrr_at_10_std + value: -44.561252349176954 + - type: nauc_mrr_at_1_diff1 + value: 79.21630894647448 + - type: nauc_mrr_at_1_max + value: 31.526303156060177 + - type: nauc_mrr_at_1_std + value: -41.887504422443136 + - type: nauc_mrr_at_20_diff1 + value: 78.42548039170424 + - type: nauc_mrr_at_20_max + value: 31.99588275070137 + - type: nauc_mrr_at_20_std + value: -44.44957722627042 + - type: nauc_mrr_at_3_diff1 + value: 78.26165151833735 + - type: nauc_mrr_at_3_max + value: 32.18028826126801 + - type: nauc_mrr_at_3_std + value: -44.6998237213182 + - type: nauc_mrr_at_5_diff1 + value: 78.34786430903962 + - type: nauc_mrr_at_5_max + value: 32.168476272879566 + - type: nauc_mrr_at_5_std + value: -44.7915919956712 + - type: nauc_ndcg_at_1000_diff1 + value: 77.79198355957816 + - type: nauc_ndcg_at_1000_max + value: 31.14363511518406 + - type: nauc_ndcg_at_1000_std + value: -46.69335151274275 + - type: nauc_ndcg_at_100_diff1 + value: 77.79898090286419 + - type: nauc_ndcg_at_100_max + value: 31.115103811629215 + - type: nauc_ndcg_at_100_std + value: -46.73078913421965 + - type: nauc_ndcg_at_10_diff1 + value: 77.74856635461343 + - type: nauc_ndcg_at_10_max + value: 30.279584686212747 + - type: nauc_ndcg_at_10_std + value: -50.23514662356807 + - type: nauc_ndcg_at_1_diff1 + value: 79.17833000040999 + - type: nauc_ndcg_at_1_max + value: 31.703788144510746 + - type: nauc_ndcg_at_1_std + value: -41.854817402870715 + - type: nauc_ndcg_at_20_diff1 + value: 77.7380353804671 + - type: nauc_ndcg_at_20_max + value: 30.622294129001553 + - type: nauc_ndcg_at_20_std + value: -49.035794761065254 + - type: nauc_ndcg_at_3_diff1 + value: 77.41476880573593 + - type: nauc_ndcg_at_3_max + value: 29.015949978243032 + - type: nauc_ndcg_at_3_std + value: -49.78627087622648 + - type: nauc_ndcg_at_5_diff1 + value: 77.64439137502896 + - type: nauc_ndcg_at_5_max + value: 29.444684897492206 + - type: nauc_ndcg_at_5_std + value: -51.21908400252501 + - type: nauc_precision_at_1000_diff1 + value: -44.92396459446822 + - type: nauc_precision_at_1000_max + value: -3.674153720989045 + - type: nauc_precision_at_1000_std + value: 39.56552468277785 + - type: nauc_precision_at_100_diff1 + value: -44.75143023259094 + - type: nauc_precision_at_100_max + value: -3.705280025140011 + - type: nauc_precision_at_100_std + value: 39.433619999113326 + - type: nauc_precision_at_10_diff1 + value: -41.0651074726579 + - type: nauc_precision_at_10_max + value: -0.21097985601783667 + - type: nauc_precision_at_10_std + value: 26.24652824589493 + - type: nauc_precision_at_1_diff1 + value: 79.17833000040999 + - type: nauc_precision_at_1_max + value: 31.703788144510746 + - type: nauc_precision_at_1_std + value: -41.854817402870715 + - type: nauc_precision_at_20_diff1 + value: -43.368001340920294 + - type: nauc_precision_at_20_max + value: -2.036990010399129 + - type: nauc_precision_at_20_std + value: 32.37747041406297 + - type: nauc_precision_at_3_diff1 + value: -22.089307548346877 + - type: nauc_precision_at_3_max + value: 6.2280973175296 + - type: nauc_precision_at_3_std + value: 5.323992514036145 + - type: nauc_precision_at_5_diff1 + value: -34.07115055244003 + - type: nauc_precision_at_5_max + value: 2.5955315789198834 + - type: nauc_precision_at_5_std + value: 16.26096689407332 + - type: nauc_recall_at_1000_diff1 + value: 58.27703860947467 + - type: nauc_recall_at_1000_max + value: 68.59835835315768 + - type: nauc_recall_at_1000_std + value: 77.96687006056064 + - type: nauc_recall_at_100_diff1 + value: 73.24371223081737 + - type: nauc_recall_at_100_max + value: 39.55925344664591 + - type: nauc_recall_at_100_std + value: -32.25605030215798 + - type: nauc_recall_at_10_diff1 + value: 73.41261201339202 + - type: nauc_recall_at_10_max + value: 26.822979434062926 + - type: nauc_recall_at_10_std + value: -74.2909332592806 + - type: nauc_recall_at_1_diff1 + value: 81.64335579973574 + - type: nauc_recall_at_1_max + value: 21.813832226652174 + - type: nauc_recall_at_1_std + value: -42.57570978190876 + - type: nauc_recall_at_20_diff1 + value: 72.7621297920656 + - type: nauc_recall_at_20_max + value: 26.02492304096079 + - type: nauc_recall_at_20_std + value: -77.8724532438279 + - type: nauc_recall_at_3_diff1 + value: 75.25149312810714 + - type: nauc_recall_at_3_max + value: 23.20545662481487 + - type: nauc_recall_at_3_std + value: -59.69689982140521 + - type: nauc_recall_at_5_diff1 + value: 73.69807273001406 + - type: nauc_recall_at_5_max + value: 24.073666798066057 + - type: nauc_recall_at_5_std + value: -67.91121268130719 + - type: ndcg_at_1 + value: 82.64 + - type: ndcg_at_10 + value: 89.58 + - type: ndcg_at_100 + value: 90.606 + - type: ndcg_at_1000 + value: 90.676 + - type: ndcg_at_20 + value: 90.132 + - type: ndcg_at_3 + value: 86.88 + - type: ndcg_at_5 + value: 88.40299999999999 + - type: precision_at_1 + value: 82.64 + - type: precision_at_10 + value: 13.604 + - type: precision_at_100 + value: 1.539 + - type: precision_at_1000 + value: 0.157 + - type: precision_at_20 + value: 7.188 + - type: precision_at_3 + value: 38.083 + - type: precision_at_5 + value: 25.018 + - type: recall_at_1 + value: 71.819 + - type: recall_at_10 + value: 96.34700000000001 + - type: recall_at_100 + value: 99.715 + - type: recall_at_1000 + value: 99.995 + - type: recall_at_20 + value: 98.073 + - type: recall_at_3 + value: 88.57300000000001 + - type: recall_at_5 + value: 92.908 + task: + type: Retrieval + - dataset: + config: default + name: MTEB RedditClustering + revision: 24640382cdbf8abc73003fb0fa6d111a705499eb + split: test + type: mteb/reddit-clustering + metrics: + - type: main_score + value: 71.18966762070158 + - type: v_measure + value: 71.18966762070158 + - type: v_measure_std + value: 2.7498969054457048 + task: + type: Clustering + - dataset: + config: default + name: MTEB RedditClusteringP2P + revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 + split: test + type: mteb/reddit-clustering-p2p + metrics: + - type: main_score + value: 74.42014716862516 + - type: v_measure + value: 74.42014716862516 + - type: v_measure_std + value: 9.909739891410648 + task: + type: Clustering + - dataset: + config: default + name: MTEB SCIDOCS + revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 + split: test + type: mteb/scidocs + metrics: + - type: main_score + value: 25.041999999999998 + - type: map_at_1 + value: 5.893000000000001 + - type: map_at_10 + value: 15.260000000000002 + - type: map_at_100 + value: 18.084 + - type: map_at_1000 + value: 18.467 + - type: map_at_20 + value: 16.675 + - type: map_at_3 + value: 10.526 + - type: map_at_5 + value: 12.775 + - type: mrr_at_1 + value: 28.999999999999996 + - type: mrr_at_10 + value: 41.03575396825395 + - type: mrr_at_100 + value: 42.136771862785835 + - type: mrr_at_1000 + value: 42.16698555415099 + - type: mrr_at_20 + value: 41.707493696104315 + - type: mrr_at_3 + value: 37.34999999999998 + - type: mrr_at_5 + value: 39.59999999999995 + - type: nauc_map_at_1000_diff1 + value: 12.080002654911883 + - type: nauc_map_at_1000_max + value: 29.813563682286276 + - type: nauc_map_at_1000_std + value: 20.36659817908673 + - type: nauc_map_at_100_diff1 + value: 12.108735517749706 + - type: nauc_map_at_100_max + value: 29.76830671710955 + - type: nauc_map_at_100_std + value: 20.3433621032846 + - type: nauc_map_at_10_diff1 + value: 12.91575031185637 + - type: nauc_map_at_10_max + value: 29.427600958386318 + - type: nauc_map_at_10_std + value: 16.89867275177153 + - type: nauc_map_at_1_diff1 + value: 19.353069488987916 + - type: nauc_map_at_1_max + value: 17.093914951159693 + - type: nauc_map_at_1_std + value: 8.19886078055046 + - type: nauc_map_at_20_diff1 + value: 11.977233457943113 + - type: nauc_map_at_20_max + value: 29.171812822948805 + - type: nauc_map_at_20_std + value: 18.780517506173965 + - type: nauc_map_at_3_diff1 + value: 14.453129464176092 + - type: nauc_map_at_3_max + value: 25.801958649112077 + - type: nauc_map_at_3_std + value: 11.572823684429643 + - type: nauc_map_at_5_diff1 + value: 13.167155808104997 + - type: nauc_map_at_5_max + value: 27.355626948365792 + - type: nauc_map_at_5_std + value: 14.414151839192183 + - type: nauc_mrr_at_1000_diff1 + value: 17.262104643988636 + - type: nauc_mrr_at_1000_max + value: 23.991373837217058 + - type: nauc_mrr_at_1000_std + value: 12.44755488671623 + - type: nauc_mrr_at_100_diff1 + value: 17.267280132318703 + - type: nauc_mrr_at_100_max + value: 24.022189287889294 + - type: nauc_mrr_at_100_std + value: 12.480695500214788 + - type: nauc_mrr_at_10_diff1 + value: 17.012383998246268 + - type: nauc_mrr_at_10_max + value: 24.192637911171722 + - type: nauc_mrr_at_10_std + value: 12.524608847408917 + - type: nauc_mrr_at_1_diff1 + value: 19.43518811038007 + - type: nauc_mrr_at_1_max + value: 17.747482933395602 + - type: nauc_mrr_at_1_std + value: 8.410779775558684 + - type: nauc_mrr_at_20_diff1 + value: 17.202663281407446 + - type: nauc_mrr_at_20_max + value: 24.091991130543118 + - type: nauc_mrr_at_20_std + value: 12.503814263019908 + - type: nauc_mrr_at_3_diff1 + value: 17.52733013432995 + - type: nauc_mrr_at_3_max + value: 23.569459518780214 + - type: nauc_mrr_at_3_std + value: 11.770846827520726 + - type: nauc_mrr_at_5_diff1 + value: 17.10817561975543 + - type: nauc_mrr_at_5_max + value: 23.945141435234678 + - type: nauc_mrr_at_5_std + value: 12.034468615317719 + - type: nauc_ndcg_at_1000_diff1 + value: 12.317811393346936 + - type: nauc_ndcg_at_1000_max + value: 30.809991350156103 + - type: nauc_ndcg_at_1000_std + value: 24.517501065205067 + - type: nauc_ndcg_at_100_diff1 + value: 12.824804203182936 + - type: nauc_ndcg_at_100_max + value: 30.895499817010748 + - type: nauc_ndcg_at_100_std + value: 25.424376279745402 + - type: nauc_ndcg_at_10_diff1 + value: 13.32724552457439 + - type: nauc_ndcg_at_10_max + value: 30.409088666807456 + - type: nauc_ndcg_at_10_std + value: 18.216330475714113 + - type: nauc_ndcg_at_1_diff1 + value: 19.43518811038007 + - type: nauc_ndcg_at_1_max + value: 17.747482933395602 + - type: nauc_ndcg_at_1_std + value: 8.410779775558684 + - type: nauc_ndcg_at_20_diff1 + value: 12.224399111852902 + - type: nauc_ndcg_at_20_max + value: 29.86352330445272 + - type: nauc_ndcg_at_20_std + value: 21.196937851331807 + - type: nauc_ndcg_at_3_diff1 + value: 15.367489533734027 + - type: nauc_ndcg_at_3_max + value: 26.76486390741532 + - type: nauc_ndcg_at_3_std + value: 12.606077508789923 + - type: nauc_ndcg_at_5_diff1 + value: 13.831157482390935 + - type: nauc_ndcg_at_5_max + value: 28.070226983968904 + - type: nauc_ndcg_at_5_std + value: 15.236787943125435 + - type: nauc_precision_at_1000_diff1 + value: 0.016122957101357048 + - type: nauc_precision_at_1000_max + value: 24.380929903557334 + - type: nauc_precision_at_1000_std + value: 34.54045112720052 + - type: nauc_precision_at_100_diff1 + value: 7.255224788507301 + - type: nauc_precision_at_100_max + value: 27.98453788447542 + - type: nauc_precision_at_100_std + value: 35.38999555441665 + - type: nauc_precision_at_10_diff1 + value: 9.69185099834181 + - type: nauc_precision_at_10_max + value: 32.532315522580454 + - type: nauc_precision_at_10_std + value: 21.48948348473612 + - type: nauc_precision_at_1_diff1 + value: 19.43518811038007 + - type: nauc_precision_at_1_max + value: 17.747482933395602 + - type: nauc_precision_at_1_std + value: 8.410779775558684 + - type: nauc_precision_at_20_diff1 + value: 6.964076536695672 + - type: nauc_precision_at_20_max + value: 29.30087236410044 + - type: nauc_precision_at_20_std + value: 26.413625895571986 + - type: nauc_precision_at_3_diff1 + value: 14.145134359925155 + - type: nauc_precision_at_3_max + value: 29.915650960808303 + - type: nauc_precision_at_3_std + value: 14.095370019867797 + - type: nauc_precision_at_5_diff1 + value: 11.043933558522692 + - type: nauc_precision_at_5_max + value: 30.93016505807111 + - type: nauc_precision_at_5_std + value: 17.749256196062603 + - type: nauc_recall_at_1000_diff1 + value: -0.7776817772090345 + - type: nauc_recall_at_1000_max + value: 23.094717340324518 + - type: nauc_recall_at_1000_std + value: 37.189908681396425 + - type: nauc_recall_at_100_diff1 + value: 6.887748742013364 + - type: nauc_recall_at_100_max + value: 27.00798435230277 + - type: nauc_recall_at_100_std + value: 35.908147807345344 + - type: nauc_recall_at_10_diff1 + value: 9.605632017480751 + - type: nauc_recall_at_10_max + value: 31.845202901168655 + - type: nauc_recall_at_10_std + value: 21.497414586634683 + - type: nauc_recall_at_1_diff1 + value: 19.353069488987916 + - type: nauc_recall_at_1_max + value: 17.093914951159693 + - type: nauc_recall_at_1_std + value: 8.19886078055046 + - type: nauc_recall_at_20_diff1 + value: 6.927503731844782 + - type: nauc_recall_at_20_max + value: 28.611698183338202 + - type: nauc_recall_at_20_std + value: 26.69018660149911 + - type: nauc_recall_at_3_diff1 + value: 14.043724087062268 + - type: nauc_recall_at_3_max + value: 29.269835821380465 + - type: nauc_recall_at_3_std + value: 14.104419605998094 + - type: nauc_recall_at_5_diff1 + value: 11.017319452873336 + - type: nauc_recall_at_5_max + value: 30.295720628306228 + - type: nauc_recall_at_5_std + value: 17.758048545573825 + - type: ndcg_at_1 + value: 28.999999999999996 + - type: ndcg_at_10 + value: 25.041999999999998 + - type: ndcg_at_100 + value: 35.045 + - type: ndcg_at_1000 + value: 40.803 + - type: ndcg_at_20 + value: 28.584 + - type: ndcg_at_3 + value: 23.249 + - type: ndcg_at_5 + value: 20.533 + - type: precision_at_1 + value: 28.999999999999996 + - type: precision_at_10 + value: 13.120000000000001 + - type: precision_at_100 + value: 2.7470000000000003 + - type: precision_at_1000 + value: 0.41200000000000003 + - type: precision_at_20 + value: 8.584999999999999 + - type: precision_at_3 + value: 21.633 + - type: precision_at_5 + value: 18.099999999999998 + - type: recall_at_1 + value: 5.893000000000001 + - type: recall_at_10 + value: 26.567 + - type: recall_at_100 + value: 55.800000000000004 + - type: recall_at_1000 + value: 83.608 + - type: recall_at_20 + value: 34.86 + - type: recall_at_3 + value: 13.153 + - type: recall_at_5 + value: 18.323 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SICK-R + revision: 20a6d6f312dd54037fe07a32d58e5e168867909d + split: test + type: mteb/sickr-sts + metrics: + - type: cosine_pearson + value: 86.57284584320382 + - type: cosine_spearman + value: 82.20531642680812 + - type: euclidean_pearson + value: 83.94261758556554 + - type: euclidean_spearman + value: 82.20721497738559 + - type: main_score + value: 82.20531642680812 + - type: manhattan_pearson + value: 84.15902154703083 + - type: manhattan_spearman + value: 82.19506027155957 + - type: pearson + value: 86.57284584320382 + - type: spearman + value: 82.20531642680812 + task: + type: STS + - dataset: + config: default + name: MTEB STS12 + revision: a0d554a64d88156834ff5ae9920b964011b16384 + split: test + type: mteb/sts12-sts + metrics: + - type: cosine_pearson + value: 86.28047602146931 + - type: cosine_spearman + value: 79.51504881448884 + - type: euclidean_pearson + value: 83.10545189967856 + - type: euclidean_spearman + value: 79.50586960492797 + - type: main_score + value: 79.51504881448884 + - type: manhattan_pearson + value: 83.44244457500889 + - type: manhattan_spearman + value: 79.730303339846 + - type: pearson + value: 86.28047602146931 + - type: spearman + value: 79.51504881448884 + task: + type: STS + - dataset: + config: default + name: MTEB STS13 + revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca + split: test + type: mteb/sts13-sts + metrics: + - type: cosine_pearson + value: 88.74723553048702 + - type: cosine_spearman + value: 89.18936052329725 + - type: euclidean_pearson + value: 88.90400878928668 + - type: euclidean_spearman + value: 89.19174821431281 + - type: main_score + value: 89.18936052329725 + - type: manhattan_pearson + value: 88.81504628424054 + - type: manhattan_spearman + value: 89.18063294142597 + - type: pearson + value: 88.74723553048702 + - type: spearman + value: 89.18936052329725 + task: + type: STS + - dataset: + config: default + name: MTEB STS14 + revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 + split: test + type: mteb/sts14-sts + metrics: + - type: cosine_pearson + value: 86.45403437836023 + - type: cosine_spearman + value: 85.14654611519086 + - type: euclidean_pearson + value: 85.87509624462743 + - type: euclidean_spearman + value: 85.1391108856681 + - type: main_score + value: 85.14654611519086 + - type: manhattan_pearson + value: 85.96635794953866 + - type: manhattan_spearman + value: 85.3271371527667 + - type: pearson + value: 86.45403437836023 + - type: spearman + value: 85.14654611519086 + task: + type: STS + - dataset: + config: default + name: MTEB STS15 + revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 + split: test + type: mteb/sts15-sts + metrics: + - type: cosine_pearson + value: 87.84742260009705 + - type: cosine_spearman + value: 89.10215217191254 + - type: euclidean_pearson + value: 88.97393286325477 + - type: euclidean_spearman + value: 89.1014105509662 + - type: main_score + value: 89.10215217191254 + - type: manhattan_pearson + value: 89.31698781090151 + - type: manhattan_spearman + value: 89.53000001764433 + - type: pearson + value: 87.84742260009705 + - type: spearman + value: 89.10215217191254 + task: + type: STS + - dataset: + config: default + name: MTEB STS16 + revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 + split: test + type: mteb/sts16-sts + metrics: + - type: cosine_pearson + value: 85.22397535461835 + - type: cosine_spearman + value: 87.14066355879785 + - type: euclidean_pearson + value: 86.31393364087295 + - type: euclidean_spearman + value: 87.14018892702765 + - type: main_score + value: 87.14066355879785 + - type: manhattan_pearson + value: 86.36366855248434 + - type: manhattan_spearman + value: 87.20858630423012 + - type: pearson + value: 85.22397535461835 + - type: spearman + value: 87.14066355879785 + task: + type: STS + - dataset: + config: en-en + name: MTEB STS17 (en-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: cosine_pearson + value: 90.66131612061355 + - type: cosine_spearman + value: 90.97082650129164 + - type: euclidean_pearson + value: 90.98181906744969 + - type: euclidean_spearman + value: 90.99008476850047 + - type: main_score + value: 90.97082650129164 + - type: manhattan_pearson + value: 90.75245040709021 + - type: manhattan_spearman + value: 90.6199877691265 + - type: pearson + value: 90.66131612061355 + - type: spearman + value: 90.97082650129164 + task: + type: STS + - dataset: + config: en + name: MTEB STS22 (en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: cosine_pearson + value: 67.270656447085 + - type: cosine_spearman + value: 67.82870469746828 + - type: euclidean_pearson + value: 69.03857775285664 + - type: euclidean_spearman + value: 67.74455108773341 + - type: main_score + value: 67.82870469746828 + - type: manhattan_pearson + value: 69.25304172245812 + - type: manhattan_spearman + value: 68.00987097916055 + - type: pearson + value: 67.270656447085 + - type: spearman + value: 67.82870469746828 + task: + type: STS + - dataset: + config: default + name: MTEB STSBenchmark + revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 + split: test + type: mteb/stsbenchmark-sts + metrics: + - type: cosine_pearson + value: 87.17245205384889 + - type: cosine_spearman + value: 87.7360146030987 + - type: euclidean_pearson + value: 87.48919412794656 + - type: euclidean_spearman + value: 87.7312047878383 + - type: main_score + value: 87.7360146030987 + - type: manhattan_pearson + value: 87.61476224354806 + - type: manhattan_spearman + value: 87.95220889254693 + - type: pearson + value: 87.17245205384889 + - type: spearman + value: 87.7360146030987 + task: + type: STS + - dataset: + config: default + name: MTEB SciDocsRR + revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab + split: test + type: mteb/scidocs-reranking + metrics: + - type: main_score + value: 88.43547871921146 + - type: map + value: 88.43547871921146 + - type: mrr + value: 96.5564473652709 + - type: nAUC_map_diff1 + value: -13.66029392579231 + - type: nAUC_map_max + value: 50.325613574053506 + - type: nAUC_map_std + value: 60.02986231275796 + - type: nAUC_mrr_diff1 + value: 23.83821476411125 + - type: nAUC_mrr_max + value: 86.72643311769906 + - type: nAUC_mrr_std + value: 72.12741063469213 + task: + type: Reranking + - dataset: + config: default + name: MTEB SciFact + revision: 0228b52cf27578f30900b9e5271d331663a030d7 + split: test + type: mteb/scifact + metrics: + - type: main_score + value: 78.233 + - type: map_at_1 + value: 61.49400000000001 + - type: map_at_10 + value: 73.30600000000001 + - type: map_at_100 + value: 73.719 + - type: map_at_1000 + value: 73.724 + - type: map_at_20 + value: 73.611 + - type: map_at_3 + value: 70.626 + - type: map_at_5 + value: 72.417 + - type: mrr_at_1 + value: 64.66666666666666 + - type: mrr_at_10 + value: 74.30357142857143 + - type: mrr_at_100 + value: 74.56950898079988 + - type: mrr_at_1000 + value: 74.57295833098681 + - type: mrr_at_20 + value: 74.46165223665226 + - type: mrr_at_3 + value: 72.3888888888889 + - type: mrr_at_5 + value: 73.60555555555557 + - type: nauc_map_at_1000_diff1 + value: 76.51524604780636 + - type: nauc_map_at_1000_max + value: 53.48521938401881 + - type: nauc_map_at_1000_std + value: -7.347799382158861 + - type: nauc_map_at_100_diff1 + value: 76.5122888096236 + - type: nauc_map_at_100_max + value: 53.49221847471618 + - type: nauc_map_at_100_std + value: -7.329683735681086 + - type: nauc_map_at_10_diff1 + value: 76.30928630674504 + - type: nauc_map_at_10_max + value: 53.00102977185941 + - type: nauc_map_at_10_std + value: -7.7467740085108705 + - type: nauc_map_at_1_diff1 + value: 79.54189281784247 + - type: nauc_map_at_1_max + value: 46.630071622109526 + - type: nauc_map_at_1_std + value: -14.395943134644112 + - type: nauc_map_at_20_diff1 + value: 76.41604361947962 + - type: nauc_map_at_20_max + value: 53.578883876146875 + - type: nauc_map_at_20_std + value: -7.403103451288041 + - type: nauc_map_at_3_diff1 + value: 76.25911617571941 + - type: nauc_map_at_3_max + value: 49.140287380513605 + - type: nauc_map_at_3_std + value: -11.35992449218983 + - type: nauc_map_at_5_diff1 + value: 76.35122077770336 + - type: nauc_map_at_5_max + value: 52.1744367901208 + - type: nauc_map_at_5_std + value: -7.85753955055384 + - type: nauc_mrr_at_1000_diff1 + value: 76.97223309515867 + - type: nauc_mrr_at_1000_max + value: 57.263787498613326 + - type: nauc_mrr_at_1000_std + value: -4.884090708840035 + - type: nauc_mrr_at_100_diff1 + value: 76.97312970894603 + - type: nauc_mrr_at_100_max + value: 57.26850730446478 + - type: nauc_mrr_at_100_std + value: -4.875200894216617 + - type: nauc_mrr_at_10_diff1 + value: 76.65927674223613 + - type: nauc_mrr_at_10_max + value: 57.30979763941454 + - type: nauc_mrr_at_10_std + value: -4.863331094022142 + - type: nauc_mrr_at_1_diff1 + value: 80.0454932568644 + - type: nauc_mrr_at_1_max + value: 56.76038421319305 + - type: nauc_mrr_at_1_std + value: -4.101939392632653 + - type: nauc_mrr_at_20_diff1 + value: 76.87237970440503 + - type: nauc_mrr_at_20_max + value: 57.33843605225869 + - type: nauc_mrr_at_20_std + value: -4.96248984417978 + - type: nauc_mrr_at_3_diff1 + value: 76.74130186666727 + - type: nauc_mrr_at_3_max + value: 56.19313244846155 + - type: nauc_mrr_at_3_std + value: -5.684365934009136 + - type: nauc_mrr_at_5_diff1 + value: 76.66406918799962 + - type: nauc_mrr_at_5_max + value: 57.56110093228628 + - type: nauc_mrr_at_5_std + value: -3.7464413085588073 + - type: nauc_ndcg_at_1000_diff1 + value: 76.19194173971773 + - type: nauc_ndcg_at_1000_max + value: 55.57464600170693 + - type: nauc_ndcg_at_1000_std + value: -6.0761689532372625 + - type: nauc_ndcg_at_100_diff1 + value: 76.14631273843654 + - type: nauc_ndcg_at_100_max + value: 55.72246565373382 + - type: nauc_ndcg_at_100_std + value: -5.595160698860595 + - type: nauc_ndcg_at_10_diff1 + value: 75.0108223611192 + - type: nauc_ndcg_at_10_max + value: 55.27894212877493 + - type: nauc_ndcg_at_10_std + value: -6.968331740214591 + - type: nauc_ndcg_at_1_diff1 + value: 80.0454932568644 + - type: nauc_ndcg_at_1_max + value: 56.76038421319305 + - type: nauc_ndcg_at_1_std + value: -4.101939392632653 + - type: nauc_ndcg_at_20_diff1 + value: 75.54887755702472 + - type: nauc_ndcg_at_20_max + value: 56.406879417251496 + - type: nauc_ndcg_at_20_std + value: -6.495231061329629 + - type: nauc_ndcg_at_3_diff1 + value: 75.03620356688509 + - type: nauc_ndcg_at_3_max + value: 52.147381077773424 + - type: nauc_ndcg_at_3_std + value: -8.448005688956199 + - type: nauc_ndcg_at_5_diff1 + value: 75.1195898074229 + - type: nauc_ndcg_at_5_max + value: 54.2321033861173 + - type: nauc_ndcg_at_5_std + value: -5.882690780895338 + - type: nauc_precision_at_1000_diff1 + value: -28.081979732100532 + - type: nauc_precision_at_1000_max + value: 35.055348014832916 + - type: nauc_precision_at_1000_std + value: 59.61280468927384 + - type: nauc_precision_at_100_diff1 + value: -25.112740730587458 + - type: nauc_precision_at_100_max + value: 38.26331300116496 + - type: nauc_precision_at_100_std + value: 62.46316222328831 + - type: nauc_precision_at_10_diff1 + value: -2.6766206473658833 + - type: nauc_precision_at_10_max + value: 45.95321867204845 + - type: nauc_precision_at_10_std + value: 45.07212468670564 + - type: nauc_precision_at_1_diff1 + value: 80.0454932568644 + - type: nauc_precision_at_1_max + value: 56.76038421319305 + - type: nauc_precision_at_1_std + value: -4.101939392632653 + - type: nauc_precision_at_20_diff1 + value: -10.698911116738385 + - type: nauc_precision_at_20_max + value: 43.467275950182994 + - type: nauc_precision_at_20_std + value: 48.00467321991766 + - type: nauc_precision_at_3_diff1 + value: 33.6344708541193 + - type: nauc_precision_at_3_max + value: 49.309242331670504 + - type: nauc_precision_at_3_std + value: 21.02940391379915 + - type: nauc_precision_at_5_diff1 + value: 13.560415600596318 + - type: nauc_precision_at_5_max + value: 48.918726500100085 + - type: nauc_precision_at_5_std + value: 39.940930429172184 + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_100_diff1 + value: 70.82166199813196 + - type: nauc_recall_at_100_max + value: 76.6106442577042 + - type: nauc_recall_at_100_std + value: 66.47992530345513 + - type: nauc_recall_at_10_diff1 + value: 62.68908885556092 + - type: nauc_recall_at_10_max + value: 58.14262437741839 + - type: nauc_recall_at_10_std + value: -12.946717875063369 + - type: nauc_recall_at_1_diff1 + value: 79.54189281784247 + - type: nauc_recall_at_1_max + value: 46.630071622109526 + - type: nauc_recall_at_1_std + value: -14.395943134644112 + - type: nauc_recall_at_20_diff1 + value: 65.79470497876567 + - type: nauc_recall_at_20_max + value: 71.68308183488456 + - type: nauc_recall_at_20_std + value: -12.556850697268453 + - type: nauc_recall_at_3_diff1 + value: 68.3240211318129 + - type: nauc_recall_at_3_max + value: 45.05998217275036 + - type: nauc_recall_at_3_std + value: -14.23179772593869 + - type: nauc_recall_at_5_diff1 + value: 67.53366869904056 + - type: nauc_recall_at_5_max + value: 53.57935627081027 + - type: nauc_recall_at_5_std + value: -3.3271112904853393 + - type: ndcg_at_1 + value: 64.667 + - type: ndcg_at_10 + value: 78.233 + - type: ndcg_at_100 + value: 79.806 + - type: ndcg_at_1000 + value: 79.92099999999999 + - type: ndcg_at_20 + value: 79.006 + - type: ndcg_at_3 + value: 74.018 + - type: ndcg_at_5 + value: 76.334 + - type: precision_at_1 + value: 64.667 + - type: precision_at_10 + value: 10.4 + - type: precision_at_100 + value: 1.1199999999999999 + - type: precision_at_1000 + value: 0.11299999999999999 + - type: precision_at_20 + value: 5.383 + - type: precision_at_3 + value: 29.444 + - type: precision_at_5 + value: 19.467000000000002 + - type: recall_at_1 + value: 61.49400000000001 + - type: recall_at_10 + value: 92.156 + - type: recall_at_100 + value: 99.167 + - type: recall_at_1000 + value: 100.0 + - type: recall_at_20 + value: 94.833 + - type: recall_at_3 + value: 80.833 + - type: recall_at_5 + value: 86.6 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SprintDuplicateQuestions + revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 + split: test + type: mteb/sprintduplicatequestions-pairclassification + metrics: + - type: cosine_accuracy + value: 99.8039603960396 + - type: cosine_accuracy_threshold + value: 84.54211950302124 + - type: cosine_ap + value: 95.59056372734358 + - type: cosine_f1 + value: 90.1394422310757 + - type: cosine_f1_threshold + value: 84.54211950302124 + - type: cosine_precision + value: 89.78174603174604 + - type: cosine_recall + value: 90.5 + - type: dot_accuracy + value: 99.80594059405941 + - type: dot_accuracy_threshold + value: 85.57180166244507 + - type: dot_ap + value: 95.53453431914399 + - type: dot_f1 + value: 90.10442565887618 + - type: dot_f1_threshold + value: 84.59715843200684 + - type: dot_precision + value: 89.61424332344214 + - type: dot_recall + value: 90.60000000000001 + - type: euclidean_accuracy + value: 99.8039603960396 + - type: euclidean_accuracy_threshold + value: 53.253382444381714 + - type: euclidean_ap + value: 95.5850992402159 + - type: euclidean_f1 + value: 90.09457441513192 + - type: euclidean_f1_threshold + value: 55.725520849227905 + - type: euclidean_precision + value: 89.69276511397423 + - type: euclidean_recall + value: 90.5 + - type: main_score + value: 95.7485189884476 + - type: manhattan_accuracy + value: 99.81485148514851 + - type: manhattan_accuracy_threshold + value: 3491.29638671875 + - type: manhattan_ap + value: 95.7485189884476 + - type: manhattan_f1 + value: 90.464048954615 + - type: manhattan_f1_threshold + value: 3491.29638671875 + - type: manhattan_precision + value: 92.2996878251821 + - type: manhattan_recall + value: 88.7 + - type: max_ap + value: 95.7485189884476 + - type: max_f1 + value: 90.464048954615 + - type: max_precision + value: 92.2996878251821 + - type: max_recall + value: 90.60000000000001 + - type: similarity_accuracy + value: 99.8039603960396 + - type: similarity_accuracy_threshold + value: 84.54211950302124 + - type: similarity_ap + value: 95.59056372734358 + - type: similarity_f1 + value: 90.1394422310757 + - type: similarity_f1_threshold + value: 84.54211950302124 + - type: similarity_precision + value: 89.78174603174604 + - type: similarity_recall + value: 90.5 + task: + type: PairClassification + - dataset: + config: default + name: MTEB StackExchangeClustering + revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 + split: test + type: mteb/stackexchange-clustering + metrics: + - type: main_score + value: 78.49205191950675 + - type: v_measure + value: 78.49205191950675 + - type: v_measure_std + value: 2.84869550699959 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackExchangeClusteringP2P + revision: 815ca46b2622cec33ccafc3735d572c266efdb44 + split: test + type: mteb/stackexchange-clustering-p2p + metrics: + - type: main_score + value: 48.90421736513028 + - type: v_measure + value: 48.90421736513028 + - type: v_measure_std + value: 1.6875865714471023 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackOverflowDupQuestions + revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 + split: test + type: mteb/stackoverflowdupquestions-reranking + metrics: + - type: main_score + value: 52.9874730481696 + - type: map + value: 52.9874730481696 + - type: mrr + value: 53.85867604617604 + - type: nAUC_map_diff1 + value: 39.633429293407616 + - type: nAUC_map_max + value: 10.236807988858546 + - type: nAUC_map_std + value: 10.276522217929674 + - type: nAUC_mrr_diff1 + value: 40.0543079218377 + - type: nAUC_mrr_max + value: 10.96209807382042 + - type: nAUC_mrr_std + value: 10.524400196109918 + task: + type: Reranking + - dataset: + config: default + name: MTEB SummEval + revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c + split: test + type: mteb/summeval + metrics: + - type: cosine_pearson + value: 30.727801109114232 + - type: cosine_spearman + value: 31.66058223980157 + - type: dot_pearson + value: 30.78818248622866 + - type: dot_spearman + value: 31.525158776890265 + - type: main_score + value: 31.66058223980157 + - type: pearson + value: 30.727801109114232 + - type: spearman + value: 31.66058223980157 + task: + type: Summarization + - dataset: + config: default + name: MTEB TRECCOVID + revision: bb9466bac8153a0349341eb1b22e06409e78ef4e + split: test + type: mteb/trec-covid + metrics: + - type: main_score + value: 85.206 + - type: map_at_1 + value: 0.246 + - type: map_at_10 + value: 2.1950000000000003 + - type: map_at_100 + value: 14.179 + - type: map_at_1000 + value: 35.037 + - type: map_at_20 + value: 4.143 + - type: map_at_3 + value: 0.7100000000000001 + - type: map_at_5 + value: 1.135 + - type: mrr_at_1 + value: 94.0 + - type: mrr_at_10 + value: 96.66666666666666 + - type: mrr_at_100 + value: 96.66666666666666 + - type: mrr_at_1000 + value: 96.66666666666666 + - type: mrr_at_20 + value: 96.66666666666666 + - type: mrr_at_3 + value: 96.66666666666666 + - type: mrr_at_5 + value: 96.66666666666666 + - type: nauc_map_at_1000_diff1 + value: -4.6264497624527525 + - type: nauc_map_at_1000_max + value: 44.594457564749355 + - type: nauc_map_at_1000_std + value: 73.17642341400133 + - type: nauc_map_at_100_diff1 + value: 23.451335157405726 + - type: nauc_map_at_100_max + value: 25.426398857299525 + - type: nauc_map_at_100_std + value: 64.07416694472633 + - type: nauc_map_at_10_diff1 + value: 46.57568738568346 + - type: nauc_map_at_10_max + value: 9.693233249079238 + - type: nauc_map_at_10_std + value: 28.549530265164357 + - type: nauc_map_at_1_diff1 + value: 53.48238396620123 + - type: nauc_map_at_1_max + value: 0.33476619393733076 + - type: nauc_map_at_1_std + value: 8.906362219128463 + - type: nauc_map_at_20_diff1 + value: 39.40719602207749 + - type: nauc_map_at_20_max + value: 9.635915072074045 + - type: nauc_map_at_20_std + value: 35.15634791346394 + - type: nauc_map_at_3_diff1 + value: 53.11784737840137 + - type: nauc_map_at_3_max + value: 3.059682761072153 + - type: nauc_map_at_3_std + value: 21.310633086556617 + - type: nauc_map_at_5_diff1 + value: 49.91570701185436 + - type: nauc_map_at_5_max + value: 8.045082896244576 + - type: nauc_map_at_5_std + value: 20.597686235051647 + - type: nauc_mrr_at_1000_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_1000_max + value: 78.24463118580779 + - type: nauc_mrr_at_1000_std + value: 0.30812324930028195 + - type: nauc_mrr_at_100_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_100_max + value: 78.24463118580779 + - type: nauc_mrr_at_100_std + value: 0.30812324930028195 + - type: nauc_mrr_at_10_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_10_max + value: 78.24463118580779 + - type: nauc_mrr_at_10_std + value: 0.30812324930028195 + - type: nauc_mrr_at_1_diff1 + value: 38.62433862433873 + - type: nauc_mrr_at_1_max + value: 80.78120136943666 + - type: nauc_mrr_at_1_std + value: -10.768751945222197 + - type: nauc_mrr_at_20_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_20_max + value: 78.24463118580779 + - type: nauc_mrr_at_20_std + value: 0.30812324930028195 + - type: nauc_mrr_at_3_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_3_max + value: 78.24463118580779 + - type: nauc_mrr_at_3_std + value: 0.30812324930028195 + - type: nauc_mrr_at_5_diff1 + value: 41.98412698412726 + - type: nauc_mrr_at_5_max + value: 78.24463118580779 + - type: nauc_mrr_at_5_std + value: 0.30812324930028195 + - type: nauc_ndcg_at_1000_diff1 + value: 0.5174948602880207 + - type: nauc_ndcg_at_1000_max + value: 48.60686602077053 + - type: nauc_ndcg_at_1000_std + value: 75.72456343175277 + - type: nauc_ndcg_at_100_diff1 + value: -20.747252137999254 + - type: nauc_ndcg_at_100_max + value: 49.985132618254994 + - type: nauc_ndcg_at_100_std + value: 61.096383293836574 + - type: nauc_ndcg_at_10_diff1 + value: 6.791377920463332 + - type: nauc_ndcg_at_10_max + value: 57.50019332833286 + - type: nauc_ndcg_at_10_std + value: 49.201028841219426 + - type: nauc_ndcg_at_1_diff1 + value: 54.92683440362145 + - type: nauc_ndcg_at_1_max + value: 83.8667228129276 + - type: nauc_ndcg_at_1_std + value: 1.6738604063586122 + - type: nauc_ndcg_at_20_diff1 + value: -5.1948699196314925 + - type: nauc_ndcg_at_20_max + value: 54.483087684806556 + - type: nauc_ndcg_at_20_std + value: 50.54823818118781 + - type: nauc_ndcg_at_3_diff1 + value: 26.267246500164372 + - type: nauc_ndcg_at_3_max + value: 63.0173212926611 + - type: nauc_ndcg_at_3_std + value: 41.025597406368256 + - type: nauc_ndcg_at_5_diff1 + value: 16.910185454343036 + - type: nauc_ndcg_at_5_max + value: 60.9328683868778 + - type: nauc_ndcg_at_5_std + value: 36.70169905857712 + - type: nauc_precision_at_1000_diff1 + value: -46.374447765983525 + - type: nauc_precision_at_1000_max + value: 35.36052337813863 + - type: nauc_precision_at_1000_std + value: 14.219220668161018 + - type: nauc_precision_at_100_diff1 + value: -29.7838083657744 + - type: nauc_precision_at_100_max + value: 43.93589400385112 + - type: nauc_precision_at_100_std + value: 55.425045718579945 + - type: nauc_precision_at_10_diff1 + value: -12.016613405227687 + - type: nauc_precision_at_10_max + value: 57.79924427743131 + - type: nauc_precision_at_10_std + value: 49.022036703550675 + - type: nauc_precision_at_1_diff1 + value: 38.62433862433873 + - type: nauc_precision_at_1_max + value: 80.78120136943666 + - type: nauc_precision_at_1_std + value: -10.768751945222197 + - type: nauc_precision_at_20_diff1 + value: -23.95633847880195 + - type: nauc_precision_at_20_max + value: 48.34715917258276 + - type: nauc_precision_at_20_std + value: 48.82198285255887 + - type: nauc_precision_at_3_diff1 + value: 6.871296905858807 + - type: nauc_precision_at_3_max + value: 70.54805793285054 + - type: nauc_precision_at_3_std + value: 44.65108624094803 + - type: nauc_precision_at_5_diff1 + value: -9.074932448759695 + - type: nauc_precision_at_5_max + value: 67.41284242437573 + - type: nauc_precision_at_5_std + value: 23.876891983919577 + - type: nauc_recall_at_1000_diff1 + value: 8.142288830293255 + - type: nauc_recall_at_1000_max + value: 38.85182826835104 + - type: nauc_recall_at_1000_std + value: 68.60783819217335 + - type: nauc_recall_at_100_diff1 + value: 34.262914076287466 + - type: nauc_recall_at_100_max + value: 12.87009658528838 + - type: nauc_recall_at_100_std + value: 56.21330603762995 + - type: nauc_recall_at_10_diff1 + value: 49.33830945338758 + - type: nauc_recall_at_10_max + value: 0.3539875530671406 + - type: nauc_recall_at_10_std + value: 26.85864465557644 + - type: nauc_recall_at_1_diff1 + value: 53.48238396620123 + - type: nauc_recall_at_1_max + value: 0.33476619393733076 + - type: nauc_recall_at_1_std + value: 8.906362219128463 + - type: nauc_recall_at_20_diff1 + value: 44.21928181266254 + - type: nauc_recall_at_20_max + value: -0.9198356057088594 + - type: nauc_recall_at_20_std + value: 31.484376992896784 + - type: nauc_recall_at_3_diff1 + value: 53.038093080990876 + - type: nauc_recall_at_3_max + value: -1.4170895916973003 + - type: nauc_recall_at_3_std + value: 21.890202855574497 + - type: nauc_recall_at_5_diff1 + value: 49.39742214825278 + - type: nauc_recall_at_5_max + value: 2.8412267611894517 + - type: nauc_recall_at_5_std + value: 18.01598921859512 + - type: ndcg_at_1 + value: 91.0 + - type: ndcg_at_10 + value: 85.206 + - type: ndcg_at_100 + value: 67.29 + - type: ndcg_at_1000 + value: 60.584 + - type: ndcg_at_20 + value: 82.321 + - type: ndcg_at_3 + value: 88.642 + - type: ndcg_at_5 + value: 87.063 + - type: precision_at_1 + value: 94.0 + - type: precision_at_10 + value: 89.8 + - type: precision_at_100 + value: 69.78 + - type: precision_at_1000 + value: 26.738 + - type: precision_at_20 + value: 87.2 + - type: precision_at_3 + value: 92.0 + - type: precision_at_5 + value: 90.8 + - type: recall_at_1 + value: 0.246 + - type: recall_at_10 + value: 2.344 + - type: recall_at_100 + value: 16.962 + - type: recall_at_1000 + value: 57.325 + - type: recall_at_20 + value: 4.517 + - type: recall_at_3 + value: 0.731 + - type: recall_at_5 + value: 1.1780000000000002 + task: + type: Retrieval + - dataset: + config: default + name: MTEB Touche2020 + revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f + split: test + type: mteb/touche2020 + metrics: + - type: main_score + value: 31.455 + - type: map_at_1 + value: 2.9739999999999998 + - type: map_at_10 + value: 12.183 + - type: map_at_100 + value: 18.772 + - type: map_at_1000 + value: 20.415 + - type: map_at_20 + value: 14.451 + - type: map_at_3 + value: 6.507000000000001 + - type: map_at_5 + value: 8.66 + - type: mrr_at_1 + value: 40.816326530612244 + - type: mrr_at_10 + value: 57.70975056689341 + - type: mrr_at_100 + value: 58.18379126542391 + - type: mrr_at_1000 + value: 58.18379126542391 + - type: mrr_at_20 + value: 57.85552316164561 + - type: mrr_at_3 + value: 54.08163265306123 + - type: mrr_at_5 + value: 56.42857142857143 + - type: nauc_map_at_1000_diff1 + value: 3.1567471051481437 + - type: nauc_map_at_1000_max + value: -1.5882060729791523 + - type: nauc_map_at_1000_std + value: 18.69622198722074 + - type: nauc_map_at_100_diff1 + value: 3.3449677678147536 + - type: nauc_map_at_100_max + value: -2.8928606866168405 + - type: nauc_map_at_100_std + value: 15.789984947653412 + - type: nauc_map_at_10_diff1 + value: 2.9696743570444264 + - type: nauc_map_at_10_max + value: -9.096749212011876 + - type: nauc_map_at_10_std + value: -5.38545817258353 + - type: nauc_map_at_1_diff1 + value: 20.680780404542546 + - type: nauc_map_at_1_max + value: -7.04722927447817 + - type: nauc_map_at_1_std + value: -7.062494733973898 + - type: nauc_map_at_20_diff1 + value: 4.070437790119271 + - type: nauc_map_at_20_max + value: -4.84491434686032 + - type: nauc_map_at_20_std + value: 0.5846341109021014 + - type: nauc_map_at_3_diff1 + value: 11.9634978045925 + - type: nauc_map_at_3_max + value: -8.27834591046608 + - type: nauc_map_at_3_std + value: -8.687615453381065 + - type: nauc_map_at_5_diff1 + value: 0.9195191526009436 + - type: nauc_map_at_5_max + value: -1.673813362719489 + - type: nauc_map_at_5_std + value: -6.67549753473631 + - type: nauc_mrr_at_1000_diff1 + value: 19.877993208719573 + - type: nauc_mrr_at_1000_max + value: -10.37776706406218 + - type: nauc_mrr_at_1000_std + value: 7.132169578056367 + - type: nauc_mrr_at_100_diff1 + value: 19.877993208719573 + - type: nauc_mrr_at_100_max + value: -10.37776706406218 + - type: nauc_mrr_at_100_std + value: 7.132169578056367 + - type: nauc_mrr_at_10_diff1 + value: 20.414285568401457 + - type: nauc_mrr_at_10_max + value: -9.677800295687861 + - type: nauc_mrr_at_10_std + value: 8.001103690180859 + - type: nauc_mrr_at_1_diff1 + value: 22.393284073955723 + - type: nauc_mrr_at_1_max + value: -5.889370191243167 + - type: nauc_mrr_at_1_std + value: -1.5183536173658247 + - type: nauc_mrr_at_20_diff1 + value: 20.455564720604055 + - type: nauc_mrr_at_20_max + value: -10.230642830103074 + - type: nauc_mrr_at_20_std + value: 7.863582453266621 + - type: nauc_mrr_at_3_diff1 + value: 17.554895390732618 + - type: nauc_mrr_at_3_max + value: -15.618463505555052 + - type: nauc_mrr_at_3_std + value: 5.913231577966864 + - type: nauc_mrr_at_5_diff1 + value: 18.393678507779914 + - type: nauc_mrr_at_5_max + value: -11.903593353147762 + - type: nauc_mrr_at_5_std + value: 7.580745996262831 + - type: nauc_ndcg_at_1000_diff1 + value: 13.746937095530473 + - type: nauc_ndcg_at_1000_max + value: -0.9319249687895838 + - type: nauc_ndcg_at_1000_std + value: 38.56328031451904 + - type: nauc_ndcg_at_100_diff1 + value: 13.854865944415895 + - type: nauc_ndcg_at_100_max + value: -7.142142012591404 + - type: nauc_ndcg_at_100_std + value: 35.61341954818848 + - type: nauc_ndcg_at_10_diff1 + value: 9.010144273248759 + - type: nauc_ndcg_at_10_max + value: -15.320014897424574 + - type: nauc_ndcg_at_10_std + value: 2.84883880489144 + - type: nauc_ndcg_at_1_diff1 + value: 20.939533945592967 + - type: nauc_ndcg_at_1_max + value: -6.387319972188946 + - type: nauc_ndcg_at_1_std + value: -0.5258673122126726 + - type: nauc_ndcg_at_20_diff1 + value: 14.660827309009496 + - type: nauc_ndcg_at_20_max + value: -13.476196120145994 + - type: nauc_ndcg_at_20_std + value: 8.22391881710838 + - type: nauc_ndcg_at_3_diff1 + value: 13.429985227235935 + - type: nauc_ndcg_at_3_max + value: -14.904544592570247 + - type: nauc_ndcg_at_3_std + value: 1.599779998183342 + - type: nauc_ndcg_at_5_diff1 + value: 8.085466231900622 + - type: nauc_ndcg_at_5_max + value: -9.09591969526831 + - type: nauc_ndcg_at_5_std + value: 3.5794092637248505 + - type: nauc_precision_at_1000_diff1 + value: -9.31941215946743 + - type: nauc_precision_at_1000_max + value: 31.52913520470716 + - type: nauc_precision_at_1000_std + value: 22.720784312185856 + - type: nauc_precision_at_100_diff1 + value: 8.958548406995279 + - type: nauc_precision_at_100_max + value: 15.100597910674104 + - type: nauc_precision_at_100_std + value: 71.04548238175113 + - type: nauc_precision_at_10_diff1 + value: 12.4698194690008 + - type: nauc_precision_at_10_max + value: -15.84870544871496 + - type: nauc_precision_at_10_std + value: 7.575297622501928 + - type: nauc_precision_at_1_diff1 + value: 22.393284073955723 + - type: nauc_precision_at_1_max + value: -5.889370191243167 + - type: nauc_precision_at_1_std + value: -1.5183536173658247 + - type: nauc_precision_at_20_diff1 + value: 15.393505718138758 + - type: nauc_precision_at_20_max + value: -3.70684298539384 + - type: nauc_precision_at_20_std + value: 29.426137824970304 + - type: nauc_precision_at_3_diff1 + value: 9.997768085465394 + - type: nauc_precision_at_3_max + value: -17.12224314347674 + - type: nauc_precision_at_3_std + value: -1.343018166772313 + - type: nauc_precision_at_5_diff1 + value: 3.8936997437913554 + - type: nauc_precision_at_5_max + value: -5.689104289687632 + - type: nauc_precision_at_5_std + value: 3.181098051304285 + - type: nauc_recall_at_1000_diff1 + value: 9.908303508158387 + - type: nauc_recall_at_1000_max + value: 6.174506592699848 + - type: nauc_recall_at_1000_std + value: 77.41931114780012 + - type: nauc_recall_at_100_diff1 + value: 10.286839241876192 + - type: nauc_recall_at_100_max + value: -6.6138697026666815 + - type: nauc_recall_at_100_std + value: 49.608313692633224 + - type: nauc_recall_at_10_diff1 + value: 2.215545846659851 + - type: nauc_recall_at_10_max + value: -17.83025802478445 + - type: nauc_recall_at_10_std + value: -3.3784768673705465 + - type: nauc_recall_at_1_diff1 + value: 20.680780404542546 + - type: nauc_recall_at_1_max + value: -7.04722927447817 + - type: nauc_recall_at_1_std + value: -7.062494733973898 + - type: nauc_recall_at_20_diff1 + value: 6.974410239251615 + - type: nauc_recall_at_20_max + value: -14.161147924731646 + - type: nauc_recall_at_20_std + value: 9.328412057721454 + - type: nauc_recall_at_3_diff1 + value: 7.904589805754212 + - type: nauc_recall_at_3_max + value: -12.1912388648593 + - type: nauc_recall_at_3_std + value: -9.221542013385555 + - type: nauc_recall_at_5_diff1 + value: -3.2604132752706914 + - type: nauc_recall_at_5_max + value: -6.886351441658915 + - type: nauc_recall_at_5_std + value: -7.014252851712789 + - type: ndcg_at_1 + value: 39.796 + - type: ndcg_at_10 + value: 31.455 + - type: ndcg_at_100 + value: 42.388999999999996 + - type: ndcg_at_1000 + value: 53.556000000000004 + - type: ndcg_at_20 + value: 30.808000000000003 + - type: ndcg_at_3 + value: 35.831 + - type: ndcg_at_5 + value: 32.845 + - type: precision_at_1 + value: 40.816 + - type: precision_at_10 + value: 27.143 + - type: precision_at_100 + value: 8.449 + - type: precision_at_1000 + value: 1.6179999999999999 + - type: precision_at_20 + value: 19.387999999999998 + - type: precision_at_3 + value: 35.374 + - type: precision_at_5 + value: 31.019999999999996 + - type: recall_at_1 + value: 2.9739999999999998 + - type: recall_at_10 + value: 19.39 + - type: recall_at_100 + value: 51.636 + - type: recall_at_1000 + value: 86.99900000000001 + - type: recall_at_20 + value: 26.478 + - type: recall_at_3 + value: 7.703 + - type: recall_at_5 + value: 11.42 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ToxicConversationsClassification + revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de + split: test + type: mteb/toxic_conversations_50k + metrics: + - type: accuracy + value: 86.9384765625 + - type: ap + value: 31.737513704141552 + - type: ap_weighted + value: 31.737513704141552 + - type: f1 + value: 71.5490757306975 + - type: f1_weighted + value: 89.14632533489856 + - type: main_score + value: 86.9384765625 + task: + type: Classification + - dataset: + config: default + name: MTEB TweetSentimentExtractionClassification + revision: d604517c81ca91fe16a244d1248fc021f9ecee7a + split: test + type: mteb/tweet_sentiment_extraction + metrics: + - type: accuracy + value: 73.57668364459535 + - type: f1 + value: 73.90467103648074 + - type: f1_weighted + value: 73.42158415034704 + - type: main_score + value: 73.57668364459535 + task: + type: Classification + - dataset: + config: default + name: MTEB TwentyNewsgroupsClustering + revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 + split: test + type: mteb/twentynewsgroups-clustering + metrics: + - type: main_score + value: 58.574148097494685 + - type: v_measure + value: 58.574148097494685 + - type: v_measure_std + value: 0.9443161637490822 + task: + type: Clustering + - dataset: + config: default + name: MTEB TwitterSemEval2015 + revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 + split: test + type: mteb/twittersemeval2015-pairclassification + metrics: + - type: cosine_accuracy + value: 88.1385229778864 + - type: cosine_accuracy_threshold + value: 83.86307954788208 + - type: cosine_ap + value: 80.17965893449055 + - type: cosine_f1 + value: 73.0614300100705 + - type: cosine_f1_threshold + value: 80.7942807674408 + - type: cosine_precision + value: 69.8603755416466 + - type: cosine_recall + value: 76.56992084432717 + - type: dot_accuracy + value: 88.2100494724921 + - type: dot_accuracy_threshold + value: 83.84793996810913 + - type: dot_ap + value: 80.18603932881858 + - type: dot_f1 + value: 73.07643714466204 + - type: dot_f1_threshold + value: 80.87586164474487 + - type: dot_precision + value: 70.10909090909091 + - type: dot_recall + value: 76.3060686015831 + - type: euclidean_accuracy + value: 88.1385229778864 + - type: euclidean_accuracy_threshold + value: 56.77661895751953 + - type: euclidean_ap + value: 80.1784070881624 + - type: euclidean_f1 + value: 73.04830369529574 + - type: euclidean_f1_threshold + value: 61.91838979721069 + - type: euclidean_precision + value: 69.96859144720948 + - type: euclidean_recall + value: 76.41160949868075 + - type: main_score + value: 80.18603932881858 + - type: manhattan_accuracy + value: 88.0431543184121 + - type: manhattan_accuracy_threshold + value: 3755.6137084960938 + - type: manhattan_ap + value: 79.98270453664578 + - type: manhattan_f1 + value: 72.68242015061023 + - type: manhattan_f1_threshold + value: 3892.494583129883 + - type: manhattan_precision + value: 71.54907975460122 + - type: manhattan_recall + value: 73.85224274406332 + - type: max_ap + value: 80.18603932881858 + - type: max_f1 + value: 73.07643714466204 + - type: max_precision + value: 71.54907975460122 + - type: max_recall + value: 76.56992084432717 + - type: similarity_accuracy + value: 88.1385229778864 + - type: similarity_accuracy_threshold + value: 83.86307954788208 + - type: similarity_ap + value: 80.17965893449055 + - type: similarity_f1 + value: 73.0614300100705 + - type: similarity_f1_threshold + value: 80.7942807674408 + - type: similarity_precision + value: 69.8603755416466 + - type: similarity_recall + value: 76.56992084432717 + task: + type: PairClassification + - dataset: + config: default + name: MTEB TwitterURLCorpus + revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf + split: test + type: mteb/twitterurlcorpus-pairclassification + metrics: + - type: cosine_accuracy + value: 89.7892653393876 + - type: cosine_accuracy_threshold + value: 79.69566583633423 + - type: cosine_ap + value: 87.4579867302024 + - type: cosine_f1 + value: 79.91620843152658 + - type: cosine_f1_threshold + value: 78.53609323501587 + - type: cosine_precision + value: 77.7155329210622 + - type: cosine_recall + value: 82.24514936864799 + - type: dot_accuracy + value: 89.78732487289945 + - type: dot_accuracy_threshold + value: 80.05315661430359 + - type: dot_ap + value: 87.44916182456272 + - type: dot_f1 + value: 79.90419878751591 + - type: dot_f1_threshold + value: 78.57890725135803 + - type: dot_precision + value: 77.73409057812728 + - type: dot_recall + value: 82.19895287958116 + - type: euclidean_accuracy + value: 89.78538440641131 + - type: euclidean_accuracy_threshold + value: 62.29925751686096 + - type: euclidean_ap + value: 87.45904868911386 + - type: euclidean_f1 + value: 79.93127404474657 + - type: euclidean_f1_threshold + value: 65.61101078987122 + - type: euclidean_precision + value: 77.62060210373595 + - type: euclidean_recall + value: 82.38373883584848 + - type: main_score + value: 87.46554314325058 + - type: manhattan_accuracy + value: 89.76597974152986 + - type: manhattan_accuracy_threshold + value: 3988.5299682617188 + - type: manhattan_ap + value: 87.46554314325058 + - type: manhattan_f1 + value: 79.97181740645973 + - type: manhattan_f1_threshold + value: 4235.905838012695 + - type: manhattan_precision + value: 77.13713427283783 + - type: manhattan_recall + value: 83.02279026793964 + - type: max_ap + value: 87.46554314325058 + - type: max_f1 + value: 79.97181740645973 + - type: max_precision + value: 77.73409057812728 + - type: max_recall + value: 83.02279026793964 + - type: similarity_accuracy + value: 89.7892653393876 + - type: similarity_accuracy_threshold + value: 79.69566583633423 + - type: similarity_ap + value: 87.4579867302024 + - type: similarity_f1 + value: 79.91620843152658 + - type: similarity_f1_threshold + value: 78.53609323501587 + - type: similarity_precision + value: 77.7155329210622 + - type: similarity_recall + value: 82.24514936864799 + task: + type: PairClassification +tags: +- mteb +- sentence-transformers +- transformers +- sentence-similarity +license: mit +--- + + + +# Updates + +New open-source models and ToDoList will be listed on https://github.com/DunZhang/Stella/blob/main/news_and_todo.md. + +You can also find these models on my [homepage](https://huggingface.co/infgrad). + +# Introduction + +The models are trained based on `Alibaba-NLP/gte-large-en-v1.5` and `Alibaba-NLP/gte-Qwen2-1.5B-instruct`. Thanks for +their contributions! + +**We simplify usage of prompts, providing two prompts for most general tasks, one is for s2p, another one is for s2s.** + +Prompt of s2p task(e.g. retrieve task): + +```text +Instruct: Given a web search query, retrieve relevant passages that answer the query.\nQuery: {query} +``` + +Prompt of s2s task(e.g. semantic textual similarity task): + +```text +Instruct: Retrieve semantically similar text.\nQuery: {query} +``` + +The models are finally trained by [MRL]((https://arxiv.org/abs/2205.13147)), so they have multiple dimensions: 512, 768, +1024, 2048, 4096, 6144 and 8192. + +The higher the dimension, the better the performance. +**Generally speaking, 1024d is good enough.** The MTEB score of 1024d is only 0.001 lower than 8192d. + +# Model directory structure + +The model directory structure is very simple, it is a standard SentenceTransformer directory **with a series +of `2_Dense_{dims}` +folders**, where `dims` represents the final vector dimension. + +For example, the `2_Dense_256` folder stores Linear weights that convert vector dimensions to 256 dimensions. +Please refer to the following chapters for specific instructions on how to use them. + +# Usage + +You can use `SentenceTransformers` or `transformers` library to encode text. + +## Sentence Transformers + +```python +from sentence_transformers import SentenceTransformer + +# This model supports two prompts: "s2p_query" and "s2s_query" for sentence-to-passage and sentence-to-sentence tasks, respectively. +# They are defined in `config_sentence_transformers.json` +query_prompt_name = "s2p_query" +queries = [ + "What are some ways to reduce stress?", + "What are the benefits of drinking green tea?", +] +# docs do not need any prompts +docs = [ + "There are many effective ways to reduce stress. Some common techniques include deep breathing, meditation, and physical activity. Engaging in hobbies, spending time in nature, and connecting with loved ones can also help alleviate stress. Additionally, setting boundaries, practicing self-care, and learning to say no can prevent stress from building up.", + "Green tea has been consumed for centuries and is known for its potential health benefits. It contains antioxidants that may help protect the body against damage caused by free radicals. Regular consumption of green tea has been associated with improved heart health, enhanced cognitive function, and a reduced risk of certain types of cancer. The polyphenols in green tea may also have anti-inflammatory and weight loss properties.", +] + +# ！The default dimension is 1024, if you need other dimensions, please clone the model and modify `modules.json` to replace `2_Dense_1024` with another dimension, e.g. `2_Dense_256` or `2_Dense_8192` ! +# on gpu +model = SentenceTransformer("dunzhang/stella_en_400M_v5", trust_remote_code=True).cuda() +# you can also use this model without the features of `use_memory_efficient_attention` and `unpad_inputs`. It can be worked in CPU. +# model = SentenceTransformer( +# "dunzhang/stella_en_400M_v5", +# trust_remote_code=True, +# device="cpu", +# config_kwargs={"use_memory_efficient_attention": False, "unpad_inputs": False} +# ) +query_embeddings = model.encode(queries, prompt_name=query_prompt_name) +doc_embeddings = model.encode(docs) +print(query_embeddings.shape, doc_embeddings.shape) +# (2, 1024) (2, 1024) + +similarities = model.similarity(query_embeddings, doc_embeddings) +print(similarities) +# tensor([[0.8398, 0.2990], +# [0.3282, 0.8095]]) +``` + +## Transformers + +```python +import os +import torch +from transformers import AutoModel, AutoTokenizer +from sklearn.preprocessing import normalize + +query_prompt = "Instruct: Given a web search query, retrieve relevant passages that answer the query.\nQuery: " +queries = [ + "What are some ways to reduce stress?", + "What are the benefits of drinking green tea?", +] +queries = [query_prompt + query for query in queries] +# docs do not need any prompts +docs = [ + "There are many effective ways to reduce stress. Some common techniques include deep breathing, meditation, and physical activity. Engaging in hobbies, spending time in nature, and connecting with loved ones can also help alleviate stress. Additionally, setting boundaries, practicing self-care, and learning to say no can prevent stress from building up.", + "Green tea has been consumed for centuries and is known for its potential health benefits. It contains antioxidants that may help protect the body against damage caused by free radicals. Regular consumption of green tea has been associated with improved heart health, enhanced cognitive function, and a reduced risk of certain types of cancer. The polyphenols in green tea may also have anti-inflammatory and weight loss properties.", +] + +# The path of your model after cloning it +model_dir = "{Your MODEL_PATH}" + +vector_dim = 1024 +vector_linear_directory = f"2_Dense_{vector_dim}" +model = AutoModel.from_pretrained(model_dir, trust_remote_code=True).cuda().eval() +# you can also use this model without the features of `use_memory_efficient_attention` and `unpad_inputs`. It can be worked in CPU. +# model = AutoModel.from_pretrained(model_dir, trust_remote_code=True,use_memory_efficient_attention=False,unpad_inputs=False).cuda().eval() +tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) +vector_linear = torch.nn.Linear(in_features=model.config.hidden_size, out_features=vector_dim) +vector_linear_dict = { + k.replace("linear.", ""): v for k, v in + torch.load(os.path.join(model_dir, f"{vector_linear_directory}/pytorch_model.bin")).items() +} +vector_linear.load_state_dict(vector_linear_dict) +vector_linear.cuda() + +# Embed the queries +with torch.no_grad(): + input_data = tokenizer(queries, padding="longest", truncation=True, max_length=512, return_tensors="pt") + input_data = {k: v.cuda() for k, v in input_data.items()} + attention_mask = input_data["attention_mask"] + last_hidden_state = model(**input_data)[0] + last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0) + query_vectors = last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] + query_vectors = normalize(vector_linear(query_vectors).cpu().numpy()) + +# Embed the documents +with torch.no_grad(): + input_data = tokenizer(docs, padding="longest", truncation=True, max_length=512, return_tensors="pt") + input_data = {k: v.cuda() for k, v in input_data.items()} + attention_mask = input_data["attention_mask"] + last_hidden_state = model(**input_data)[0] + last_hidden = last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0) + docs_vectors = last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] + docs_vectors = normalize(vector_linear(docs_vectors).cpu().numpy()) + +print(query_vectors.shape, docs_vectors.shape) +# (2, 1024) (2, 1024) + +similarities = query_vectors @ docs_vectors.T +print(similarities) +# [[0.8397531 0.29900077] +# [0.32818374 0.80954516]] +``` + +# FAQ + +Q: The details of training? + +A: The training method and datasets will be released in the future. (specific time unknown, may be provided in a paper) + +Q: How to choose a suitable prompt for my own task? + +A: In most cases, please use the s2p and s2s prompts. These two prompts account for the vast majority of the training +data. + +Q: How to reproduce MTEB results? + +A: Please use evaluation scripts in `Alibaba-NLP/gte-Qwen2-1.5B-instruct` or `intfloat/e5-mistral-7b-instruct` + +Q: Why each dimension has a linear weight? + +A: MRL has multiple training methods, we choose this method which has the best performance. + +Q: What is the sequence length of models? + +A: 512 is recommended, in our experiments, almost all models perform poorly on specialized long text retrieval datasets. Besides, the +model is trained on datasets of 512 length. This may be an optimization term. + If you have any questions, please start a discussion on community. \ No newline at end of file