metadata
tags:
- mteb
model-index:
- name: piccolo-base-zh
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 49.16558217326158
- type: cos_sim_spearman
value: 51.4049475858823
- type: euclidean_pearson
value: 49.85853741070363
- type: euclidean_spearman
value: 51.501428092542234
- type: manhattan_pearson
value: 49.746099634926296
- type: manhattan_spearman
value: 51.41081804320127
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 52.385361699031854
- type: cos_sim_spearman
value: 52.59114913702212
- type: euclidean_pearson
value: 54.994530439418355
- type: euclidean_spearman
value: 52.54102886188004
- type: manhattan_pearson
value: 54.9503071669608
- type: manhattan_spearman
value: 52.51465652540901
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 40.236
- type: f1
value: 39.43040092463147
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 60.98952187211432
- type: cos_sim_spearman
value: 62.68189713123115
- type: euclidean_pearson
value: 61.089426749761344
- type: euclidean_spearman
value: 62.41743375544581
- type: manhattan_pearson
value: 61.14747216341409
- type: manhattan_spearman
value: 62.488918956547046
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 38.36392300667918
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 35.645927581489175
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 85.25085782849087
- type: mrr
value: 87.77154761904762
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 86.15357754080844
- type: mrr
value: 88.53547619047617
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 23.683
- type: map_at_10
value: 35.522999999999996
- type: map_at_100
value: 37.456
- type: map_at_1000
value: 37.576
- type: map_at_3
value: 31.584
- type: map_at_5
value: 33.684999999999995
- type: mrr_at_1
value: 36.459
- type: mrr_at_10
value: 44.534
- type: mrr_at_100
value: 45.6
- type: mrr_at_1000
value: 45.647
- type: mrr_at_3
value: 42.186
- type: mrr_at_5
value: 43.482
- type: ndcg_at_1
value: 36.459
- type: ndcg_at_10
value: 42.025
- type: ndcg_at_100
value: 49.754
- type: ndcg_at_1000
value: 51.815999999999995
- type: ndcg_at_3
value: 37.056
- type: ndcg_at_5
value: 38.962
- type: precision_at_1
value: 36.459
- type: precision_at_10
value: 9.485000000000001
- type: precision_at_100
value: 1.567
- type: precision_at_1000
value: 0.183
- type: precision_at_3
value: 21.13
- type: precision_at_5
value: 15.209
- type: recall_at_1
value: 23.683
- type: recall_at_10
value: 52.190999999999995
- type: recall_at_100
value: 84.491
- type: recall_at_1000
value: 98.19600000000001
- type: recall_at_3
value: 37.09
- type: recall_at_5
value: 43.262
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 74.20324714371618
- type: cos_sim_ap
value: 82.32631646194994
- type: cos_sim_f1
value: 76.64052827073876
- type: cos_sim_precision
value: 68.58725761772854
- type: cos_sim_recall
value: 86.83656768763151
- type: dot_accuracy
value: 70.33072760072159
- type: dot_ap
value: 77.46972172609794
- type: dot_f1
value: 73.6668924804026
- type: dot_precision
value: 62.84676354029062
- type: dot_recall
value: 88.98760813654431
- type: euclidean_accuracy
value: 74.78051713770296
- type: euclidean_ap
value: 82.65778389584023
- type: euclidean_f1
value: 77.1843623157445
- type: euclidean_precision
value: 71.05211406096362
- type: euclidean_recall
value: 84.47509936871639
- type: manhattan_accuracy
value: 74.76849067949489
- type: manhattan_ap
value: 82.55694030572194
- type: manhattan_f1
value: 77.1776459569154
- type: manhattan_precision
value: 69.5423855963991
- type: manhattan_recall
value: 86.69628244096329
- type: max_accuracy
value: 74.78051713770296
- type: max_ap
value: 82.65778389584023
- type: max_f1
value: 77.1843623157445
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 72.99799999999999
- type: map_at_10
value: 81.271
- type: map_at_100
value: 81.53399999999999
- type: map_at_1000
value: 81.535
- type: map_at_3
value: 80.049
- type: map_at_5
value: 80.793
- type: mrr_at_1
value: 73.13
- type: mrr_at_10
value: 81.193
- type: mrr_at_100
value: 81.463
- type: mrr_at_1000
value: 81.464
- type: mrr_at_3
value: 80.067
- type: mrr_at_5
value: 80.741
- type: ndcg_at_1
value: 73.34
- type: ndcg_at_10
value: 84.503
- type: ndcg_at_100
value: 85.643
- type: ndcg_at_1000
value: 85.693
- type: ndcg_at_3
value: 82.135
- type: ndcg_at_5
value: 83.401
- type: precision_at_1
value: 73.34
- type: precision_at_10
value: 9.536
- type: precision_at_100
value: 1.004
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 29.54
- type: precision_at_5
value: 18.398
- type: recall_at_1
value: 72.99799999999999
- type: recall_at_10
value: 94.31
- type: recall_at_100
value: 99.368
- type: recall_at_1000
value: 99.789
- type: recall_at_3
value: 87.935
- type: recall_at_5
value: 90.991
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 26.537
- type: map_at_10
value: 81.292
- type: map_at_100
value: 84.031
- type: map_at_1000
value: 84.066
- type: map_at_3
value: 56.571000000000005
- type: map_at_5
value: 71.082
- type: mrr_at_1
value: 91.2
- type: mrr_at_10
value: 93.893
- type: mrr_at_100
value: 93.955
- type: mrr_at_1000
value: 93.95700000000001
- type: mrr_at_3
value: 93.61699999999999
- type: mrr_at_5
value: 93.767
- type: ndcg_at_1
value: 91.2
- type: ndcg_at_10
value: 88.255
- type: ndcg_at_100
value: 90.813
- type: ndcg_at_1000
value: 91.144
- type: ndcg_at_3
value: 87.435
- type: ndcg_at_5
value: 85.961
- type: precision_at_1
value: 91.2
- type: precision_at_10
value: 42.14
- type: precision_at_100
value: 4.817
- type: precision_at_1000
value: 0.48900000000000005
- type: precision_at_3
value: 78.467
- type: precision_at_5
value: 65.75999999999999
- type: recall_at_1
value: 26.537
- type: recall_at_10
value: 89.262
- type: recall_at_100
value: 97.783
- type: recall_at_1000
value: 99.49799999999999
- type: recall_at_3
value: 58.573
- type: recall_at_5
value: 75.154
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 48.5
- type: map_at_10
value: 57.898
- type: map_at_100
value: 58.599000000000004
- type: map_at_1000
value: 58.616
- type: map_at_3
value: 55.1
- type: map_at_5
value: 56.80500000000001
- type: mrr_at_1
value: 48.5
- type: mrr_at_10
value: 57.898
- type: mrr_at_100
value: 58.599000000000004
- type: mrr_at_1000
value: 58.616
- type: mrr_at_3
value: 55.1
- type: mrr_at_5
value: 56.80500000000001
- type: ndcg_at_1
value: 48.5
- type: ndcg_at_10
value: 62.876
- type: ndcg_at_100
value: 66.00200000000001
- type: ndcg_at_1000
value: 66.467
- type: ndcg_at_3
value: 57.162
- type: ndcg_at_5
value: 60.263999999999996
- type: precision_at_1
value: 48.5
- type: precision_at_10
value: 7.870000000000001
- type: precision_at_100
value: 0.927
- type: precision_at_1000
value: 0.096
- type: precision_at_3
value: 21.032999999999998
- type: precision_at_5
value: 14.14
- type: recall_at_1
value: 48.5
- type: recall_at_10
value: 78.7
- type: recall_at_100
value: 92.7
- type: recall_at_1000
value: 96.39999999999999
- type: recall_at_3
value: 63.1
- type: recall_at_5
value: 70.7
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 44.34782608695652
- type: f1
value: 36.401426200836205
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 84.25891181988743
- type: ap
value: 50.54636280166089
- type: f1
value: 78.55080202541332
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 70.02878561337955
- type: cos_sim_spearman
value: 75.39509553139982
- type: euclidean_pearson
value: 73.92598696939956
- type: euclidean_spearman
value: 75.5471147196853
- type: manhattan_pearson
value: 73.88049486090739
- type: manhattan_spearman
value: 75.51361990583285
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 64.739
- type: map_at_10
value: 74.039
- type: map_at_100
value: 74.38
- type: map_at_1000
value: 74.39099999999999
- type: map_at_3
value: 72.074
- type: map_at_5
value: 73.29299999999999
- type: mrr_at_1
value: 66.92
- type: mrr_at_10
value: 74.636
- type: mrr_at_100
value: 74.94
- type: mrr_at_1000
value: 74.95
- type: mrr_at_3
value: 72.911
- type: mrr_at_5
value: 73.981
- type: ndcg_at_1
value: 66.92
- type: ndcg_at_10
value: 77.924
- type: ndcg_at_100
value: 79.471
- type: ndcg_at_1000
value: 79.73400000000001
- type: ndcg_at_3
value: 74.17200000000001
- type: ndcg_at_5
value: 76.236
- type: precision_at_1
value: 66.92
- type: precision_at_10
value: 9.5
- type: precision_at_100
value: 1.027
- type: precision_at_1000
value: 0.105
- type: precision_at_3
value: 27.989000000000004
- type: precision_at_5
value: 17.874000000000002
- type: recall_at_1
value: 64.739
- type: recall_at_10
value: 89.324
- type: recall_at_100
value: 96.342
- type: recall_at_1000
value: 98.38900000000001
- type: recall_at_3
value: 79.378
- type: recall_at_5
value: 84.28099999999999
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 68.97108271687962
- type: f1
value: 66.8625981386677
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 73.32212508406187
- type: f1
value: 73.33875034670166
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 49
- type: map_at_10
value: 55.022999999999996
- type: map_at_100
value: 55.550999999999995
- type: map_at_1000
value: 55.608000000000004
- type: map_at_3
value: 53.417
- type: map_at_5
value: 54.372
- type: mrr_at_1
value: 49.3
- type: mrr_at_10
value: 55.176
- type: mrr_at_100
value: 55.703
- type: mrr_at_1000
value: 55.76
- type: mrr_at_3
value: 53.567
- type: mrr_at_5
value: 54.522000000000006
- type: ndcg_at_1
value: 49
- type: ndcg_at_10
value: 58.089999999999996
- type: ndcg_at_100
value: 60.988
- type: ndcg_at_1000
value: 62.580999999999996
- type: ndcg_at_3
value: 54.803000000000004
- type: ndcg_at_5
value: 56.508
- type: precision_at_1
value: 49
- type: precision_at_10
value: 6.78
- type: precision_at_100
value: 0.8210000000000001
- type: precision_at_1000
value: 0.095
- type: precision_at_3
value: 19.6
- type: precision_at_5
value: 12.58
- type: recall_at_1
value: 49
- type: recall_at_10
value: 67.80000000000001
- type: recall_at_100
value: 82.1
- type: recall_at_1000
value: 94.8
- type: recall_at_3
value: 58.8
- type: recall_at_5
value: 62.9
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 28.87237408060796
- type: mrr
value: 27.83015873015873
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 70.25
- type: f1
value: 70.29055400149645
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 65.56578234975636
- type: cos_sim_ap
value: 70.89354058570412
- type: cos_sim_f1
value: 71.21024370095002
- type: cos_sim_precision
value: 58.48032564450475
- type: cos_sim_recall
value: 91.02428722280888
- type: dot_accuracy
value: 64.86193827828912
- type: dot_ap
value: 70.17697803463875
- type: dot_f1
value: 70.68676716917922
- type: dot_precision
value: 58.57043719639139
- type: dot_recall
value: 89.1235480464625
- type: euclidean_accuracy
value: 64.86193827828912
- type: euclidean_ap
value: 70.26847152773904
- type: euclidean_f1
value: 70.9984152139461
- type: euclidean_precision
value: 56.81674064679771
- type: euclidean_recall
value: 94.61457233368532
- type: manhattan_accuracy
value: 65.40335679480238
- type: manhattan_ap
value: 70.22941558736018
- type: manhattan_f1
value: 71.09712937475423
- type: manhattan_precision
value: 56.64160401002506
- type: manhattan_recall
value: 95.45934530095037
- type: max_accuracy
value: 65.56578234975636
- type: max_ap
value: 70.89354058570412
- type: max_f1
value: 71.21024370095002
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 89.92999999999999
- type: ap
value: 87.16059195012956
- type: f1
value: 89.90917477839415
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 27.74161502387672
- type: cos_sim_spearman
value: 31.58353529723325
- type: euclidean_pearson
value: 32.43729673844635
- type: euclidean_spearman
value: 31.59527486602242
- type: manhattan_pearson
value: 32.37467059678786
- type: manhattan_spearman
value: 31.44408004951894
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 36.233749845501194
- type: cos_sim_spearman
value: 36.47808586229587
- type: euclidean_pearson
value: 32.663447466546806
- type: euclidean_spearman
value: 34.45830454037139
- type: manhattan_pearson
value: 32.80239212096335
- type: manhattan_spearman
value: 34.581060433895125
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 63.05131937664673
- type: cos_sim_spearman
value: 66.51353746725948
- type: euclidean_pearson
value: 61.24016998745561
- type: euclidean_spearman
value: 66.07115266049276
- type: manhattan_pearson
value: 64.55660243659054
- type: manhattan_spearman
value: 66.80282149562386
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 70.45533692882996
- type: cos_sim_spearman
value: 70.6045637565602
- type: euclidean_pearson
value: 72.75588977483554
- type: euclidean_spearman
value: 73.36630581886473
- type: manhattan_pearson
value: 72.72517409326954
- type: manhattan_spearman
value: 73.35358940437355
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 66.45779474032288
- type: mrr
value: 76.0782192023729
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 26.458
- type: map_at_10
value: 74.355
- type: map_at_100
value: 78.158
- type: map_at_1000
value: 78.233
- type: map_at_3
value: 52.2
- type: map_at_5
value: 64.14
- type: mrr_at_1
value: 88.37
- type: mrr_at_10
value: 91.117
- type: mrr_at_100
value: 91.231
- type: mrr_at_1000
value: 91.23599999999999
- type: mrr_at_3
value: 90.645
- type: mrr_at_5
value: 90.948
- type: ndcg_at_1
value: 88.37
- type: ndcg_at_10
value: 82.384
- type: ndcg_at_100
value: 86.431
- type: ndcg_at_1000
value: 87.163
- type: ndcg_at_3
value: 83.993
- type: ndcg_at_5
value: 82.411
- type: precision_at_1
value: 88.37
- type: precision_at_10
value: 41.131
- type: precision_at_100
value: 4.9799999999999995
- type: precision_at_1000
value: 0.515
- type: precision_at_3
value: 73.651
- type: precision_at_5
value: 61.634
- type: recall_at_1
value: 26.458
- type: recall_at_10
value: 81.3
- type: recall_at_100
value: 94.342
- type: recall_at_1000
value: 98.103
- type: recall_at_3
value: 54.020999999999994
- type: recall_at_5
value: 67.781
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 46.814
- type: f1
value: 45.580027683507666
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 61.43613064816144
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 53.01838461793776
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 59.3
- type: map_at_10
value: 69.158
- type: map_at_100
value: 69.60300000000001
- type: map_at_1000
value: 69.611
- type: map_at_3
value: 67.467
- type: map_at_5
value: 68.432
- type: mrr_at_1
value: 59.199999999999996
- type: mrr_at_10
value: 69.108
- type: mrr_at_100
value: 69.553
- type: mrr_at_1000
value: 69.56099999999999
- type: mrr_at_3
value: 67.417
- type: mrr_at_5
value: 68.382
- type: ndcg_at_1
value: 59.3
- type: ndcg_at_10
value: 73.54
- type: ndcg_at_100
value: 75.652
- type: ndcg_at_1000
value: 75.868
- type: ndcg_at_3
value: 70.074
- type: ndcg_at_5
value: 71.808
- type: precision_at_1
value: 59.3
- type: precision_at_10
value: 8.709999999999999
- type: precision_at_100
value: 0.9690000000000001
- type: precision_at_1000
value: 0.099
- type: precision_at_3
value: 25.867
- type: precision_at_5
value: 16.36
- type: recall_at_1
value: 59.3
- type: recall_at_10
value: 87.1
- type: recall_at_100
value: 96.89999999999999
- type: recall_at_1000
value: 98.6
- type: recall_at_3
value: 77.60000000000001
- type: recall_at_5
value: 81.8
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 84.69999999999999
- type: ap
value: 66.65020528563207
- type: f1
value: 83.00542769081453