--- tags: - mteb model-index: - name: piccolo-base-zh results: - task: type: STS dataset: type: C-MTEB/AFQMC name: MTEB AFQMC config: default split: validation revision: None metrics: - type: cos_sim_pearson value: 49.16558217326158 - type: cos_sim_spearman value: 51.4049475858823 - type: euclidean_pearson value: 49.85853741070363 - type: euclidean_spearman value: 51.501428092542234 - type: manhattan_pearson value: 49.746099634926296 - type: manhattan_spearman value: 51.41081804320127 - task: type: STS dataset: type: C-MTEB/ATEC name: MTEB ATEC config: default split: test revision: None metrics: - type: cos_sim_pearson value: 52.385361699031854 - type: cos_sim_spearman value: 52.59114913702212 - type: euclidean_pearson value: 54.994530439418355 - type: euclidean_spearman value: 52.54102886188004 - type: manhattan_pearson value: 54.9503071669608 - type: manhattan_spearman value: 52.51465652540901 - task: type: Classification dataset: type: mteb/amazon_reviews_multi name: MTEB AmazonReviewsClassification (zh) config: zh split: test revision: 1399c76144fd37290681b995c656ef9b2e06e26d metrics: - type: accuracy value: 40.236 - type: f1 value: 39.43040092463147 - task: type: STS dataset: type: C-MTEB/BQ name: MTEB BQ config: default split: test revision: None metrics: - type: cos_sim_pearson value: 60.98952187211432 - type: cos_sim_spearman value: 62.68189713123115 - type: euclidean_pearson value: 61.089426749761344 - type: euclidean_spearman value: 62.41743375544581 - type: manhattan_pearson value: 61.14747216341409 - type: manhattan_spearman value: 62.488918956547046 - task: type: Clustering dataset: type: C-MTEB/CLSClusteringP2P name: MTEB CLSClusteringP2P config: default split: test revision: None metrics: - type: v_measure value: 38.36392300667918 - task: type: Clustering dataset: type: C-MTEB/CLSClusteringS2S name: MTEB CLSClusteringS2S config: default split: test revision: None metrics: - type: v_measure value: 35.645927581489175 - task: type: Reranking dataset: type: C-MTEB/CMedQAv1-reranking name: MTEB CMedQAv1 config: default split: test revision: None metrics: - type: map value: 85.25085782849087 - type: mrr value: 87.77154761904762 - task: type: Reranking dataset: type: C-MTEB/CMedQAv2-reranking name: MTEB CMedQAv2 config: default split: test revision: None metrics: - type: map value: 86.15357754080844 - type: mrr value: 88.53547619047617 - task: type: Retrieval dataset: type: C-MTEB/CmedqaRetrieval name: MTEB CmedqaRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 23.683 - type: map_at_10 value: 35.522999999999996 - type: map_at_100 value: 37.456 - type: map_at_1000 value: 37.576 - type: map_at_3 value: 31.584 - type: map_at_5 value: 33.684999999999995 - type: mrr_at_1 value: 36.459 - type: mrr_at_10 value: 44.534 - type: mrr_at_100 value: 45.6 - type: mrr_at_1000 value: 45.647 - type: mrr_at_3 value: 42.186 - type: mrr_at_5 value: 43.482 - type: ndcg_at_1 value: 36.459 - type: ndcg_at_10 value: 42.025 - type: ndcg_at_100 value: 49.754 - type: ndcg_at_1000 value: 51.815999999999995 - type: ndcg_at_3 value: 37.056 - type: ndcg_at_5 value: 38.962 - type: precision_at_1 value: 36.459 - type: precision_at_10 value: 9.485000000000001 - type: precision_at_100 value: 1.567 - type: precision_at_1000 value: 0.183 - type: precision_at_3 value: 21.13 - type: precision_at_5 value: 15.209 - type: recall_at_1 value: 23.683 - type: recall_at_10 value: 52.190999999999995 - type: recall_at_100 value: 84.491 - type: recall_at_1000 value: 98.19600000000001 - type: recall_at_3 value: 37.09 - type: recall_at_5 value: 43.262 - task: type: PairClassification dataset: type: C-MTEB/CMNLI name: MTEB Cmnli config: default split: validation revision: None metrics: - type: cos_sim_accuracy value: 74.20324714371618 - type: cos_sim_ap value: 82.32631646194994 - type: cos_sim_f1 value: 76.64052827073876 - type: cos_sim_precision value: 68.58725761772854 - type: cos_sim_recall value: 86.83656768763151 - type: dot_accuracy value: 70.33072760072159 - type: dot_ap value: 77.46972172609794 - type: dot_f1 value: 73.6668924804026 - type: dot_precision value: 62.84676354029062 - type: dot_recall value: 88.98760813654431 - type: euclidean_accuracy value: 74.78051713770296 - type: euclidean_ap value: 82.65778389584023 - type: euclidean_f1 value: 77.1843623157445 - type: euclidean_precision value: 71.05211406096362 - type: euclidean_recall value: 84.47509936871639 - type: manhattan_accuracy value: 74.76849067949489 - type: manhattan_ap value: 82.55694030572194 - type: manhattan_f1 value: 77.1776459569154 - type: manhattan_precision value: 69.5423855963991 - type: manhattan_recall value: 86.69628244096329 - type: max_accuracy value: 74.78051713770296 - type: max_ap value: 82.65778389584023 - type: max_f1 value: 77.1843623157445 - task: type: Retrieval dataset: type: C-MTEB/CovidRetrieval name: MTEB CovidRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 72.99799999999999 - type: map_at_10 value: 81.271 - type: map_at_100 value: 81.53399999999999 - type: map_at_1000 value: 81.535 - type: map_at_3 value: 80.049 - type: map_at_5 value: 80.793 - type: mrr_at_1 value: 73.13 - type: mrr_at_10 value: 81.193 - type: mrr_at_100 value: 81.463 - type: mrr_at_1000 value: 81.464 - type: mrr_at_3 value: 80.067 - type: mrr_at_5 value: 80.741 - type: ndcg_at_1 value: 73.34 - type: ndcg_at_10 value: 84.503 - type: ndcg_at_100 value: 85.643 - type: ndcg_at_1000 value: 85.693 - type: ndcg_at_3 value: 82.135 - type: ndcg_at_5 value: 83.401 - type: precision_at_1 value: 73.34 - type: precision_at_10 value: 9.536 - type: precision_at_100 value: 1.004 - type: precision_at_1000 value: 0.101 - type: precision_at_3 value: 29.54 - type: precision_at_5 value: 18.398 - type: recall_at_1 value: 72.99799999999999 - type: recall_at_10 value: 94.31 - type: recall_at_100 value: 99.368 - type: recall_at_1000 value: 99.789 - type: recall_at_3 value: 87.935 - type: recall_at_5 value: 90.991 - task: type: Retrieval dataset: type: C-MTEB/DuRetrieval name: MTEB DuRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 26.537 - type: map_at_10 value: 81.292 - type: map_at_100 value: 84.031 - type: map_at_1000 value: 84.066 - type: map_at_3 value: 56.571000000000005 - type: map_at_5 value: 71.082 - type: mrr_at_1 value: 91.2 - type: mrr_at_10 value: 93.893 - type: mrr_at_100 value: 93.955 - type: mrr_at_1000 value: 93.95700000000001 - type: mrr_at_3 value: 93.61699999999999 - type: mrr_at_5 value: 93.767 - type: ndcg_at_1 value: 91.2 - type: ndcg_at_10 value: 88.255 - type: ndcg_at_100 value: 90.813 - type: ndcg_at_1000 value: 91.144 - type: ndcg_at_3 value: 87.435 - type: ndcg_at_5 value: 85.961 - type: precision_at_1 value: 91.2 - type: precision_at_10 value: 42.14 - type: precision_at_100 value: 4.817 - type: precision_at_1000 value: 0.48900000000000005 - type: precision_at_3 value: 78.467 - type: precision_at_5 value: 65.75999999999999 - type: recall_at_1 value: 26.537 - type: recall_at_10 value: 89.262 - type: recall_at_100 value: 97.783 - type: recall_at_1000 value: 99.49799999999999 - type: recall_at_3 value: 58.573 - type: recall_at_5 value: 75.154 - task: type: Retrieval dataset: type: C-MTEB/EcomRetrieval name: MTEB EcomRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 48.5 - type: map_at_10 value: 57.898 - type: map_at_100 value: 58.599000000000004 - type: map_at_1000 value: 58.616 - type: map_at_3 value: 55.1 - type: map_at_5 value: 56.80500000000001 - type: mrr_at_1 value: 48.5 - type: mrr_at_10 value: 57.898 - type: mrr_at_100 value: 58.599000000000004 - type: mrr_at_1000 value: 58.616 - type: mrr_at_3 value: 55.1 - type: mrr_at_5 value: 56.80500000000001 - type: ndcg_at_1 value: 48.5 - type: ndcg_at_10 value: 62.876 - type: ndcg_at_100 value: 66.00200000000001 - type: ndcg_at_1000 value: 66.467 - type: ndcg_at_3 value: 57.162 - type: ndcg_at_5 value: 60.263999999999996 - type: precision_at_1 value: 48.5 - type: precision_at_10 value: 7.870000000000001 - type: precision_at_100 value: 0.927 - type: precision_at_1000 value: 0.096 - type: precision_at_3 value: 21.032999999999998 - type: precision_at_5 value: 14.14 - type: recall_at_1 value: 48.5 - type: recall_at_10 value: 78.7 - type: recall_at_100 value: 92.7 - type: recall_at_1000 value: 96.39999999999999 - type: recall_at_3 value: 63.1 - type: recall_at_5 value: 70.7 - task: type: Classification dataset: type: C-MTEB/IFlyTek-classification name: MTEB IFlyTek config: default split: validation revision: None metrics: - type: accuracy value: 44.34782608695652 - type: f1 value: 36.401426200836205 - task: type: Classification dataset: type: C-MTEB/JDReview-classification name: MTEB JDReview config: default split: test revision: None metrics: - type: accuracy value: 84.25891181988743 - type: ap value: 50.54636280166089 - type: f1 value: 78.55080202541332 - task: type: STS dataset: type: C-MTEB/LCQMC name: MTEB LCQMC config: default split: test revision: None metrics: - type: cos_sim_pearson value: 70.02878561337955 - type: cos_sim_spearman value: 75.39509553139982 - type: euclidean_pearson value: 73.92598696939956 - type: euclidean_spearman value: 75.5471147196853 - type: manhattan_pearson value: 73.88049486090739 - type: manhattan_spearman value: 75.51361990583285 - task: type: Retrieval dataset: type: C-MTEB/MMarcoRetrieval name: MTEB MMarcoRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 64.739 - type: map_at_10 value: 74.039 - type: map_at_100 value: 74.38 - type: map_at_1000 value: 74.39099999999999 - type: map_at_3 value: 72.074 - type: map_at_5 value: 73.29299999999999 - type: mrr_at_1 value: 66.92 - type: mrr_at_10 value: 74.636 - type: mrr_at_100 value: 74.94 - type: mrr_at_1000 value: 74.95 - type: mrr_at_3 value: 72.911 - type: mrr_at_5 value: 73.981 - type: ndcg_at_1 value: 66.92 - type: ndcg_at_10 value: 77.924 - type: ndcg_at_100 value: 79.471 - type: ndcg_at_1000 value: 79.73400000000001 - type: ndcg_at_3 value: 74.17200000000001 - type: ndcg_at_5 value: 76.236 - type: precision_at_1 value: 66.92 - type: precision_at_10 value: 9.5 - type: precision_at_100 value: 1.027 - type: precision_at_1000 value: 0.105 - type: precision_at_3 value: 27.989000000000004 - type: precision_at_5 value: 17.874000000000002 - type: recall_at_1 value: 64.739 - type: recall_at_10 value: 89.324 - type: recall_at_100 value: 96.342 - type: recall_at_1000 value: 98.38900000000001 - type: recall_at_3 value: 79.378 - type: recall_at_5 value: 84.28099999999999 - task: type: Classification dataset: type: mteb/amazon_massive_intent name: MTEB MassiveIntentClassification (zh-CN) config: zh-CN split: test revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 metrics: - type: accuracy value: 68.97108271687962 - type: f1 value: 66.8625981386677 - task: type: Classification dataset: type: mteb/amazon_massive_scenario name: MTEB MassiveScenarioClassification (zh-CN) config: zh-CN split: test revision: 7d571f92784cd94a019292a1f45445077d0ef634 metrics: - type: accuracy value: 73.32212508406187 - type: f1 value: 73.33875034670166 - task: type: Retrieval dataset: type: C-MTEB/MedicalRetrieval name: MTEB MedicalRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 49.0 - type: map_at_10 value: 55.022999999999996 - type: map_at_100 value: 55.550999999999995 - type: map_at_1000 value: 55.608000000000004 - type: map_at_3 value: 53.417 - type: map_at_5 value: 54.372 - type: mrr_at_1 value: 49.3 - type: mrr_at_10 value: 55.176 - type: mrr_at_100 value: 55.703 - type: mrr_at_1000 value: 55.76 - type: mrr_at_3 value: 53.567 - type: mrr_at_5 value: 54.522000000000006 - type: ndcg_at_1 value: 49.0 - type: ndcg_at_10 value: 58.089999999999996 - type: ndcg_at_100 value: 60.988 - type: ndcg_at_1000 value: 62.580999999999996 - type: ndcg_at_3 value: 54.803000000000004 - type: ndcg_at_5 value: 56.508 - type: precision_at_1 value: 49.0 - type: precision_at_10 value: 6.78 - type: precision_at_100 value: 0.8210000000000001 - type: precision_at_1000 value: 0.095 - type: precision_at_3 value: 19.6 - type: precision_at_5 value: 12.58 - type: recall_at_1 value: 49.0 - type: recall_at_10 value: 67.80000000000001 - type: recall_at_100 value: 82.1 - type: recall_at_1000 value: 94.8 - type: recall_at_3 value: 58.8 - type: recall_at_5 value: 62.9 - task: type: Reranking dataset: type: C-MTEB/Mmarco-reranking name: MTEB MMarcoReranking config: default split: dev revision: None metrics: - type: map value: 28.87237408060796 - type: mrr value: 27.83015873015873 - task: type: Classification dataset: type: C-MTEB/MultilingualSentiment-classification name: MTEB MultilingualSentiment config: default split: validation revision: None metrics: - type: accuracy value: 70.25 - type: f1 value: 70.29055400149645 - task: type: PairClassification dataset: type: C-MTEB/OCNLI name: MTEB Ocnli config: default split: validation revision: None metrics: - type: cos_sim_accuracy value: 65.56578234975636 - type: cos_sim_ap value: 70.89354058570412 - type: cos_sim_f1 value: 71.21024370095002 - type: cos_sim_precision value: 58.48032564450475 - type: cos_sim_recall value: 91.02428722280888 - type: dot_accuracy value: 64.86193827828912 - type: dot_ap value: 70.17697803463875 - type: dot_f1 value: 70.68676716917922 - type: dot_precision value: 58.57043719639139 - type: dot_recall value: 89.1235480464625 - type: euclidean_accuracy value: 64.86193827828912 - type: euclidean_ap value: 70.26847152773904 - type: euclidean_f1 value: 70.9984152139461 - type: euclidean_precision value: 56.81674064679771 - type: euclidean_recall value: 94.61457233368532 - type: manhattan_accuracy value: 65.40335679480238 - type: manhattan_ap value: 70.22941558736018 - type: manhattan_f1 value: 71.09712937475423 - type: manhattan_precision value: 56.64160401002506 - type: manhattan_recall value: 95.45934530095037 - type: max_accuracy value: 65.56578234975636 - type: max_ap value: 70.89354058570412 - type: max_f1 value: 71.21024370095002 - task: type: Classification dataset: type: C-MTEB/OnlineShopping-classification name: MTEB OnlineShopping config: default split: test revision: None metrics: - type: accuracy value: 89.92999999999999 - type: ap value: 87.16059195012956 - type: f1 value: 89.90917477839415 - task: type: STS dataset: type: C-MTEB/PAWSX name: MTEB PAWSX config: default split: test revision: None metrics: - type: cos_sim_pearson value: 27.74161502387672 - type: cos_sim_spearman value: 31.58353529723325 - type: euclidean_pearson value: 32.43729673844635 - type: euclidean_spearman value: 31.59527486602242 - type: manhattan_pearson value: 32.37467059678786 - type: manhattan_spearman value: 31.44408004951894 - task: type: STS dataset: type: C-MTEB/QBQTC name: MTEB QBQTC config: default split: test revision: None metrics: - type: cos_sim_pearson value: 36.233749845501194 - type: cos_sim_spearman value: 36.47808586229587 - type: euclidean_pearson value: 32.663447466546806 - type: euclidean_spearman value: 34.45830454037139 - type: manhattan_pearson value: 32.80239212096335 - type: manhattan_spearman value: 34.581060433895125 - task: type: STS dataset: type: mteb/sts22-crosslingual-sts name: MTEB STS22 (zh) config: zh split: test revision: None metrics: - type: cos_sim_pearson value: 63.05131937664673 - type: cos_sim_spearman value: 66.51353746725948 - type: euclidean_pearson value: 61.24016998745561 - type: euclidean_spearman value: 66.07115266049276 - type: manhattan_pearson value: 64.55660243659054 - type: manhattan_spearman value: 66.80282149562386 - task: type: STS dataset: type: C-MTEB/STSB name: MTEB STSB config: default split: test revision: None metrics: - type: cos_sim_pearson value: 70.45533692882996 - type: cos_sim_spearman value: 70.6045637565602 - type: euclidean_pearson value: 72.75588977483554 - type: euclidean_spearman value: 73.36630581886473 - type: manhattan_pearson value: 72.72517409326954 - type: manhattan_spearman value: 73.35358940437355 - task: type: Reranking dataset: type: C-MTEB/T2Reranking name: MTEB T2Reranking config: default split: dev revision: None metrics: - type: map value: 66.45779474032288 - type: mrr value: 76.0782192023729 - task: type: Retrieval dataset: type: C-MTEB/T2Retrieval name: MTEB T2Retrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 26.458 - type: map_at_10 value: 74.355 - type: map_at_100 value: 78.158 - type: map_at_1000 value: 78.233 - type: map_at_3 value: 52.2 - type: map_at_5 value: 64.14 - type: mrr_at_1 value: 88.37 - type: mrr_at_10 value: 91.117 - type: mrr_at_100 value: 91.231 - type: mrr_at_1000 value: 91.23599999999999 - type: mrr_at_3 value: 90.645 - type: mrr_at_5 value: 90.948 - type: ndcg_at_1 value: 88.37 - type: ndcg_at_10 value: 82.384 - type: ndcg_at_100 value: 86.431 - type: ndcg_at_1000 value: 87.163 - type: ndcg_at_3 value: 83.993 - type: ndcg_at_5 value: 82.411 - type: precision_at_1 value: 88.37 - type: precision_at_10 value: 41.131 - type: precision_at_100 value: 4.9799999999999995 - type: precision_at_1000 value: 0.515 - type: precision_at_3 value: 73.651 - type: precision_at_5 value: 61.634 - type: recall_at_1 value: 26.458 - type: recall_at_10 value: 81.3 - type: recall_at_100 value: 94.342 - type: recall_at_1000 value: 98.103 - type: recall_at_3 value: 54.020999999999994 - type: recall_at_5 value: 67.781 - task: type: Classification dataset: type: C-MTEB/TNews-classification name: MTEB TNews config: default split: validation revision: None metrics: - type: accuracy value: 46.814 - type: f1 value: 45.580027683507666 - task: type: Clustering dataset: type: C-MTEB/ThuNewsClusteringP2P name: MTEB ThuNewsClusteringP2P config: default split: test revision: None metrics: - type: v_measure value: 61.43613064816144 - task: type: Clustering dataset: type: C-MTEB/ThuNewsClusteringS2S name: MTEB ThuNewsClusteringS2S config: default split: test revision: None metrics: - type: v_measure value: 53.01838461793776 - task: type: Retrieval dataset: type: C-MTEB/VideoRetrieval name: MTEB VideoRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 59.3 - type: map_at_10 value: 69.158 - type: map_at_100 value: 69.60300000000001 - type: map_at_1000 value: 69.611 - type: map_at_3 value: 67.467 - type: map_at_5 value: 68.432 - type: mrr_at_1 value: 59.199999999999996 - type: mrr_at_10 value: 69.108 - type: mrr_at_100 value: 69.553 - type: mrr_at_1000 value: 69.56099999999999 - type: mrr_at_3 value: 67.417 - type: mrr_at_5 value: 68.382 - type: ndcg_at_1 value: 59.3 - type: ndcg_at_10 value: 73.54 - type: ndcg_at_100 value: 75.652 - type: ndcg_at_1000 value: 75.868 - type: ndcg_at_3 value: 70.074 - type: ndcg_at_5 value: 71.808 - type: precision_at_1 value: 59.3 - type: precision_at_10 value: 8.709999999999999 - type: precision_at_100 value: 0.9690000000000001 - type: precision_at_1000 value: 0.099 - type: precision_at_3 value: 25.867 - type: precision_at_5 value: 16.36 - type: recall_at_1 value: 59.3 - type: recall_at_10 value: 87.1 - type: recall_at_100 value: 96.89999999999999 - type: recall_at_1000 value: 98.6 - type: recall_at_3 value: 77.60000000000001 - type: recall_at_5 value: 81.8 - task: type: Classification dataset: type: C-MTEB/waimai-classification name: MTEB Waimai config: default split: test revision: None metrics: - type: accuracy value: 84.69999999999999 - type: ap value: 66.65020528563207 - type: f1 value: 83.00542769081453 --- ## piccolo-base-zh