Upload tokenizer

added_tokens.json

@@ -1,5 +1,4 @@
 {
-  "</s>": 103,
-  "<s>": 102,
-  "[PAD]": 101
+  "</s>": 827,
+  "<s>": 826
 }

tokenizer_config.json

@@ -1,6 +1,6 @@
 {
   "added_tokens_decoder": {
-    "100": {
+    "824": {
       "content": "[UNK]",
       "lstrip": true,
       "normalized": false,
@@ -8,7 +8,7 @@
       "single_word": false,
       "special": false
     },
-    "101": {
+    "825": {
       "content": "[PAD]",
       "lstrip": true,
       "normalized": false,
@@ -16,7 +16,7 @@
       "single_word": false,
       "special": false
     },
-    "102": {
+    "826": {
       "content": "<s>",
       "lstrip": false,
       "normalized": true,
@@ -24,7 +24,7 @@
       "single_word": false,
       "special": true
     },
-    "103": {
+    "827": {
       "content": "</s>",
       "lstrip": false,
       "normalized": true,

vocab.json

@@ -1,104 +1,828 @@
 {
-  "&": 12,
-  "'": 20,
-  "[PAD]": 101,
-  "[UNK]": 100,
-  "a": 40,
-  "b": 33,
-  "c": 65,
-  "d": 41,
-  "e": 84,
-  "f": 14,
-  "g": 26,
-  "h": 13,
-  "i": 87,
-  "j": 91,
-  "k": 28,
-  "l": 86,
-  "m": 53,
-  "n": 5,
-  "o": 7,
-  "p": 39,
-  "q": 30,
-  "r": 19,
-  "s": 71,
-  "t": 75,
-  "u": 17,
-  "v": 46,
-  "w": 90,
-  "x": 16,
-  "y": 10,
-  "z": 97,
-  "|": 43,
-  "ँ": 32,
-  "ं": 23,
-  "ः": 38,
-  "अ": 42,
-  "आ": 2,
-  "इ": 36,
-  "ई": 80,
-  "उ": 62,
-  "ऊ": 100,
-  "ऋ": 24,
-  "ए": 98,
-  "ऐ": 55,
-  "ऑ": 47,
-  "ओ": 72,
-  "औ": 18,
-  "क": 63,
-  "ख": 1,
-  "ग": 9,
-  "घ": 15,
-  "च": 45,
-  "छ": 6,
-  "ज": 64,
-  "झ": 11,
-  "ञ": 4,
-  "ट": 34,
-  "ठ": 25,
-  "ड": 59,
-  "ढ": 37,
-  "ण": 27,
-  "त": 31,
-  "थ": 88,
-  "द": 48,
-  "ध": 8,
-  "न": 95,
-  "प": 92,
-  "फ": 57,
-  "ब": 68,
-  "भ": 99,
-  "म": 54,
-  "य": 73,
-  "र": 61,
-  "ल": 79,
-  "व": 85,
-  "श": 22,
-  "ष": 96,
-  "स": 51,
-  "ह": 50,
-  "़": 44,
-  "ा": 66,
-  "ि": 74,
-  "ी": 49,
-  "ु": 70,
-  "ू": 89,
-  "ृ": 69,
-  "ॅ": 76,
-  "े": 21,
-  "ै": 0,
-  "ॉ": 58,
-  "ो": 67,
-  "ौ": 94,
-  "्": 35,
-  "क़": 56,
-  "ख़": 78,
-  "ग़": 82,
-  "ज़": 29,
-  "ड़": 83,
-  "ढ़": 77,
-  "फ़": 93,
-  "।": 3,
-  "–": 52,
-  "’": 81
+  "[PAD]": 825,
+  "[UNK]": 824,
+  "d": 552,
+  "|": 192,
+  "、": 247,
+  "。": 337,
+  "《": 458,
+  "》": 378,
+  "「": 133,
+  "」": 713,
+  "㗎": 202,
+  "一": 568,
+  "丈": 6,
+  "三": 223,
+  "上": 121,
+  "下": 60,
+  "不": 768,
+  "且": 444,
+  "世": 706,
+  "並": 259,
+  "中": 188,
+  "丹": 717,
+  "主": 185,
+  "久": 561,
+  "之": 12,
+  "乎": 69,
+  "乜": 490,
+  "九": 120,
+  "也": 533,
+  "乾": 658,
+  "了": 107,
+  "事": 439,
+  "二": 141,
+  "于": 508,
+  "些": 285,
+  "亞": 631,
+  "交": 753,
+  "京": 611,
+  "人": 318,
+  "什": 65,
+  "今": 243,
+  "仔": 272,
+  "他": 441,
+  "代": 551,
+  "以": 130,
+  "仲": 662,
+  "件": 149,
+  "任": 655,
+  "份": 295,
+  "企": 750,
+  "伙": 248,
+  "似": 710,
+  "但": 245,
+  "位": 530,
+  "住": 689,
+  "何": 288,
+  "作": 691,
+  "你": 609,
+  "佢": 735,
+  "使": 403,
+  "來": 600,
+  "供": 37,
+  "依": 240,
+  "便": 368,
+  "係": 423,
+  "保": 524,
+  "信": 674,
+  "俾": 86,
+  "個": 352,
+  "們": 514,
+  "借": 819,
+  "值": 415,
+  "假": 437,
+  "做": 421,
+  "傑": 422,
+  "傢": 389,
+  "債": 462,
+  "像": 438,
+  "億": 711,
+  "儲": 163,
+  "元": 145,
+  "充": 712,
+  "兇": 183,
+  "先": 529,
+  "光": 109,
+  "免": 675,
+  "入": 161,
+  "內": 428,
+  "全": 88,
+  "兩": 292,
+  "八": 550,
+  "公": 803,
+  "六": 209,
+  "共": 181,
+  "其": 637,
+  "冇": 613,
+  "再": 698,
+  "冤": 641,
+  "冬": 237,
+  "冷": 195,
+  "出": 657,
+  "分": 653,
+  "切": 677,
+  "別": 410,
+  "到": 20,
+  "制": 743,
+  "刷": 46,
+  "刻": 426,
+  "前": 455,
+  "剛": 635,
+  "功": 112,
+  "加": 50,
+  "助": 395,
+  "勇": 106,
+  "動": 302,
+  "務": 526,
+  "包": 186,
+  "化": 119,
+  "北": 607,
+  "匯": 510,
+  "區": 227,
+  "十": 481,
+  "協": 560,
+  "博": 350,
+  "卡": 562,
+  "即": 362,
+  "卻": 708,
+  "去": 757,
+  "又": 427,
+  "及": 605,
+  "友": 95,
+  "反": 738,
+  "取": 135,
+  "受": 305,
+  "口": 336,
+  "古": 399,
+  "句": 173,
+  "另": 235,
+  "只": 206,
+  "叫": 277,
+  "可": 257,
+  "台": 75,
+  "吃": 180,
+  "各": 773,
+  "合": 114,
+  "同": 10,
+  "名": 723,
+  "向": 585,
+  "吓": 55,
+  "吞": 299,
+  "否": 225,
+  "呀": 212,
+  "告": 129,
+  "呢": 412,
+  "味": 659,
+  "呼": 816,
+  "命": 810,
+  "咀": 384,
+  "咁": 624,
+  "和": 468,
+  "咗": 286,
+  "咩": 105,
+  "咪": 267,
+  "哋": 610,
+  "員": 787,
+  "哥": 58,
+  "哩": 495,
+  "唔": 747,
+  "售": 749,
+  "唱": 474,
+  "啊": 392,
+  "問": 343,
+  "啤": 47,
+  "啦": 615,
+  "啫": 453,
+  "啱": 556,
+  "啲": 376,
+  "喇": 11,
+  "喊": 280,
+  "喎": 446,
+  "喜": 29,
+  "喝": 139,
+  "喺": 396,
+  "嗰": 39,
+  "嗱": 582,
+  "嘅": 527,
+  "嘢": 213,
+  "嘴": 23,
+  "噃": 564,
+  "器": 52,
+  "噬": 275,
+  "嚟": 449,
+  "嚿": 127,
+  "四": 532,
+  "回": 356,
+  "因": 193,
+  "國": 38,
+  "圍": 790,
+  "園": 366,
+  "團": 494,
+  "土": 281,
+  "在": 669,
+  "地": 4,
+  "坐": 236,
+  "坑": 330,
+  "型": 766,
+  "埋": 496,
+  "城": 661,
+  "基": 387,
+  "堅": 598,
+  "報": 440,
+  "場": 172,
+  "塊": 545,
+  "墓": 593,
+  "增": 74,
+  "壞": 602,
+  "士": 406,
+  "外": 664,
+  "多": 633,
+  "夠": 639,
+  "大": 469,
+  "天": 381,
+  "奈": 159,
+  "奮": 394,
+  "她": 357,
+  "好": 16,
+  "如": 795,
+  "妙": 413,
+  "妹": 164,
+  "姐": 813,
+  "姑": 62,
+  "委": 264,
+  "婆": 520,
+  "嬲": 703,
+  "子": 408,
+  "字": 342,
+  "存": 500,
+  "學": 459,
+  "它": 140,
+  "完": 594,
+  "定": 184,
+  "宜": 205,
+  "客": 683,
+  "家": 503,
+  "容": 113,
+  "寃": 43,
+  "實": 805,
+  "審": 92,
+  "封": 274,
+  "射": 688,
+  "將": 104,
+  "對": 621,
+  "導": 98,
+  "小": 197,
+  "尚": 781,
+  "就": 475,
+  "尼": 116,
+  "局": 632,
+  "居": 756,
+  "屋": 377,
+  "屎": 759,
+  "展": 547,
+  "屯": 152,
+  "山": 630,
+  "島": 167,
+  "嶼": 358,
+  "工": 335,
+  "左": 779,
+  "差": 409,
+  "己": 686,
+  "已": 271,
+  "巴": 313,
+  "市": 126,
+  "常": 687,
+  "幕": 48,
+  "幫": 179,
+  "平": 760,
+  "年": 588,
+  "幾": 622,
+  "底": 647,
+  "店": 341,
+  "府": 731,
+  "度": 333,
+  "庫": 315,
+  "康": 517,
+  "建": 263,
+  "弊": 190,
+  "式": 303,
+  "張": 668,
+  "強": 772,
+  "彈": 729,
+  "形": 162,
+  "彩": 3,
+  "影": 154,
+  "往": 217,
+  "很": 604,
+  "律": 87,
+  "後": 591,
+  "徑": 809,
+  "得": 136,
+  "心": 528,
+  "快": 775,
+  "念": 726,
+  "怒": 45,
+  "怕": 745,
+  "思": 375,
+  "怪": 461,
+  "息": 788,
+  "情": 767,
+  "惠": 784,
+  "想": 146,
+  "意": 802,
+  "愛": 433,
+  "感": 370,
+  "慘": 53,
+  "慢": 466,
+  "慧": 56,
+  "慮": 380,
+  "憑": 746,
+  "憶": 764,
+  "懵": 19,
+  "懷": 420,
+  "成": 513,
+  "我": 450,
+  "或": 241,
+  "戰": 817,
+  "戲": 608,
+  "戶": 82,
+  "所": 290,
+  "手": 644,
+  "才": 783,
+  "扒": 230,
+  "打": 595,
+  "托": 108,
+  "承": 273,
+  "把": 344,
+  "投": 531,
+  "抗": 525,
+  "折": 124,
+  "抵": 676,
+  "拉": 454,
+  "拌": 483,
+  "招": 718,
+  "拜": 634,
+  "拿": 770,
+  "按": 404,
+  "捨": 83,
+  "掂": 601,
+  "排": 374,
+  "接": 666,
+  "控": 211,
+  "推": 208,
+  "提": 28,
+  "揚": 463,
+  "揪": 73,
+  "搞": 80,
+  "搭": 436,
+  "撚": 623,
+  "撲": 616,
+  "擊": 118,
+  "擔": 284,
+  "擴": 36,
+  "攪": 435,
+  "支": 606,
+  "攰": 820,
+  "攻": 160,
+  "放": 1,
+  "政": 581,
+  "救": 266,
+  "散": 651,
+  "敵": 692,
+  "數": 0,
+  "文": 519,
+  "料": 229,
+  "新": 570,
+  "斷": 216,
+  "方": 89,
+  "於": 252,
+  "施": 518,
+  "旅": 171,
+  "族": 640,
+  "既": 799,
+  "日": 558,
+  "旦": 419,
+  "早": 627,
+  "旬": 702,
+  "旺": 549,
+  "明": 291,
+  "星": 447,
+  "昨": 93,
+  "是": 7,
+  "時": 300,
+  "晚": 575,
+  "智": 191,
+  "曉": 91,
+  "書": 569,
+  "最": 492,
+  "會": 798,
+  "月": 255,
+  "有": 246,
+  "朋": 736,
+  "服": 383,
+  "朗": 24,
+  "望": 182,
+  "期": 150,
+  "木": 535,
+  "未": 792,
+  "本": 684,
+  "村": 361,
+  "杯": 823,
+  "枉": 619,
+  "果": 233,
+  "枝": 417,
+  "架": 555,
+  "柏": 346,
+  "柔": 429,
+  "柴": 349,
+  "校": 434,
+  "桂": 111,
+  "案": 100,
+  "梅": 814,
+  "梨": 249,
+  "棗": 786,
+  "棚": 289,
+  "椒": 589,
+  "極": 153,
+  "榜": 201,
+  "榮": 477,
+  "樂": 78,
+  "樣": 721,
+  "樽": 327,
+  "機": 782,
+  "檢": 402,
+  "檸": 822,
+  "欠": 670,
+  "次": 797,
+  "歡": 448,
+  "步": 572,
+  "每": 701,
+  "比": 5,
+  "毫": 360,
+  "氣": 18,
+  "水": 323,
+  "永": 656,
+  "求": 665,
+  "池": 57,
+  "沒": 521,
+  "沙": 584,
+  "法": 54,
+  "波": 625,
+  "泰": 470,
+  "泳": 125,
+  "洞": 671,
+  "派": 61,
+  "海": 573,
+  "消": 332,
+  "涌": 443,
+  "涼": 354,
+  "淋": 559,
+  "淨": 30,
+  "淫": 347,
+  "深": 667,
+  "渡": 586,
+  "港": 590,
+  "游": 250,
+  "源": 541,
+  "漏": 2,
+  "潛": 704,
+  "澳": 620,
+  "濃": 806,
+  "濛": 279,
+  "瀝": 97,
+  "灣": 128,
+  "炆": 326,
+  "為": 543,
+  "無": 33,
+  "然": 646,
+  "煎": 331,
+  "照": 629,
+  "煲": 110,
+  "煽": 563,
+  "燒": 724,
+  "營": 626,
+  "爆": 334,
+  "爬": 539,
+  "爲": 15,
+  "父": 158,
+  "爽": 363,
+  "版": 157,
+  "牙": 71,
+  "牛": 800,
+  "物": 432,
+  "特": 132,
+  "獎": 70,
+  "獨": 405,
+  "王": 66,
+  "玩": 226,
+  "珠": 324,
+  "班": 411,
+  "現": 681,
+  "球": 719,
+  "理": 79,
+  "瑧": 498,
+  "環": 142,
+  "璽": 714,
+  "瓏": 636,
+  "瓦": 359,
+  "生": 509,
+  "產": 77,
+  "用": 294,
+  "田": 516,
+  "由": 174,
+  "男": 733,
+  "畀": 254,
+  "界": 400,
+  "畐": 382,
+  "留": 194,
+  "略": 553,
+  "疆": 214,
+  "瘀": 583,
+  "發": 652,
+  "白": 253,
+  "百": 506,
+  "的": 388,
+  "盒": 709,
+  "監": 566,
+  "直": 728,
+  "相": 262,
+  "省": 778,
+  "看": 177,
+  "真": 680,
+  "眼": 740,
+  "眾": 102,
+  "睇": 122,
+  "睜": 144,
+  "知": 430,
+  "石": 628,
+  "研": 599,
+  "破": 578,
+  "碼": 737,
+  "磡": 464,
+  "礎": 425,
+  "神": 476,
+  "票": 755,
+  "禮": 115,
+  "科": 769,
+  "程": 391,
+  "稍": 452,
+  "種": 203,
+  "稱": 204,
+  "積": 25,
+  "空": 386,
+  "窩": 690,
+  "立": 21,
+  "站": 312,
+  "竹": 699,
+  "笑": 322,
+  "等": 282,
+  "策": 761,
+  "算": 456,
+  "粒": 807,
+  "精": 41,
+  "糕": 320,
+  "系": 311,
+  "紀": 251,
+  "約": 306,
+  "紅": 187,
+  "紙": 612,
+  "級": 59,
+  "索": 497,
+  "累": 234,
+  "細": 537,
+  "終": 222,
+  "組": 369,
+  "結": 309,
+  "絡": 407,
+  "給": 316,
+  "經": 143,
+  "網": 645,
+  "緊": 307,
+  "總": 13,
+  "繼": 238,
+  "續": 663,
+  "羅": 265,
+  "羊": 269,
+  "美": 287,
+  "群": 771,
+  "義": 742,
+  "習": 103,
+  "翠": 67,
+  "耀": 672,
+  "老": 131,
+  "考": 94,
+  "者": 218,
+  "而": 579,
+  "耐": 617,
+  "耳": 317,
+  "聖": 365,
+  "聞": 808,
+  "聲": 465,
+  "聽": 705,
+  "胃": 777,
+  "背": 165,
+  "能": 774,
+  "脹": 220,
+  "腍": 748,
+  "腦": 178,
+  "腩": 451,
+  "腸": 587,
+  "膠": 301,
+  "膽": 654,
+  "臣": 293,
+  "自": 501,
+  "至": 207,
+  "致": 351,
+  "舅": 431,
+  "舊": 618,
+  "般": 557,
+  "色": 821,
+  "花": 596,
+  "苑": 597,
+  "茂": 499,
+  "茄": 390,
+  "茶": 580,
+  "荔": 811,
+  "莫": 796,
+  "菜": 685,
+  "菩": 156,
+  "華": 379,
+  "萊": 81,
+  "萬": 355,
+  "落": 147,
+  "著": 175,
+  "蓉": 99,
+  "蔔": 35,
+  "薈": 707,
+  "藥": 648,
+  "蘿": 196,
+  "處": 482,
+  "號": 679,
+  "蝦": 31,
+  "行": 504,
+  "街": 283,
+  "衛": 321,
+  "表": 155,
+  "裏": 328,
+  "補": 22,
+  "裝": 8,
+  "裡": 260,
+  "襯": 256,
+  "西": 76,
+  "要": 314,
+  "見": 791,
+  "視": 345,
+  "親": 398,
+  "覺": 804,
+  "角": 68,
+  "解": 571,
+  "言": 536,
+  "計": 650,
+  "討": 741,
+  "記": 693,
+  "設": 542,
+  "許": 242,
+  "訴": 297,
+  "評": 169,
+  "試": 487,
+  "話": 9,
+  "該": 219,
+  "認": 101,
+  "誕": 416,
+  "語": 17,
+  "說": 734,
+  "説": 592,
+  "請": 682,
+  "論": 739,
+  "諗": 325,
+  "謁": 484,
+  "講": 123,
+  "謝": 166,
+  "證": 232,
+  "識": 170,
+  "警": 373,
+  "變": 457,
+  "讓": 200,
+  "豆": 40,
+  "象": 789,
+  "豪": 812,
+  "豬": 117,
+  "貓": 522,
+  "貝": 638,
+  "財": 540,
+  "貪": 319,
+  "貴": 780,
+  "買": 567,
+  "資": 296,
+  "賢": 348,
+  "賣": 215,
+  "質": 278,
+  "賽": 138,
+  "贏": 649,
+  "走": 548,
+  "起": 727,
+  "趁": 424,
+  "超": 776,
+  "足": 758,
+  "跑": 491,
+  "路": 14,
+  "踢": 90,
+  "蹄": 239,
+  "身": 700,
+  "車": 673,
+  "輕": 258,
+  "轉": 445,
+  "轍": 534,
+  "近": 276,
+  "返": 26,
+  "迪": 353,
+  "退": 725,
+  "送": 554,
+  "逐": 793,
+  "這": 473,
+  "通": 695,
+  "逢": 308,
+  "進": 231,
+  "遇": 544,
+  "運": 574,
+  "過": 210,
+  "道": 505,
+  "達": 538,
+  "遠": 480,
+  "適": 485,
+  "還": 715,
+  "邊": 42,
+  "那": 678,
+  "郁": 603,
+  "部": 168,
+  "郵": 489,
+  "都": 414,
+  "鄉": 523,
+  "配": 199,
+  "酒": 732,
+  "酥": 642,
+  "醒": 372,
+  "采": 298,
+  "里": 49,
+  "重": 515,
+  "量": 51,
+  "金": 785,
+  "鈴": 244,
+  "銀": 385,
+  "銅": 32,
+  "錯": 577,
+  "鍾": 460,
+  "鏰": 493,
+  "鐘": 148,
+  "鐵": 261,
+  "鑫": 63,
+  "鑼": 694,
+  "長": 502,
+  "門": 763,
+  "開": 660,
+  "間": 815,
+  "關": 722,
+  "阻": 472,
+  "附": 818,
+  "院": 364,
+  "陣": 371,
+  "陰": 401,
+  "陷": 576,
+  "隔": 486,
+  "隨": 367,
+  "隻": 270,
+  "雅": 304,
+  "雖": 224,
+  "雞": 189,
+  "離": 765,
+  "難": 393,
+  "雪": 546,
+  "零": 697,
+  "電": 751,
+  "需": 339,
+  "靚": 134,
+  "非": 85,
+  "面": 744,
+  "預": 754,
+  "頒": 467,
+  "頭": 310,
+  "題": 643,
+  "願": 27,
+  "類": 752,
+  "飛": 340,
+  "食": 329,
+  "飯": 762,
+  "飽": 84,
+  "餅": 44,
+  "養": 64,
+  "餵": 176,
+  "香": 479,
+  "馬": 488,
+  "驗": 512,
+  "骨": 794,
+  "體": 507,
+  "高": 397,
+  "髮": 151,
+  "鬧": 34,
+  "鬼": 471,
+  "魔": 418,
+  "魚": 696,
+  "鮮": 614,
+  "鹽": 137,
+  "麵": 511,
+  "麼": 478,
+  "黃": 72,
+  "黎": 268,
+  "黐": 720,
+  "黑": 565,
+  "點": 442,
+  "黨": 730,
+  "鼠": 228,
+  "鼻": 221,
+  "齊": 96,
+  "龍": 338,
+  "！": 716,
+  "，": 198,
+  "？": 801
 }