Anujgr8's picture
Upload tokenizer
2b3b470 verified
raw
history blame
1.39 kB
{
"!": 1,
"%": 2,
"'": 3,
"-": 4,
".": 5,
"/": 6,
"0": 7,
"1": 8,
"2": 9,
"4": 10,
":": 11,
"B": 12,
"[PAD]": 110,
"[UNK]": 109,
"a": 13,
"b": 14,
"c": 15,
"d": 16,
"f": 17,
"g": 18,
"i": 19,
"l": 20,
"n": 21,
"o": 22,
"p": 23,
"r": 24,
"s": 25,
"t": 26,
"u": 27,
"v": 28,
"w": 29,
"z": 30,
"|": 0,
"“": 31,
"”": 32,
"।": 33,
"ঁ": 34,
"ং": 35,
"ঃ": 36,
"অ": 37,
"আ": 38,
"ই": 39,
"ঈ": 40,
"উ": 41,
"ঋ": 42,
"এ": 43,
"ঐ": 44,
"ও": 45,
"ঔ": 46,
"ক": 47,
"খ": 48,
"গ": 49,
"ঘ": 50,
"ঙ": 51,
"চ": 52,
"ছ": 53,
"জ": 54,
"ঝ": 55,
"ঞ": 56,
"ট": 57,
"ঠ": 58,
"ড": 59,
"ঢ": 60,
"ণ": 61,
"ত": 62,
"থ": 63,
"দ": 64,
"ধ": 65,
"ন": 66,
"প": 67,
"ফ": 68,
"ব": 69,
"ভ": 70,
"ম": 71,
"য": 72,
"র": 73,
"ল": 74,
"শ": 75,
"ষ": 76,
"স": 77,
"হ": 78,
"়": 79,
"া": 80,
"ি": 81,
"ী": 82,
"ু": 83,
"ূ": 84,
"ৃ": 85,
"ে": 86,
"ৈ": 87,
"ো": 88,
"ৌ": 89,
"্": 90,
"ৎ": 91,
"ড়": 92,
"য়": 93,
"০": 94,
"১": 95,
"২": 96,
"৩": 97,
"৪": 98,
"৫": 99,
"৬": 100,
"৭": 101,
"৮": 102,
"৯": 103,
"‌": 104,
"‍": 105,
"‘": 106,
"’": 107,
"…": 108
}