Anujgr8's picture
Upload tokenizer
57c1c7f verified
{
"[PAD]": 84,
"[UNK]": 83,
"_": 1,
"e": 2,
"g": 3,
"l": 4,
"r": 5,
"t": 6,
"u": 7,
"|": 0,
"ઁ": 8,
"ં": 9,
"ઃ": 10,
"અ": 11,
"આ": 12,
"ઇ": 13,
"ઈ": 14,
"ઉ": 15,
"ઊ": 16,
"ઋ": 17,
"એ": 18,
"ઐ": 19,
"ઑ": 20,
"ઓ": 21,
"ઔ": 22,
"ક": 23,
"ખ": 24,
"ગ": 25,
"ઘ": 26,
"ચ": 27,
"છ": 28,
"જ": 29,
"ઝ": 30,
"ઞ": 31,
"ટ": 32,
"ઠ": 33,
"ડ": 34,
"ઢ": 35,
"ણ": 36,
"ત": 37,
"થ": 38,
"દ": 39,
"ધ": 40,
"ન": 41,
"પ": 42,
"ફ": 43,
"બ": 44,
"ભ": 45,
"મ": 46,
"ય": 47,
"ર": 48,
"લ": 49,
"ળ": 50,
"વ": 51,
"શ": 52,
"ષ": 53,
"સ": 54,
"હ": 55,
"઼": 56,
"ા": 57,
"િ": 58,
"ી": 59,
"ુ": 60,
"ૂ": 61,
"ૃ": 62,
"ૅ": 63,
"ે": 64,
"ૈ": 65,
"ૉ": 66,
"ો": 67,
"ૌ": 68,
"્": 69,
"ૠ": 70,
"ૢ": 71,
"૦": 72,
"૧": 73,
"૩": 74,
"૫": 75,
"૬": 76,
"૭": 77,
"૮": 78,
"૯": 79,
"‌": 80,
"‍": 81,
"’": 82
}