ndeclarke's picture
Upload tokenizer
4e5edfa verified
raw
history blame
4.91 kB
{
"tam-128": {
"&": 1,
"[PAD]": 53,
"[UNK]": 52,
"_": 2,
"|": 0,
"ஃ": 3,
"அ": 4,
"ஆ": 5,
"இ": 6,
"ஈ": 7,
"உ": 8,
"ஊ": 9,
"எ": 10,
"ஏ": 11,
"ஐ": 12,
"ஒ": 13,
"ஓ": 14,
"ஔ": 15,
"க": 16,
"ங": 17,
"ச": 18,
"ஜ": 19,
"ஞ": 20,
"ட": 21,
"ண": 22,
"த": 23,
"ந": 24,
"ன": 25,
"ப": 26,
"ம": 27,
"ய": 28,
"ர": 29,
"ற": 30,
"ல": 31,
"ள": 32,
"ழ": 33,
"வ": 34,
"ஷ": 35,
"ஸ": 36,
"ஹ": 37,
"ா": 38,
"ி": 39,
"ீ": 40,
"ு": 41,
"ூ": 42,
"ெ": 43,
"ே": 44,
"ை": 45,
"ொ": 46,
"ோ": 47,
"ௌ": 48,
"்": 49,
"ௗ": 50,
"ഥ": 51
},
"tam-2048": {
"&": 1,
"[PAD]": 53,
"[UNK]": 52,
"_": 2,
"|": 0,
"ஃ": 3,
"அ": 4,
"ஆ": 5,
"இ": 6,
"ஈ": 7,
"உ": 8,
"ஊ": 9,
"எ": 10,
"ஏ": 11,
"ஐ": 12,
"ஒ": 13,
"ஓ": 14,
"ஔ": 15,
"க": 16,
"ங": 17,
"ச": 18,
"ஜ": 19,
"ஞ": 20,
"ட": 21,
"ண": 22,
"த": 23,
"ந": 24,
"ன": 25,
"ப": 26,
"ம": 27,
"ய": 28,
"ர": 29,
"ற": 30,
"ல": 31,
"ள": 32,
"ழ": 33,
"வ": 34,
"ஷ": 35,
"ஸ": 36,
"ஹ": 37,
"ா": 38,
"ி": 39,
"ீ": 40,
"ு": 41,
"ூ": 42,
"ெ": 43,
"ே": 44,
"ை": 45,
"ொ": 46,
"ோ": 47,
"ௌ": 48,
"்": 49,
"ௗ": 50,
"ഥ": 51
},
"tam-32": {
"&": 1,
"[PAD]": 53,
"[UNK]": 52,
"_": 2,
"|": 0,
"ஃ": 3,
"அ": 4,
"ஆ": 5,
"இ": 6,
"ஈ": 7,
"உ": 8,
"ஊ": 9,
"எ": 10,
"ஏ": 11,
"ஐ": 12,
"ஒ": 13,
"ஓ": 14,
"ஔ": 15,
"க": 16,
"ங": 17,
"ச": 18,
"ஜ": 19,
"ஞ": 20,
"ட": 21,
"ண": 22,
"த": 23,
"ந": 24,
"ன": 25,
"ப": 26,
"ம": 27,
"ய": 28,
"ர": 29,
"ற": 30,
"ல": 31,
"ள": 32,
"ழ": 33,
"வ": 34,
"ஷ": 35,
"ஸ": 36,
"ஹ": 37,
"ா": 38,
"ி": 39,
"ீ": 40,
"ு": 41,
"ூ": 42,
"ெ": 43,
"ே": 44,
"ை": 45,
"ொ": 46,
"ோ": 47,
"ௌ": 48,
"்": 49,
"ௗ": 50,
"ഥ": 51
},
"tam-32768": {
"&": 1,
"[PAD]": 53,
"[UNK]": 52,
"_": 2,
"|": 0,
"ஃ": 3,
"அ": 4,
"ஆ": 5,
"இ": 6,
"ஈ": 7,
"உ": 8,
"ஊ": 9,
"எ": 10,
"ஏ": 11,
"ஐ": 12,
"ஒ": 13,
"ஓ": 14,
"ஔ": 15,
"க": 16,
"ங": 17,
"ச": 18,
"ஜ": 19,
"ஞ": 20,
"ட": 21,
"ண": 22,
"த": 23,
"ந": 24,
"ன": 25,
"ப": 26,
"ம": 27,
"ய": 28,
"ர": 29,
"ற": 30,
"ல": 31,
"ள": 32,
"ழ": 33,
"வ": 34,
"ஷ": 35,
"ஸ": 36,
"ஹ": 37,
"ா": 38,
"ி": 39,
"ீ": 40,
"ு": 41,
"ூ": 42,
"ெ": 43,
"ே": 44,
"ை": 45,
"ொ": 46,
"ோ": 47,
"ௌ": 48,
"்": 49,
"ௗ": 50,
"ഥ": 51
},
"tam-512": {
"&": 1,
"[PAD]": 53,
"[UNK]": 52,
"_": 2,
"|": 0,
"ஃ": 3,
"அ": 4,
"ஆ": 5,
"இ": 6,
"ஈ": 7,
"உ": 8,
"ஊ": 9,
"எ": 10,
"ஏ": 11,
"ஐ": 12,
"ஒ": 13,
"ஓ": 14,
"ஔ": 15,
"க": 16,
"ங": 17,
"ச": 18,
"ஜ": 19,
"ஞ": 20,
"ட": 21,
"ண": 22,
"த": 23,
"ந": 24,
"ன": 25,
"ப": 26,
"ம": 27,
"ய": 28,
"ர": 29,
"ற": 30,
"ல": 31,
"ள": 32,
"ழ": 33,
"வ": 34,
"ஷ": 35,
"ஸ": 36,
"ஹ": 37,
"ா": 38,
"ி": 39,
"ீ": 40,
"ு": 41,
"ூ": 42,
"ெ": 43,
"ே": 44,
"ை": 45,
"ொ": 46,
"ோ": 47,
"ௌ": 48,
"்": 49,
"ௗ": 50,
"ഥ": 51
},
"tam-8192": {
"&": 1,
"[PAD]": 53,
"[UNK]": 52,
"_": 2,
"|": 0,
"ஃ": 3,
"அ": 4,
"ஆ": 5,
"இ": 6,
"ஈ": 7,
"உ": 8,
"ஊ": 9,
"எ": 10,
"ஏ": 11,
"ஐ": 12,
"ஒ": 13,
"ஓ": 14,
"ஔ": 15,
"க": 16,
"ங": 17,
"ச": 18,
"ஜ": 19,
"ஞ": 20,
"ட": 21,
"ண": 22,
"த": 23,
"ந": 24,
"ன": 25,
"ப": 26,
"ம": 27,
"ய": 28,
"ர": 29,
"ற": 30,
"ல": 31,
"ள": 32,
"ழ": 33,
"வ": 34,
"ஷ": 35,
"ஸ": 36,
"ஹ": 37,
"ா": 38,
"ி": 39,
"ீ": 40,
"ு": 41,
"ூ": 42,
"ெ": 43,
"ே": 44,
"ை": 45,
"ொ": 46,
"ோ": 47,
"ௌ": 48,
"்": 49,
"ௗ": 50,
"ഥ": 51
}
}