Upload tokenizer
72733ec
|
{ |
|
"kaz": { |
|
"\n": 0, |
|
"/": 2, |
|
"[": 3, |
|
"[PAD]": 125, |
|
"[UNK]": 124, |
|
"]": 4, |
|
"a": 5, |
|
"b": 6, |
|
"c": 7, |
|
"d": 8, |
|
"e": 9, |
|
"f": 10, |
|
"g": 11, |
|
"h": 12, |
|
"i": 13, |
|
"j": 14, |
|
"k": 15, |
|
"l": 16, |
|
"m": 17, |
|
"n": 18, |
|
"o": 19, |
|
"p": 20, |
|
"q": 21, |
|
"r": 22, |
|
"s": 23, |
|
"t": 24, |
|
"u": 25, |
|
"v": 26, |
|
"w": 27, |
|
"x": 28, |
|
"y": 29, |
|
"z": 30, |
|
"{": 31, |
|
"|": 1, |
|
"}": 32, |
|
"®": 33, |
|
"»": 34, |
|
"ã": 35, |
|
"ä": 36, |
|
"ó": 37, |
|
"ö": 38, |
|
"ü": 39, |
|
"ğ": 40, |
|
"ū": 41, |
|
"ə": 42, |
|
"ɵ": 43, |
|
"ϵ": 44, |
|
"а": 45, |
|
"б": 46, |
|
"в": 47, |
|
"г": 48, |
|
"д": 49, |
|
"е": 50, |
|
"ж": 51, |
|
"з": 52, |
|
"и": 53, |
|
"й": 54, |
|
"к": 55, |
|
"л": 56, |
|
"м": 57, |
|
"н": 58, |
|
"о": 59, |
|
"п": 60, |
|
"р": 61, |
|
"с": 62, |
|
"т": 63, |
|
"у": 64, |
|
"ф": 65, |
|
"х": 66, |
|
"ц": 67, |
|
"ч": 68, |
|
"ш": 69, |
|
"щ": 70, |
|
"ъ": 71, |
|
"ы": 72, |
|
"ь": 73, |
|
"э": 74, |
|
"ю": 75, |
|
"я": 76, |
|
"ё": 77, |
|
"і": 78, |
|
"ғ": 79, |
|
"қ": 80, |
|
"ң": 81, |
|
"ү": 82, |
|
"ұ": 83, |
|
"һ": 84, |
|
"ә": 85, |
|
"ө": 86, |
|
"٪": 87, |
|
"–": 88, |
|
"™": 89, |
|
"⏳": 90, |
|
"⚐": 91, |
|
"⚑": 92, |
|
"️": 93, |
|
"﹪": 94, |
|
"🌍": 95, |
|
"🌏": 96, |
|
"🎉": 97, |
|
"🎭": 98, |
|
"🎰": 99, |
|
"🏠": 100, |
|
"🏢": 101, |
|
"🏬": 102, |
|
"🏭": 103, |
|
"🏳": 104, |
|
"👅": 105, |
|
"👑": 106, |
|
"👤": 107, |
|
"👽": 108, |
|
"💯": 109, |
|
"💲": 110, |
|
"💼": 111, |
|
"📆": 112, |
|
"📈": 113, |
|
"📖": 114, |
|
"📝": 115, |
|
"📞": 116, |
|
"🔠": 117, |
|
"🔢": 118, |
|
"🔤": 119, |
|
"🕑": 120, |
|
"😷": 121, |
|
"🧮": 122, |
|
"🪶": 123 |
|
} |
|
} |
|
|
