davidilag's picture
Upload tokenizer
af86681 verified
raw
history blame contribute delete
517 Bytes
{
"'": 41,
"2": 26,
"3": 7,
"4": 11,
"5": 40,
"6": 23,
"[PAD]": 45,
"[UNK]": 44,
"_": 33,
"a": 28,
"b": 32,
"c": 4,
"d": 21,
"e": 35,
"f": 34,
"g": 12,
"h": 9,
"i": 31,
"j": 15,
"k": 39,
"l": 20,
"m": 42,
"n": 10,
"o": 38,
"p": 17,
"q": 30,
"r": 13,
"s": 24,
"t": 36,
"u": 37,
"v": 16,
"w": 0,
"x": 25,
"y": 6,
"z": 2,
"|": 14,
"á": 8,
"æ": 22,
"é": 18,
"í": 27,
"ð": 5,
"ó": 19,
"ø": 29,
"ú": 1,
"ü": 43,
"ý": 3
}