ad019el commited on
Commit
b28b38c
1 Parent(s): 5d44eb9

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +92 -61
vocab.json CHANGED
@@ -1,64 +1,95 @@
1
  {
2
- "!": 6,
3
- "\"": 3,
4
- "'": 54,
5
- "(": 26,
6
- ")": 1,
7
- ",": 39,
8
- "-": 4,
9
- ".": 51,
10
- ":": 21,
11
- ";": 12,
12
- "?": 49,
13
- "A": 56,
14
- "B": 47,
15
- "C": 45,
16
- "D": 38,
17
- "E": 32,
18
- "F": 46,
19
- "G": 59,
20
- "H": 48,
21
- "I": 17,
22
- "J": 30,
23
- "K": 13,
24
- "L": 50,
25
- "M": 33,
26
- "N": 19,
27
- "O": 31,
28
- "P": 18,
29
- "R": 40,
30
- "S": 24,
31
- "T": 34,
32
- "U": 52,
33
- "V": 23,
34
- "[PAD]": 61,
35
- "[UNK]": 60,
36
- "a": 53,
37
- "b": 29,
38
- "c": 37,
39
- "d": 27,
40
- "e": 42,
41
- "f": 8,
42
- "g": 57,
43
- "h": 25,
44
- "i": 35,
45
- "j": 20,
46
- "k": 36,
47
- "l": 44,
48
- "m": 22,
49
  "n": 0,
50
- "o": 14,
51
- "p": 9,
52
- "q": 5,
53
- "r": 16,
54
- "s": 28,
55
- "t": 43,
56
- "u": 2,
57
- "v": 7,
58
- "w": 15,
59
- "y": 11,
60
- "z": 10,
61
- "|": 41,
62
- "Ĵ": 58,
63
- "": 55
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
64
  }
 
1
  {
2
+ ":": 59,
3
+ "J": 76,
4
+ "[PAD]": 92,
5
+ "[UNK]": 91,
6
+ "a": 21,
7
+ "b": 86,
8
+ "c": 53,
9
+ "d": 8,
10
+ "e": 29,
11
+ "f": 41,
12
+ "g": 11,
13
+ "h": 54,
14
+ "i": 7,
15
+ "j": 25,
16
+ "k": 60,
17
+ "l": 75,
18
+ "m": 89,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  "n": 0,
20
+ "o": 67,
21
+ "p": 3,
22
+ "q": 47,
23
+ "r": 1,
24
+ "s": 69,
25
+ "t": 46,
26
+ "u": 44,
27
+ "v": 56,
28
+ "w": 82,
29
+ "y": 35,
30
+ "z": 43,
31
+ "|": 64,
32
+ "°": 40,
33
+ "à": 84,
34
+ "ä": 83,
35
+ "å": 23,
36
+ "è": 18,
37
+ "ì": 37,
38
+ "ò": 78,
39
+ "ö": 17,
40
+ "ù": 85,
41
+ "ą": 22,
42
+ "č": 80,
43
+ "ė": 28,
44
+ "ę": 24,
45
+ "į": 72,
46
+ "š": 65,
47
+ "ū": 61,
48
+ "ų": 45,
49
+ "ž": 73,
50
+ "அ": 42,
51
+ "ஆ": 66,
52
+ "இ": 62,
53
+ "உ": 70,
54
+ "ஊ": 16,
55
+ "எ": 63,
56
+ "ஏ": 51,
57
+ "ஐ": 10,
58
+ "ஒ": 32,
59
+ "ஓ": 49,
60
+ "க": 48,
61
+ "ங": 26,
62
+ "ச": 87,
63
+ "ஜ": 88,
64
+ "ஞ": 79,
65
+ "ட": 31,
66
+ "ண": 74,
67
+ "த": 20,
68
+ "ந": 36,
69
+ "ன": 34,
70
+ "ப": 9,
71
+ "ம": 6,
72
+ "ய": 15,
73
+ "ர": 52,
74
+ "ற": 57,
75
+ "ல": 55,
76
+ "ள": 12,
77
+ "ழ": 5,
78
+ "வ": 50,
79
+ "ஷ": 38,
80
+ "ஸ": 77,
81
+ "ா": 58,
82
+ "ி": 14,
83
+ "ீ": 30,
84
+ "ு": 4,
85
+ "ூ": 19,
86
+ "ெ": 71,
87
+ "ே": 33,
88
+ "ை": 81,
89
+ "ொ": 90,
90
+ "ோ": 13,
91
+ "்": 27,
92
+ "’": 39,
93
+ "„": 2,
94
+ "…": 68
95
  }