ad019el commited on
Commit
97c7c0c
1 Parent(s): 4f96dd3

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +70 -83
vocab.json CHANGED
@@ -1,87 +1,74 @@
1
  {
2
- "!": 19,
3
- "\"": 73,
4
- "'": 23,
5
- "(": 50,
6
- ")": 37,
7
- ",": 24,
8
- "-": 20,
9
- ".": 42,
10
- ":": 67,
11
- ";": 38,
12
- "?": 4,
13
- "A": 16,
14
- "B": 17,
15
- "C": 58,
16
- "D": 12,
17
- "E": 70,
18
- "F": 10,
19
- "G": 60,
20
- "H": 34,
21
- "I": 77,
22
- "J": 8,
23
- "K": 11,
24
- "L": 31,
25
- "M": 65,
26
- "N": 48,
27
- "O": 69,
28
- "P": 36,
29
- "R": 41,
30
- "S": 49,
31
  "T": 1,
32
- "U": 68,
33
- "V": 64,
34
- "Z": 79,
35
- "[PAD]": 84,
36
- "[UNK]": 83,
37
- "a": 66,
38
- "b": 63,
39
- "c": 15,
40
- "d": 71,
41
- "e": 76,
42
- "f": 25,
43
- "g": 27,
44
- "h": 35,
45
- "i": 6,
46
- "j": 81,
47
- "k": 57,
48
- "l": 39,
49
- "m": 46,
50
- "n": 33,
51
- "o": 47,
52
- "p": 29,
53
- "q": 2,
54
- "r": 56,
55
- "s": 7,
56
- "t": 51,
57
- "u": 54,
58
- "v": 28,
59
- "w": 72,
60
- "y": 32,
61
- "z": 5,
62
- "|": 18,
63
- "«": 61,
64
- "é": 9,
65
- "Ĉ": 82,
66
- "ĉ": 55,
67
- "Ĝ": 78,
68
- "ĝ": 40,
69
- "Ĥ": 59,
70
- "ĥ": 21,
71
- "Ĵ": 75,
72
- "ĵ": 14,
73
- "Ŝ": 22,
74
- "ŝ": 30,
75
- "Ŭ": 80,
76
- "ŭ": 13,
77
- "ٱ": 52,
78
- "–": 53,
79
- "—": 43,
80
- "‘": 3,
81
- "’": 26,
82
- "“": 74,
83
- "”": 44,
84
- "„": 45,
85
  "fi": 0,
86
- "fl": 62
87
  }
 
1
  {
2
+ "!": 24,
3
+ "\"": 60,
4
+ "'": 29,
5
+ "(": 13,
6
+ ")": 52,
7
+ ",": 30,
8
+ "-": 25,
9
+ ".": 59,
10
+ ":": 43,
11
+ ";": 54,
12
+ "?": 7,
13
+ "A": 19,
14
+ "B": 20,
15
+ "C": 28,
16
+ "D": 17,
17
+ "E": 53,
18
+ "F": 14,
19
+ "G": 32,
20
+ "H": 49,
21
+ "I": 66,
22
+ "J": 11,
23
+ "K": 16,
24
+ "L": 44,
25
+ "M": 40,
26
+ "N": 6,
27
+ "O": 48,
28
+ "P": 51,
29
+ "R": 58,
30
+ "S": 12,
31
  "T": 1,
32
+ "U": 47,
33
+ "V": 38,
34
+ "Z": 67,
35
+ "[PAD]": 71,
36
+ "[UNK]": 70,
37
+ "a": 42,
38
+ "b": 37,
39
+ "c": 18,
40
+ "d": 55,
41
+ "e": 65,
42
+ "f": 31,
43
+ "g": 35,
44
+ "h": 50,
45
+ "i": 9,
46
+ "j": 69,
47
+ "k": 27,
48
+ "l": 56,
49
+ "m": 2,
50
+ "n": 46,
51
+ "o": 5,
52
+ "p": 41,
53
+ "q": 3,
54
+ "r": 26,
55
+ "s": 10,
56
+ "t": 15,
57
+ "u": 23,
58
+ "v": 39,
59
+ "w": 57,
60
+ "y": 45,
61
+ "z": 8,
62
+ "|": 21,
63
+ "«": 33,
64
+ "Ĵ": 62,
65
+ "": 22,
66
+ "": 63,
67
+ "": 4,
68
+ "": 34,
69
+ "": 61,
70
+ "": 64,
71
+ "": 68,
 
 
 
 
 
 
 
 
 
 
 
 
 
72
  "fi": 0,
73
+ "fl": 36
74
  }