ad019el commited on
Commit
b537a64
1 Parent(s): bea4347

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +62 -72
vocab.json CHANGED
@@ -1,74 +1,64 @@
1
  {
2
- "!": 18,
3
- "\"": 14,
4
- "'": 16,
5
- "(": 35,
6
- ")": 3,
7
- ",": 13,
8
- "-": 0,
9
- ".": 31,
10
- ":": 62,
11
- ";": 33,
12
- "?": 45,
13
- "A": 29,
14
- "B": 58,
15
- "C": 63,
16
- "D": 61,
17
- "E": 2,
18
- "F": 50,
19
- "G": 11,
20
- "H": 66,
21
- "I": 69,
22
- "J": 37,
23
- "K": 41,
24
- "L": 48,
25
- "M": 65,
26
- "N": 32,
27
- "O": 21,
28
- "P": 36,
29
- "R": 20,
30
- "S": 5,
31
- "T": 56,
32
- "U": 10,
33
- "V": 26,
34
- "Z": 46,
35
- "[PAD]": 71,
36
- "[UNK]": 70,
37
- "a": 1,
38
- "b": 67,
39
- "c": 54,
40
- "d": 53,
41
- "e": 30,
42
- "f": 57,
43
- "g": 43,
44
- "h": 23,
45
- "i": 8,
46
- "j": 55,
47
- "k": 15,
48
- "l": 59,
49
- "m": 42,
50
- "n": 25,
51
- "o": 47,
52
- "p": 27,
53
- "q": 4,
54
- "r": 52,
55
- "s": 24,
56
- "t": 34,
57
- "u": 6,
58
- "v": 19,
59
- "w": 7,
60
- "y": 60,
61
- "z": 51,
62
- "|": 68,
63
- "«": 28,
64
- "Ĵ": 44,
65
- "–": 17,
66
- "—": 40,
67
- "‘": 22,
68
- "’": 64,
69
- "“": 12,
70
- "”": 39,
71
- "„": 49,
72
- "fi": 9,
73
- "fl": 38
74
  }
 
1
  {
2
+ "!": 6,
3
+ "\"": 3,
4
+ "'": 54,
5
+ "(": 26,
6
+ ")": 1,
7
+ ",": 39,
8
+ "-": 4,
9
+ ".": 51,
10
+ ":": 21,
11
+ ";": 12,
12
+ "?": 49,
13
+ "A": 56,
14
+ "B": 47,
15
+ "C": 45,
16
+ "D": 38,
17
+ "E": 32,
18
+ "F": 46,
19
+ "G": 59,
20
+ "H": 48,
21
+ "I": 17,
22
+ "J": 30,
23
+ "K": 13,
24
+ "L": 50,
25
+ "M": 33,
26
+ "N": 19,
27
+ "O": 31,
28
+ "P": 18,
29
+ "R": 40,
30
+ "S": 24,
31
+ "T": 34,
32
+ "U": 52,
33
+ "V": 23,
34
+ "[PAD]": 61,
35
+ "[UNK]": 60,
36
+ "a": 53,
37
+ "b": 29,
38
+ "c": 37,
39
+ "d": 27,
40
+ "e": 42,
41
+ "f": 8,
42
+ "g": 57,
43
+ "h": 25,
44
+ "i": 35,
45
+ "j": 20,
46
+ "k": 36,
47
+ "l": 44,
48
+ "m": 22,
49
+ "n": 0,
50
+ "o": 14,
51
+ "p": 9,
52
+ "q": 5,
53
+ "r": 16,
54
+ "s": 28,
55
+ "t": 43,
56
+ "u": 2,
57
+ "v": 7,
58
+ "w": 15,
59
+ "y": 11,
60
+ "z": 10,
61
+ "|": 41,
62
+ "Ĵ": 58,
63
+ "": 55
 
 
 
 
 
 
 
 
 
 
64
  }