mirfan899 commited on
Commit
73fe5df
1 Parent(s): c8b03a0

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +29 -29
vocab.json CHANGED
@@ -1,35 +1,35 @@
1
  {
2
- ",": 1,
3
- "-": 19,
4
- "0": 16,
5
- "1": 10,
6
- "2": 6,
7
- "?": 4,
8
- "A": 7,
9
- "B": 27,
10
- "C": 23,
11
- "D": 11,
12
- "E": 18,
13
- "F": 3,
14
- "G": 25,
15
- "H": 5,
16
- "I": 13,
17
  "J": 12,
18
- "K": 0,
19
- "L": 22,
20
- "M": 20,
21
- "N": 30,
22
- "O": 2,
23
- "P": 28,
24
  "R": 8,
25
- "S": 17,
26
- "T": 21,
27
- "U": 29,
28
- "V": 14,
29
- "W": 24,
30
- "Y": 26,
31
- "Z": 15,
32
  "[PAD]": 32,
33
  "[UNK]": 31,
34
- "|": 9
35
  }
 
1
  {
2
+ ",": 21,
3
+ "-": 0,
4
+ "0": 6,
5
+ "1": 30,
6
+ "2": 27,
7
+ "?": 10,
8
+ "A": 3,
9
+ "B": 2,
10
+ "C": 28,
11
+ "D": 23,
12
+ "E": 1,
13
+ "F": 11,
14
+ "G": 17,
15
+ "H": 15,
16
+ "I": 5,
17
  "J": 12,
18
+ "K": 25,
19
+ "L": 14,
20
+ "M": 26,
21
+ "N": 13,
22
+ "O": 4,
23
+ "P": 7,
24
  "R": 8,
25
+ "S": 24,
26
+ "T": 22,
27
+ "U": 16,
28
+ "V": 19,
29
+ "W": 18,
30
+ "Y": 29,
31
+ "Z": 9,
32
  "[PAD]": 32,
33
  "[UNK]": 31,
34
+ "|": 20
35
  }