Joseph717171 commited on
Commit
49511ce
1 Parent(s): 08118a5

Update added_tokens.json

Browse files
Files changed (1) hide show
  1. added_tokens.json +32 -2
added_tokens.json CHANGED
@@ -1,4 +1,34 @@
1
  {
2
  "<|im_end|>": 32000,
3
- "<|im_start|>": 32001
4
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  {
2
  "<|im_end|>": 32000,
3
+ "<|im_start|>": 32001,
4
+ "<pad2>": 32002,
5
+ "<pad3>": 32003,
6
+ "<pad4>": 32004,
7
+ "<pad5>": 32005,
8
+ "<pad6>": 32006,
9
+ "<pad7>": 32007,
10
+ "<pad8>": 32008,
11
+ "<pad9>": 32009,
12
+ "<pad10>": 32010,
13
+ "<pad11>": 32011,
14
+ "<pad12>": 32012,
15
+ "<pad13>": 32013,
16
+ "<pad14>": 32014,
17
+ "<pad15>": 32015,
18
+ "<pad16>": 32016,
19
+ "<pad17>": 32017,
20
+ "<pad18>": 32018,
21
+ "<pad19>": 32019,
22
+ "<pad20>": 32020,
23
+ "<pad21>": 32021,
24
+ "<pad22>": 32022,
25
+ "<pad23>": 32023,
26
+ "<pad24>": 32024,
27
+ "<pad25>": 32025,
28
+ "<pad26>": 32026,
29
+ "<pad27>": 32027,
30
+ "<pad28>": 32028,
31
+ "<pad29>": 32029,
32
+ "<pad30>": 32030,
33
+ "<pad31>": 32031
34
+ }