AswanthCManoj commited on
Commit
bfdd0e6
1 Parent(s): 2daf650

Upload tokenizer

Browse files
Files changed (1) hide show
  1. added_tokens.json +12 -12
added_tokens.json CHANGED
@@ -37,16 +37,16 @@
37
  " ": 50259,
38
  " ": 50258,
39
  " ": 50257,
40
- "<PAD>": 50304,
41
- "<|end_of_turn|>": 50301,
42
- "<|end|>": 50306,
43
- "<|function_call|>": 50296,
44
- "<|response|>": 50299,
45
- "<|thoughts|>": 50300,
46
- "Azma": 50305,
47
- "Azma:": 50295,
48
- "Observation:": 50303,
49
- "Reference:": 50302,
50
- "System:": 50298,
51
- "User:": 50297
52
  }
 
37
  " ": 50259,
38
  " ": 50258,
39
  " ": 50257,
40
+ "<PAD>": 50296,
41
+ "<|end_of_turn|>": 50298,
42
+ "<|end|>": 50297,
43
+ "<|function_call|>": 50301,
44
+ "<|response|>": 50305,
45
+ "<|thoughts|>": 50304,
46
+ "Azma": 50303,
47
+ "Azma:": 50302,
48
+ "Observation:": 50299,
49
+ "Reference:": 50295,
50
+ "System:": 50300,
51
+ "User:": 50306
52
  }