AswanthCManoj
commited on
Commit
•
bfdd0e6
1
Parent(s):
2daf650
Upload tokenizer
Browse files- added_tokens.json +12 -12
added_tokens.json
CHANGED
@@ -37,16 +37,16 @@
|
|
37 |
" ": 50259,
|
38 |
" ": 50258,
|
39 |
" ": 50257,
|
40 |
-
"<PAD>":
|
41 |
-
"<|end_of_turn|>":
|
42 |
-
"<|end|>":
|
43 |
-
"<|function_call|>":
|
44 |
-
"<|response|>":
|
45 |
-
"<|thoughts|>":
|
46 |
-
"Azma":
|
47 |
-
"Azma:":
|
48 |
-
"Observation:":
|
49 |
-
"Reference:":
|
50 |
-
"System:":
|
51 |
-
"User:":
|
52 |
}
|
|
|
37 |
" ": 50259,
|
38 |
" ": 50258,
|
39 |
" ": 50257,
|
40 |
+
"<PAD>": 50296,
|
41 |
+
"<|end_of_turn|>": 50298,
|
42 |
+
"<|end|>": 50297,
|
43 |
+
"<|function_call|>": 50301,
|
44 |
+
"<|response|>": 50305,
|
45 |
+
"<|thoughts|>": 50304,
|
46 |
+
"Azma": 50303,
|
47 |
+
"Azma:": 50302,
|
48 |
+
"Observation:": 50299,
|
49 |
+
"Reference:": 50295,
|
50 |
+
"System:": 50300,
|
51 |
+
"User:": 50306
|
52 |
}
|