add im_start and im_end (#2)

- add im_start and im_end (0518759c65f09e0dba2afab61b11e695c0bddf65)


Co-authored-by: Apolinário from multimodal AI art <multimodalart@users.noreply.huggingface.co>

tokenizer.json

@@ -47,6 +47,24 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 100264,
+      "content": "<|im_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 100265,
+      "content": "<|im_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -100345,7 +100363,9 @@
       "<|fim_prefix|>": 100258,
       "<|fim_middle|>": 100259,
       "<|fim_suffix|>": 100260,
-      "<|endofprompt|>": 100276
+      "<|endofprompt|>": 100276,
+      "<|im_start|>": 100264,
+      "<|im_end|>": 100265
     },
     "merges": [
       "Ġ Ġ",

vocab.json

@@ -100259,5 +100259,7 @@
   "<|fim_prefix|>": 100258,
   "<|fim_middle|>": 100259,
   "<|fim_suffix|>": 100260,
-  "<|endofprompt|>": 100276
+  "<|endofprompt|>": 100276,
+  "<|im_start|>": 100264,
+  "<|im_end|>": 100265
 }