anicolson commited on
Commit
1838aab
1 Parent(s): 1400f7f

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -0
tokenizer.json CHANGED
@@ -80,6 +80,7 @@
80
  "end_of_word_suffix": null,
81
  "fuse_unk": false,
82
  "byte_fallback": false,
 
83
  "vocab": {
84
  "[UNK]": 0,
85
  "[BOS]": 1,
 
80
  "end_of_word_suffix": null,
81
  "fuse_unk": false,
82
  "byte_fallback": false,
83
+ "ignore_merges": false,
84
  "vocab": {
85
  "[UNK]": 0,
86
  "[BOS]": 1,