tiiuae
/

falcon-mamba-7b

Text Generation

Inference Endpoints

Model card Files Files and versions Community

Gkunsch commited on Aug 15

Commit

2d34e76

•

1 Parent(s): f48fe69

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +1 -1

tokenizer_config.json CHANGED Viewed

@@ -114,7 +114,7 @@
     ">>MIDDLE<<"
   ],
   "clean_up_tokenization_spaces": true,
-  "eos_token": ">>TITLE<<",
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": ">>TITLE<<",
   "tokenizer_class": "PreTrainedTokenizerFast"

     ">>MIDDLE<<"
   ],
   "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": ">>TITLE<<",
   "tokenizer_class": "PreTrainedTokenizerFast"