jon-tow commited on
Commit
e1520e0
1 Parent(s): 542dce2

fix(tokenizer): set `mode_max_length=4096`

Browse files

Thanks to

@ctranslate2-4you

Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -40,7 +40,7 @@
40
  "clean_up_tokenization_spaces": true,
41
  "eos_token": "<|endoftext|>",
42
  "tokenizer_class": "GPT2TokenizerFast",
43
- "model_max_length": 2048,
44
  "pad_token": "<|endoftext|>",
45
  "unk_token": "<|endoftext|>"
46
  }
 
40
  "clean_up_tokenization_spaces": true,
41
  "eos_token": "<|endoftext|>",
42
  "tokenizer_class": "GPT2TokenizerFast",
43
+ "model_max_length": 4096,
44
  "pad_token": "<|endoftext|>",
45
  "unk_token": "<|endoftext|>"
46
  }