chenxingphh commited on
Commit
333c788
1 Parent(s): d35efa7

replace YiTokenizer to LlamaTokenizer to support load tokenizer without trsut_remote_code

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "auto_map": {
3
- "AutoTokenizer": ["tokenization_yi.YiTokenizer", null]
4
  },
5
  "add_bos_token": false,
6
  "add_eos_token": false,
7
  "model_max_length": 4096,
8
- "tokenizer_class": "YiTokenizer"
9
  }
 
1
  {
2
  "auto_map": {
3
+ "AutoTokenizer": ["LlamaTokenizer"]
4
  },
5
  "add_bos_token": false,
6
  "add_eos_token": false,
7
  "model_max_length": 4096,
8
+ "tokenizer_class": "LlamaTokenizer"
9
  }