temp-kgsf-gen / tokenizer_config.json
ZhankuiHe's picture
Upload tokenizer
09823ba verified
raw
history blame
197 Bytes
{
"tokenizers": {
"word": "KGSFBaseTokenizer",
"item": "EntityTokenizer",
"entity": "KGSFBaseTokenizer",
"concept": "KGSFBaseTokenizer"
},
"tokenizer_key_for_decode": "word"
}