|
{ |
|
"version": "1.0", |
|
"truncation": null, |
|
"padding": null, |
|
"added_tokens": [], |
|
"normalizer": { |
|
"type": "NFKC" |
|
}, |
|
"pre_tokenizer": { |
|
"type": "Metaspace", |
|
"replacement": "▁", |
|
"prepend_scheme": "always", |
|
"split": true |
|
}, |
|
"post_processor": null, |
|
"decoder": { |
|
"type": "Metaspace", |
|
"replacement": "▁", |
|
"prepend_scheme": "always", |
|
"split": true |
|
}, |
|
"model": { |
|
"type": "BPE", |
|
"dropout": null, |
|
"unk_token": "<unk>", |
|
"continuing_subword_prefix": null, |
|
"end_of_word_suffix": null, |
|
"fuse_unk": false, |
|
"byte_fallback": false, |
|
"ignore_merges": false, |
|
"vocab": {}, |
|
"merges": [] |
|
} |
|
} |