Upload tokenizer.json
bd9e75b
|
{ |
|
"version": "1.0", |
|
"truncation": null, |
|
"padding": null, |
|
"added_tokens": [ |
|
{ |
|
"id": 1, |
|
"content": "0", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 2, |
|
"content": "1", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 3, |
|
"content": "2", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 4, |
|
"content": "3", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 5, |
|
"content": "4", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 6, |
|
"content": "5", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 7, |
|
"content": "6", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 8, |
|
"content": "7", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 9, |
|
"content": "8", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 10, |
|
"content": "9", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 11, |
|
"content": "+", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 12, |
|
"content": "-", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 13, |
|
"content": "=", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 14, |
|
"content": " ", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
}, |
|
{ |
|
"id": 15, |
|
"content": "<", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": true, |
|
"special": false |
|
} |
|
], |
|
"normalizer": null, |
|
"pre_tokenizer": null, |
|
"post_processor": null, |
|
"decoder": null, |
|
"model": { |
|
"type": "Unigram", |
|
"unk_id": 0, |
|
"vocab": [ |
|
[ |
|
"<unk>", |
|
0.0 |
|
] |
|
], |
|
"byte_fallback": false |
|
} |
|
} |
