{ | |
"version": "1.0", | |
"truncation": null, | |
"padding": null, | |
"added_tokens": [ | |
{ | |
"id": 32, | |
"content": "[SEP]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 33, | |
"content": "[PAD]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 34, | |
"content": "[CLS]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 35, | |
"content": "[MASK]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
} | |
], | |
"normalizer": null, | |
"pre_tokenizer": null, | |
"post_processor": null, | |
"decoder": null, | |
"model": { | |
"type": "BPE", | |
"dropout": null, | |
"unk_token": null, | |
"continuing_subword_prefix": null, | |
"end_of_word_suffix": null, | |
"fuse_unk": false, | |
"byte_fallback": false, | |
"ignore_merges": false, | |
"vocab": { | |
"-": 0, | |
".": 1, | |
"0": 2, | |
"1": 3, | |
"2": 4, | |
"3": 5, | |
"4": 6, | |
"5": 7, | |
"6": 8, | |
"7": 9, | |
"8": 10, | |
"9": 11, | |
"B": 12, | |
"K": 13, | |
"N": 14, | |
"P": 15, | |
"Q": 16, | |
"R": 17, | |
"a": 18, | |
"b": 19, | |
"c": 20, | |
"d": 21, | |
"e": 22, | |
"f": 23, | |
"g": 24, | |
"h": 25, | |
"k": 26, | |
"n": 27, | |
"p": 28, | |
"q": 29, | |
"r": 30, | |
"w": 31 | |
}, | |
"merges": [] | |
} | |
} |