Commit
•
6816409
1
Parent(s):
b450a7e
Upload tokenizer
Browse files- tokenizer.json +2 -4
tokenizer.json
CHANGED
|
@@ -950,8 +950,7 @@
|
|
| 950 |
"pre_tokenizer": {
|
| 951 |
"type": "Metaspace",
|
| 952 |
"replacement": "▁",
|
| 953 |
-
"add_prefix_space": true
|
| 954 |
-
"prepend_scheme": "always"
|
| 955 |
},
|
| 956 |
"post_processor": {
|
| 957 |
"type": "TemplateProcessing",
|
|
@@ -1010,8 +1009,7 @@
|
|
| 1010 |
"decoder": {
|
| 1011 |
"type": "Metaspace",
|
| 1012 |
"replacement": "▁",
|
| 1013 |
-
"add_prefix_space": true
|
| 1014 |
-
"prepend_scheme": "always"
|
| 1015 |
},
|
| 1016 |
"model": {
|
| 1017 |
"type": "Unigram",
|
|
|
|
| 950 |
"pre_tokenizer": {
|
| 951 |
"type": "Metaspace",
|
| 952 |
"replacement": "▁",
|
| 953 |
+
"add_prefix_space": true
|
|
|
|
| 954 |
},
|
| 955 |
"post_processor": {
|
| 956 |
"type": "TemplateProcessing",
|
|
|
|
| 1009 |
"decoder": {
|
| 1010 |
"type": "Metaspace",
|
| 1011 |
"replacement": "▁",
|
| 1012 |
+
"add_prefix_space": true
|
|
|
|
| 1013 |
},
|
| 1014 |
"model": {
|
| 1015 |
"type": "Unigram",
|