fix
Browse files- sentencepiece_ja.py +1 -1
sentencepiece_ja.py
CHANGED
@@ -4,7 +4,7 @@ from typing import Union, List, Optional, Tuple
|
|
4 |
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
|
5 |
|
6 |
class SentencePieceJA(PreTrainedTokenizer):
|
7 |
-
def __init__(self, model_path, **kwargs):
|
8 |
super().__init__(**kwargs)
|
9 |
from tokenizers import Tokenizer
|
10 |
self._tokenizer = Tokenizer.from_file(model_path)
|
|
|
4 |
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
|
5 |
|
6 |
class SentencePieceJA(PreTrainedTokenizer):
|
7 |
+
def __init__(self, model_path = "./tokenizer.json", **kwargs):
|
8 |
super().__init__(**kwargs)
|
9 |
from tokenizers import Tokenizer
|
10 |
self._tokenizer = Tokenizer.from_file(model_path)
|