if001 commited on
Commit
9e3e2aa
1 Parent(s): 4bb85ed
Files changed (1) hide show
  1. sentencepiece_ja.py +1 -1
sentencepiece_ja.py CHANGED
@@ -4,7 +4,7 @@ from typing import Union, List, Optional, Tuple
4
  from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
5
 
6
  class SentencePieceJA(PreTrainedTokenizer):
7
- def __init__(self, model_path, **kwargs):
8
  super().__init__(**kwargs)
9
  from tokenizers import Tokenizer
10
  self._tokenizer = Tokenizer.from_file(model_path)
 
4
  from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
5
 
6
  class SentencePieceJA(PreTrainedTokenizer):
7
+ def __init__(self, model_path = "./tokenizer.json", **kwargs):
8
  super().__init__(**kwargs)
9
  from tokenizers import Tokenizer
10
  self._tokenizer = Tokenizer.from_file(model_path)