fix vocab_size to property
Browse files- sentencepiece_ja.py +3 -2
- tokenizer_config.json +2 -1
sentencepiece_ja.py
CHANGED
@@ -28,10 +28,11 @@ class SentencePieceJA(PreTrainedTokenizer):
|
|
28 |
'unk_token': unk,
|
29 |
'mask_token': mask
|
30 |
})
|
31 |
-
|
32 |
def get_vocab(self) -> int:
|
33 |
return self._tokenizer.get_vocab()
|
34 |
-
|
|
|
35 |
def vocab_size(self) -> int:
|
36 |
return self._tokenizer.get_vocab_size()
|
37 |
|
|
|
28 |
'unk_token': unk,
|
29 |
'mask_token': mask
|
30 |
})
|
31 |
+
|
32 |
def get_vocab(self) -> int:
|
33 |
return self._tokenizer.get_vocab()
|
34 |
+
|
35 |
+
@property
|
36 |
def vocab_size(self) -> int:
|
37 |
return self._tokenizer.get_vocab_size()
|
38 |
|
tokenizer_config.json
CHANGED
@@ -9,5 +9,6 @@
|
|
9 |
"tokenizer_class": "SentencePieceJA",
|
10 |
"auto_map": {
|
11 |
"AutoTokenizer": ["","sentencepiece_ja.SentencePieceJA"]
|
12 |
-
}
|
|
|
13 |
}
|
|
|
9 |
"tokenizer_class": "SentencePieceJA",
|
10 |
"auto_map": {
|
11 |
"AutoTokenizer": ["","sentencepiece_ja.SentencePieceJA"]
|
12 |
+
},
|
13 |
+
"transformers_version": " 4.34.1"
|
14 |
}
|