File size: 285 Bytes
e443128 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
from transformers import GPT2Tokenizer
class GPTMoEMCTSTokenizer(GPT2Tokenizer):
def __init__(
self,
vocab_file,
merges_file,
**kwargs
):
super().__init__(
vocab_file,
merges_file,
**kwargs
)
|