if001 commited on
Commit
c50283d
1 Parent(s): 004c3d7

add config

Browse files
Files changed (2) hide show
  1. config.json +6 -0
  2. tokenizer_config.json +1 -1
config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "sentencepiece_ja",
3
+ "auto_map": {
4
+ "AutoTokenizer": "sentencepiece_ja.SentencePieceJA"
5
+ }
6
+ }
tokenizer_config.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
  "clean_up_tokenization_spaces": true,
3
- "tokenizer_class": "sentencepiece_ja.SentencePieceJA"
4
  }
 
1
  {
2
  "clean_up_tokenization_spaces": true,
3
+ "tokenizer_class": "SentencePieceJA"
4
  }