herwoww commited on
Commit
9578e76
1 Parent(s): 3223063

Upload processor

Browse files
Files changed (3) hide show
  1. added_tokens.json +1 -3
  2. spm_char.model +2 -2
  3. tokenizer_config.json +6 -6
added_tokens.json CHANGED
@@ -1,5 +1,3 @@
1
  {
2
- "<ctc_blank>": 88,
3
- "<mask>": 87,
4
- "<pad>": 86
5
  }
 
1
  {
2
+ "<ctc_blank>": 97
 
 
3
  }
spm_char.model CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4f4b56b4bbf0637c59a5c9c8531362bf8ce2bcde4fcd28ce0a0ec26cdf70a905
3
- size 403344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:195a4a0a66f1eb25404206ae3e595975a1264e85639bbc0e0c0489d065b78b70
3
+ size 403545
tokenizer_config.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  "added_tokens_decoder": {
3
  "0": {
4
- "content": "<unk>",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
@@ -9,7 +9,7 @@
9
  "special": true
10
  },
11
  "1": {
12
- "content": "<s>",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
@@ -24,15 +24,15 @@
24
  "single_word": false,
25
  "special": true
26
  },
27
- "86": {
28
- "content": "<pad>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
32
  "single_word": false,
33
  "special": true
34
  },
35
- "87": {
36
  "content": "<mask>",
37
  "lstrip": true,
38
  "normalized": true,
@@ -40,7 +40,7 @@
40
  "single_word": false,
41
  "special": true
42
  },
43
- "88": {
44
  "content": "<ctc_blank>",
45
  "lstrip": false,
46
  "normalized": true,
 
1
  {
2
  "added_tokens_decoder": {
3
  "0": {
4
+ "content": "<s>",
5
  "lstrip": false,
6
  "normalized": false,
7
  "rstrip": false,
 
9
  "special": true
10
  },
11
  "1": {
12
+ "content": "<pad>",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
 
24
  "single_word": false,
25
  "special": true
26
  },
27
+ "3": {
28
+ "content": "<unk>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
32
  "single_word": false,
33
  "special": true
34
  },
35
+ "96": {
36
  "content": "<mask>",
37
  "lstrip": true,
38
  "normalized": true,
 
40
  "single_word": false,
41
  "special": true
42
  },
43
+ "97": {
44
  "content": "<ctc_blank>",
45
  "lstrip": false,
46
  "normalized": true,