ndeclarke commited on
Commit
a5e0e38
1 Parent(s): 81d5dd8

Upload tokenizer

Browse files
Files changed (3) hide show
  1. README.md +9 -9
  2. tokenizer_config.json +1 -2
  3. vocab.json +56 -0
README.md CHANGED
@@ -1,20 +1,20 @@
1
  ---
2
- library_name: transformers
3
- license: cc-by-nc-4.0
4
  base_model: facebook/mms-1b-all
5
- tags:
6
- - generated_from_trainer
7
  datasets:
8
  - common_voice_17_0
 
 
9
  metrics:
10
  - wer
11
  - bleu
 
 
12
  model-index:
13
  - name: wav2vec2-mms-1b-CV17.0-training_set_variations
14
  results:
15
  - task:
16
- name: Automatic Speech Recognition
17
  type: automatic-speech-recognition
 
18
  dataset:
19
  name: common_voice_17_0
20
  type: common_voice_17_0
@@ -22,12 +22,12 @@ model-index:
22
  split: validation
23
  args: ta
24
  metrics:
25
- - name: Wer
26
- type: wer
27
  value: 0.45588302699729566
28
- - name: Bleu
29
- type: bleu
30
  value: 0.30120570288375
 
31
  ---
32
 
33
  <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 
1
  ---
 
 
2
  base_model: facebook/mms-1b-all
 
 
3
  datasets:
4
  - common_voice_17_0
5
+ library_name: transformers
6
+ license: cc-by-nc-4.0
7
  metrics:
8
  - wer
9
  - bleu
10
+ tags:
11
+ - generated_from_trainer
12
  model-index:
13
  - name: wav2vec2-mms-1b-CV17.0-training_set_variations
14
  results:
15
  - task:
 
16
  type: automatic-speech-recognition
17
+ name: Automatic Speech Recognition
18
  dataset:
19
  name: common_voice_17_0
20
  type: common_voice_17_0
 
22
  split: validation
23
  args: ta
24
  metrics:
25
+ - type: wer
 
26
  value: 0.45588302699729566
27
+ name: Wer
28
+ - type: bleu
29
  value: 0.30120570288375
30
+ name: Bleu
31
  ---
32
 
33
  <!-- This model card has been generated automatically according to the information the Trainer had access to. You
tokenizer_config.json CHANGED
@@ -39,9 +39,8 @@
39
  "eos_token": "</s>",
40
  "model_max_length": 1000000000000000019884624838656,
41
  "pad_token": "[PAD]",
42
- "processor_class": "Wav2Vec2Processor",
43
  "replace_word_delimiter_char": " ",
44
- "target_lang": "tam-128",
45
  "tokenizer_class": "Wav2Vec2CTCTokenizer",
46
  "unk_token": "[UNK]",
47
  "word_delimiter_token": "|"
 
39
  "eos_token": "</s>",
40
  "model_max_length": 1000000000000000019884624838656,
41
  "pad_token": "[PAD]",
 
42
  "replace_word_delimiter_char": " ",
43
+ "target_lang": "tam-256",
44
  "tokenizer_class": "Wav2Vec2CTCTokenizer",
45
  "unk_token": "[UNK]",
46
  "word_delimiter_token": "|"
vocab.json CHANGED
@@ -111,6 +111,62 @@
111
  "ௗ": 50,
112
  "ഥ": 51
113
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114
  "tam-32": {
115
  "&": 1,
116
  "[PAD]": 53,
 
111
  "ௗ": 50,
112
  "ഥ": 51
113
  },
114
+ "tam-256": {
115
+ "&": 1,
116
+ "[PAD]": 53,
117
+ "[UNK]": 52,
118
+ "_": 2,
119
+ "|": 0,
120
+ "ஃ": 3,
121
+ "அ": 4,
122
+ "ஆ": 5,
123
+ "இ": 6,
124
+ "ஈ": 7,
125
+ "உ": 8,
126
+ "ஊ": 9,
127
+ "எ": 10,
128
+ "ஏ": 11,
129
+ "ஐ": 12,
130
+ "ஒ": 13,
131
+ "ஓ": 14,
132
+ "ஔ": 15,
133
+ "க": 16,
134
+ "ங": 17,
135
+ "ச": 18,
136
+ "ஜ": 19,
137
+ "ஞ": 20,
138
+ "ட": 21,
139
+ "ண": 22,
140
+ "த": 23,
141
+ "ந": 24,
142
+ "ன": 25,
143
+ "ப": 26,
144
+ "ம": 27,
145
+ "ய": 28,
146
+ "ர": 29,
147
+ "ற": 30,
148
+ "ல": 31,
149
+ "ள": 32,
150
+ "ழ": 33,
151
+ "வ": 34,
152
+ "ஷ": 35,
153
+ "ஸ": 36,
154
+ "ஹ": 37,
155
+ "ா": 38,
156
+ "ி": 39,
157
+ "ீ": 40,
158
+ "ு": 41,
159
+ "ூ": 42,
160
+ "ெ": 43,
161
+ "ே": 44,
162
+ "ை": 45,
163
+ "ொ": 46,
164
+ "ோ": 47,
165
+ "ௌ": 48,
166
+ "்": 49,
167
+ "ௗ": 50,
168
+ "ഥ": 51
169
+ },
170
  "tam-32": {
171
  "&": 1,
172
  "[PAD]": 53,