Update README.md

README.md

@@ -132,22 +132,33 @@ test_dataset = test_dataset.map(speech_file_to_array_fn)
 # Preprocessing the datasets.
 # We need to read the audio files as arrays
 def evaluate(batch):
-	inputs = processor(batch["speech"], sampling_rate=16_000, return_tensors="pt", padding=True)
+\tinputs = processor(batch["speech"], sampling_rate=16_000, return_tensors="pt", padding=True)
 
-	with torch.no_grad():
-		logits = model(inputs.input_values.to(DEVICE), attention_mask=inputs.attention_mask.to(DEVICE)).logits
+\twith torch.no_grad():
+\t\tlogits = model(inputs.input_values.to(DEVICE), attention_mask=inputs.attention_mask.to(DEVICE)).logits
 
-	pred_ids = torch.argmax(logits, dim=-1)
-	batch["pred_strings"] = processor.batch_decode(pred_ids)
-	return batch
+\tpred_ids = torch.argmax(logits, dim=-1)
+\tbatch["pred_strings"] = processor.batch_decode(pred_ids)
+\treturn batch
 
 result = test_dataset.map(evaluate, batched=True, batch_size=8)
 
-print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"], chunk_size=1000)))
-print("CER: {:2f}".format(100 * cer.compute(predictions=result["pred_strings"], references=result["sentence"], chunk_size=1000)))
+predictions = [x.upper() for x in result["pred_strings"]]
+references = [x.upper() for x in result["sentence"]]
+
+print(f"WER: {wer.compute(predictions=predictions, references=references, chunk_size=1000) * 100}")
+print(f"CER: {cer.compute(predictions=predictions, references=references, chunk_size=1000) * 100}")
 ```
 
 **Test Result**:
 
-- WER: 13.60%
-- CER: 4.45%
+My model may report better scores than others because of some specificity of my evaluation script, so I ran the same evaluation script on other models (on 2021-04-21) to make a fairer comparison.
+
+| Model | WER | CER |
+| ------------- | ------------- | ------------- |
+| jonatasgrosman/wav2vec2-large-xlsr-53-dutch | **13.60%** | **4.45%** |
+| wietsedv/wav2vec2-large-xlsr-53-dutch | 16.78% | 5.60% |
+| facebook/wav2vec2-large-xlsr-53-dutch | 20.97% | 7.24% |
+| nithinholla/wav2vec2-large-xlsr-53-dutch | 21.39% | 7.29% |
+| MehdiHosseiniMoghadam/wav2vec2-large-xlsr-53-Dutch | 25.89% | 9.12% |
+| simonsr/wav2vec2-large-xlsr-dutch | 38.34% | 13.29% |