Initial commit

README.md

@@ -1,27 +1,1540 @@
 ---
 language:
+- af
+- de
+- en
+- fy
 - gmw
-- gmw
-datasets:
-- opus
+- gos
+- hrx
+- lb
+- nds
+- nl
+- pdc
+- yi
+
 tags:
 - translation
+
 license: cc-by-4.0
 model-index:
 - name: opus-mt-tc-base-gmw-gmw
   results:
-  - task: 
-      name: Translation AFR - DEU
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-news-test2008
+      type: WMT-news-test2008
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 23.8
+       - name: chr-F2
+         type: chrf
+         value: 0.518
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-news-test2008
+      type: WMT-news-test2008
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 19.3
+       - name: chr-F2
+         type: chrf
+         value: 0.492
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newssyscomb2009
+      type: WMT-newssyscomb2009
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 25.4
+       - name: chr-F2
+         type: chrf
+         value: 0.527
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newssyscomb2009
+      type: WMT-newssyscomb2009
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 19.3
+       - name: chr-F2
+         type: chrf
+         value: 0.504
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2009
+      type: WMT-newstest2009
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 23.4
+       - name: chr-F2
+         type: chrf
+         value: 0.516
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2009
+      type: WMT-newstest2009
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 18.8
+       - name: chr-F2
+         type: chrf
+         value: 0.498
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2010
+      type: WMT-newstest2010
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 25.8
+       - name: chr-F2
+         type: chrf
+         value: 0.546
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2010
+      type: WMT-newstest2010
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 20.7
+       - name: chr-F2
+         type: chrf
+         value: 0.508
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2011
+      type: WMT-newstest2011
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 23.7
+       - name: chr-F2
+         type: chrf
+         value: 0.524
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2011
+      type: WMT-newstest2011
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 19.2
+       - name: chr-F2
+         type: chrf
+         value: 0.493
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2012
+      type: WMT-newstest2012
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 24.8
+       - name: chr-F2
+         type: chrf
+         value: 0.532
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2012
+      type: WMT-newstest2012
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 19.5
+       - name: chr-F2
+         type: chrf
+         value: 0.493
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2013
+      type: WMT-newstest2013
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 27.7
+       - name: chr-F2
+         type: chrf
+         value: 0.548
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2013
+      type: WMT-newstest2013
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 22.5
+       - name: chr-F2
+         type: chrf
+         value: 0.517
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2014-deen
+      type: WMT-newstest2014-deen
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 27.3
+       - name: chr-F2
+         type: chrf
+         value: 0.548
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2014-deen
+      type: WMT-newstest2014-deen
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 22.0
+       - name: chr-F2
+         type: chrf
+         value: 0.532
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2015-deen
+      type: WMT-newstest2015-deen
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 28.6
+       - name: chr-F2
+         type: chrf
+         value: 0.553
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2015-ende
+      type: WMT-newstest2015-ende
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 25.7
+       - name: chr-F2
+         type: chrf
+         value: 0.544
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2016-deen
+      type: WMT-newstest2016-deen
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 33.3
+       - name: chr-F2
+         type: chrf
+         value: 0.596
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2016-ende
+      type: WMT-newstest2016-ende
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 30.0
+       - name: chr-F2
+         type: chrf
+         value: 0.580
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2017-deen
+      type: WMT-newstest2017-deen
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 29.5
+       - name: chr-F2
+         type: chrf
+         value: 0.561
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2017-ende
+      type: WMT-newstest2017-ende
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 24.1
+       - name: chr-F2
+         type: chrf
+         value: 0.535
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2018-deen
+      type: WMT-newstest2018-deen
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 36.1
+       - name: chr-F2
+         type: chrf
+         value: 0.610
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2018-ende
+      type: WMT-newstest2018-ende
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 35.4
+       - name: chr-F2
+         type: chrf
+         value: 0.613
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2019-deen
+      type: WMT-newstest2019-deen
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 32.3
+       - name: chr-F2
+         type: chrf
+         value: 0.582
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2019-ende
+      type: WMT-newstest2019-ende
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 31.2
+       - name: chr-F2
+         type: chrf
+         value: 0.583
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstest2020-deen
+      type: WMT-newstest2020-deen
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 32.0
+       - name: chr-F2
+         type: chrf
+         value: 0.604
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstest2020-ende
+      type: WMT-newstest2020-ende
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 23.9
+       - name: chr-F2
+         type: chrf
+         value: 0.542
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: WMT-newstestB2020-deen
+      type: WMT-newstestB2020-deen
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 31.2
+       - name: chr-F2
+         type: chrf
+         value: 0.598
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: WMT-newstestB2020-ende
+      type: WMT-newstestB2020-ende
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 23.3
+       - name: chr-F2
+         type: chrf
+         value: 0.532
+  - task::
+      name: Translation afr-deu
+      type: translation
+      args: afr-deu
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: afr-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 21.6
+       - name: chr-F2
+         type: chrf
+         value: 0.524
+  - task::
+      name: Translation afr-eng
+      type: translation
+      args: afr-eng
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: afr-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 46.8
+       - name: chr-F2
+         type: chrf
+         value: 0.693
+  - task::
+      name: Translation afr-ltz
+      type: translation
+      args: afr-ltz
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: afr-ltz 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 5.6
+       - name: chr-F2
+         type: chrf
+         value: 0.338
+  - task::
+      name: Translation afr-nld
+      type: translation
+      args: afr-nld
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: afr-nld 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 18.4
+       - name: chr-F2
+         type: chrf
+         value: 0.509
+  - task::
+      name: Translation deu-afr
+      type: translation
+      args: deu-afr
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: deu-afr 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 21.4
+       - name: chr-F2
+         type: chrf
+         value: 0.534
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 33.8
+       - name: chr-F2
+         type: chrf
+         value: 0.616
+  - task::
+      name: Translation deu-ltz
+      type: translation
+      args: deu-ltz
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: deu-ltz 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 7.9
+       - name: chr-F2
+         type: chrf
+         value: 0.381
+  - task::
+      name: Translation deu-nld
+      type: translation
+      args: deu-nld
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: deu-nld 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 19.2
+       - name: chr-F2
+         type: chrf
+         value: 0.516
+  - task::
+      name: Translation eng-afr
+      type: translation
+      args: eng-afr
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: eng-afr 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 33.8
+       - name: chr-F2
+         type: chrf
+         value: 0.628
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 29.1
+       - name: chr-F2
+         type: chrf
+         value: 0.581
+  - task::
+      name: Translation eng-ltz
+      type: translation
+      args: eng-ltz
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: eng-ltz 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 6.3
+       - name: chr-F2
+         type: chrf
+         value: 0.351
+  - task::
+      name: Translation eng-nld
+      type: translation
+      args: eng-nld
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: eng-nld 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 21.0
+       - name: chr-F2
+         type: chrf
+         value: 0.533
+  - task::
+      name: Translation ltz-afr
+      type: translation
+      args: ltz-afr
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: ltz-afr 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 12.9
+       - name: chr-F2
+         type: chrf
+         value: 0.430
+  - task::
+      name: Translation ltz-deu
+      type: translation
+      args: ltz-deu
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: ltz-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 17.1
+       - name: chr-F2
+         type: chrf
+         value: 0.482
+  - task::
+      name: Translation ltz-eng
+      type: translation
+      args: ltz-eng
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: ltz-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 18.8
+       - name: chr-F2
+         type: chrf
+         value: 0.468
+  - task::
+      name: Translation ltz-nld
+      type: translation
+      args: ltz-nld
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: ltz-nld 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 10.7
+       - name: chr-F2
+         type: chrf
+         value: 0.409
+  - task::
+      name: Translation nld-afr
+      type: translation
+      args: nld-afr
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: nld-afr 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 16.8
+       - name: chr-F2
+         type: chrf
+         value: 0.494
+  - task::
+      name: Translation nld-deu
+      type: translation
+      args: nld-deu
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: nld-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 17.9
+       - name: chr-F2
+         type: chrf
+         value: 0.501
+  - task::
+      name: Translation nld-eng
+      type: translation
+      args: nld-eng
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: nld-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 25.6
+       - name: chr-F2
+         type: chrf
+         value: 0.551
+  - task::
+      name: Translation nld-ltz
+      type: translation
+      args: nld-ltz
+    dataset:
+      name: flores101-devtest
+      type: flores101-devtest
+      args: nld-ltz 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 4.7
+       - name: chr-F2
+         type: chrf
+         value: 0.324
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: multi30k_task2_test_2016
+      type: multi30k_task2_test_2016
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 3.1
+       - name: chr-F2
+         type: chrf
+         value: 0.201
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: multi30k_task2_test_2016
+      type: multi30k_task2_test_2016
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 2.3
+       - name: chr-F2
+         type: chrf
+         value: 0.258
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: multi30k_test_2016_flickr
+      type: multi30k_test_2016_flickr
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 32.2
+       - name: chr-F2
+         type: chrf
+         value: 0.546
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: multi30k_test_2016_flickr
+      type: multi30k_test_2016_flickr
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 28.8
+       - name: chr-F2
+         type: chrf
+         value: 0.582
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: multi30k_test_2017_flickr
+      type: multi30k_test_2017_flickr
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 32.7
+       - name: chr-F2
+         type: chrf
+         value: 0.561
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: multi30k_test_2017_flickr
+      type: multi30k_test_2017_flickr
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 27.6
+       - name: chr-F2
+         type: chrf
+         value: 0.573
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: multi30k_test_2017_mscoco
+      type: multi30k_test_2017_mscoco
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 25.5
+       - name: chr-F2
+         type: chrf
+         value: 0.499
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: multi30k_test_2017_mscoco
+      type: multi30k_test_2017_mscoco
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 22.0
+       - name: chr-F2
+         type: chrf
+         value: 0.514
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: multi30k_test_2018_flickr
+      type: multi30k_test_2018_flickr
+      args: deu-eng 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 30.0
+       - name: chr-F2
+         type: chrf
+         value: 0.535
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: multi30k_test_2018_flickr
+      type: multi30k_test_2018_flickr
+      args: eng-deu 
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 25.3
+       - name: chr-F2
+         type: chrf
+         value: 0.547
+  - task::
+      name: Translation afr-deu
       type: translation
       args: afr-deu
     dataset:
-      name: Tatoeba - Test
-      type: tatoeba
-      args: gmw-gmw
+      name: tatoeba-test
+      type: tatoeba-test
+      args: afr-deu v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 48.1
+       - name: chr-F2
+         type: chrf
+         value: 0.674
+  - task::
+      name: Translation afr-eng
+      type: translation
+      args: afr-eng
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: afr-eng v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 58.8
+       - name: chr-F2
+         type: chrf
+         value: 0.728
+  - task::
+      name: Translation afr-nld
+      type: translation
+      args: afr-nld
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: afr-nld v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 54.5
+       - name: chr-F2
+         type: chrf
+         value: 0.711
+  - task::
+      name: Translation ang-eng
+      type: translation
+      args: ang-eng
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: ang-eng v2021-03-30
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 7.8
+       - name: chr-F2
+         type: chrf
+         value: 0.215
+  - task::
+      name: Translation deu-afr
+      type: translation
+      args: deu-afr
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: deu-afr v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 52.4
+       - name: chr-F2
+         type: chrf
+         value: 0.696
+  - task::
+      name: Translation deu-eng
+      type: translation
+      args: deu-eng
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: deu-eng v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 42.1
+       - name: chr-F2
+         type: chrf
+         value: 0.609
+  - task::
+      name: Translation deu-ltz
+      type: translation
+      args: deu-ltz
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: deu-ltz v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 13.4
+       - name: chr-F2
+         type: chrf
+         value: 0.282
+  - task::
+      name: Translation deu-nds
+      type: translation
+      args: deu-nds
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: deu-nds v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 18.6
+       - name: chr-F2
+         type: chrf
+         value: 0.442
+  - task::
+      name: Translation deu-nld
+      type: translation
+      args: deu-nld
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: deu-nld v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 48.7
+       - name: chr-F2
+         type: chrf
+         value: 0.672
+  - task::
+      name: Translation deu-yid
+      type: translation
+      args: deu-yid
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: deu-yid v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 2.3
+       - name: chr-F2
+         type: chrf
+         value: 0.198
+  - task::
+      name: Translation eng-afr
+      type: translation
+      args: eng-afr
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: eng-afr v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 56.5
+       - name: chr-F2
+         type: chrf
+         value: 0.735
+  - task::
+      name: Translation eng-deu
+      type: translation
+      args: eng-deu
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: eng-deu v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 35.9
+       - name: chr-F2
+         type: chrf
+         value: 0.580
+  - task::
+      name: Translation eng-fry
+      type: translation
+      args: eng-fry
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: eng-fry v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 17.2
+       - name: chr-F2
+         type: chrf
+         value: 0.389
+  - task::
+      name: Translation eng-ltz
+      type: translation
+      args: eng-ltz
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: eng-ltz v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 14.0
+       - name: chr-F2
+         type: chrf
+         value: 0.288
+  - task::
+      name: Translation eng-nds
+      type: translation
+      args: eng-nds
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: eng-nds v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 16.6
+       - name: chr-F2
+         type: chrf
+         value: 0.412
+  - task::
+      name: Translation eng-nld
+      type: translation
+      args: eng-nld
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: eng-nld v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 48.3
+       - name: chr-F2
+         type: chrf
+         value: 0.663
+  - task::
+      name: Translation eng-yid
+      type: translation
+      args: eng-yid
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: eng-yid v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 2.8
+       - name: chr-F2
+         type: chrf
+         value: 0.207
+  - task::
+      name: Translation frr-deu
+      type: translation
+      args: frr-deu
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: frr-deu v2021-08-07
     metrics:
        - name: BLEU
          type: bleu
-         value: 48.5
+         value: 3.0
+       - name: chr-F2
+         type: chrf
+         value: 0.198
+  - task::
+      name: Translation fry-eng
+      type: translation
+      args: fry-eng
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: fry-eng v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 32.5
+       - name: chr-F2
+         type: chrf
+         value: 0.500
+  - task::
+      name: Translation fry-nld
+      type: translation
+      args: fry-nld
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: fry-nld v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 43.1
+       - name: chr-F2
+         type: chrf
+         value: 0.633
+  - task::
+      name: Translation gos-deu
+      type: translation
+      args: gos-deu
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: gos-deu v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 12.7
+       - name: chr-F2
+         type: chrf
+         value: 0.370
+  - task::
+      name: Translation gos-eng
+      type: translation
+      args: gos-eng
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: gos-eng v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 14.6
+       - name: chr-F2
+         type: chrf
+         value: 0.316
+  - task::
+      name: Translation gos-nld
+      type: translation
+      args: gos-nld
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: gos-nld v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 15.6
+       - name: chr-F2
+         type: chrf
+         value: 0.405
+  - task::
+      name: Translation gsw-eng
+      type: translation
+      args: gsw-eng
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: gsw-eng v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 14.0
+       - name: chr-F2
+         type: chrf
+         value: 0.312
+  - task::
+      name: Translation hrx-deu
+      type: translation
+      args: hrx-deu
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: hrx-deu v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 24.7
+       - name: chr-F2
+         type: chrf
+         value: 0.484
+  - task::
+      name: Translation hrx-eng
+      type: translation
+      args: hrx-eng
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: hrx-eng v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 20.4
+       - name: chr-F2
+         type: chrf
+         value: 0.362
+  - task::
+      name: Translation ltz-deu
+      type: translation
+      args: ltz-deu
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: ltz-deu v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 37.2
+       - name: chr-F2
+         type: chrf
+         value: 0.556
+  - task::
+      name: Translation ltz-eng
+      type: translation
+      args: ltz-eng
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: ltz-eng v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 32.4
+       - name: chr-F2
+         type: chrf
+         value: 0.485
+  - task::
+      name: Translation ltz-nld
+      type: translation
+      args: ltz-nld
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: ltz-nld v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 39.3
+       - name: chr-F2
+         type: chrf
+         value: 0.534
+  - task::
+      name: Translation nds-deu
+      type: translation
+      args: nds-deu
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: nds-deu v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 34.5
+       - name: chr-F2
+         type: chrf
+         value: 0.572
+  - task::
+      name: Translation nds-eng
+      type: translation
+      args: nds-eng
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: nds-eng v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 29.9
+       - name: chr-F2
+         type: chrf
+         value: 0.493
+  - task::
+      name: Translation nds-nld
+      type: translation
+      args: nds-nld
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: nds-nld v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 42.3
+       - name: chr-F2
+         type: chrf
+         value: 0.621
+  - task::
+      name: Translation nld-afr
+      type: translation
+      args: nld-afr
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: nld-afr v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 58.8
+       - name: chr-F2
+         type: chrf
+         value: 0.755
+  - task::
+      name: Translation nld-deu
+      type: translation
+      args: nld-deu
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: nld-deu v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 50.4
+       - name: chr-F2
+         type: chrf
+         value: 0.686
+  - task::
+      name: Translation nld-eng
+      type: translation
+      args: nld-eng
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: nld-eng v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 53.1
+       - name: chr-F2
+         type: chrf
+         value: 0.690
+  - task::
+      name: Translation nld-fry
+      type: translation
+      args: nld-fry
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: nld-fry v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 25.1
+       - name: chr-F2
+         type: chrf
+         value: 0.478
+  - task::
+      name: Translation nld-ltz
+      type: translation
+      args: nld-ltz
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: nld-ltz v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 14.1
+       - name: chr-F2
+         type: chrf
+         value: 0.301
+  - task::
+      name: Translation nld-nds
+      type: translation
+      args: nld-nds
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: nld-nds v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 21.4
+       - name: chr-F2
+         type: chrf
+         value: 0.462
+  - task::
+      name: Translation swg-deu
+      type: translation
+      args: swg-deu
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: swg-deu v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 8.6
+       - name: chr-F2
+         type: chrf
+         value: 0.335
+  - task::
+      name: Translation yid-deu
+      type: translation
+      args: yid-deu
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: yid-deu v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 17.6
+       - name: chr-F2
+         type: chrf
+         value: 0.381
+  - task::
+      name: Translation yid-eng
+      type: translation
+      args: yid-eng
+    dataset:
+      name: tatoeba-test
+      type: tatoeba-test
+      args: yid-eng v2021-08-07
+    metrics:
+       - name: BLEU
+         type: bleu
+         value: 14.2
+       - name: chr-F2
+         type: chrf
+         value: 0.324
 ---
 # opus-mt-tc-base-gmw-gmw
 
@@ -31,6 +1544,32 @@ This model is part of the [OPUS-MT project](https://github.com/Helsinki-NLP/Opus
 
 * Publications: [OPUS-MT – Building open translation services for the World](https://aclanthology.org/2020.eamt-1.61/) and [The Tatoeba Translation Challenge – Realistic Data Sets for Low Resource and Multilingual MT](https://aclanthology.org/2020.wmt-1.139/) (Please, cite if you use this model.)
 
+```
+@inproceedings{tiedemann-thottingal-2020-opus,
+    title = "{OPUS}-{MT} {--} Building open translation services for the World",
+    author = {Tiedemann, J{\"o}rg  and Thottingal, Santhosh},
+    booktitle = "Proceedings of the 22nd Annual Conference of the European Association for Machine Translation",
+    month = nov,
+    year = "2020",
+    address = "Lisboa, Portugal",
+    publisher = "European Association for Machine Translation",
+    url = "https://aclanthology.org/2020.eamt-1.61",
+    pages = "479--480",
+}
+
+@inproceedings{tiedemann-2020-tatoeba,
+    title = "The Tatoeba Translation Challenge {--} Realistic Data Sets for Low Resource and Multilingual {MT}",
+    author = {Tiedemann, J{\"o}rg},
+    booktitle = "Proceedings of the Fifth Conference on Machine Translation",
+    month = nov,
+    year = "2020",
+    address = "Online",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2020.wmt-1.139",
+    pages = "1174--1182",
+}
+```
+
 ## Model info
 
 * Release: 2021-02-23
@@ -41,82 +1580,136 @@ This model is part of the [OPUS-MT project](https://github.com/Helsinki-NLP/Opus
 * data: opus ([source](https://github.com/Helsinki-NLP/Tatoeba-Challenge))
 * tokenization: SentencePiece (spm32k,spm32k)
 * original model: [opus-2021-02-23.zip](https://object.pouta.csc.fi/Tatoeba-MT-models/gmw-gmw/opus-2021-02-23.zip)
-* more information: [OPUS-MT models (gmw-gmw)](https://github.com/Helsinki-NLP/Tatoeba-Challenge/tree/master/models/gmw-gmw/README.md)
+* more information released models: [OPUS-MT gmw-gmw README](https://github.com/Helsinki-NLP/Tatoeba-Challenge/tree/master/models/gmw-gmw/README.md)
+* more information about the model: [MarianMT](https://huggingface.co/docs/transformers/model_doc/marian)
 
 This is a multilingual translation model with multiple target languages. A sentence initial language token is required in the form of `>>id<<` (id = valid target language ID), e.g. `>>afr<<`
 
 ## Usage
 
-You can use OPUS-MT models with the transformers pipelines, for example:
+A short example code:
+
+```python
+from transformers import MarianMTModel, MarianTokenizer
+
+src_text = [
+    ">>nld<< You need help.",
+    ">>afr<< I love your son."
+]
+
+model_name = "pytorch-models/opus-mt-tc-base-en-fi"
+tokenizer = MarianTokenizer.from_pretrained(model_name)
+model = MarianMTModel.from_pretrained(model_name)
+translated = model.generate(**tokenizer(src_text, return_tensors="pt", padding=True))
+
+for t in translated:
+    print( tokenizer.decode(t, skip_special_tokens=True) )
+
+# expected output:
+#     Je hebt hulp nodig.
+#     Ek is lief vir jou seun.
+```
+
+You can also use OPUS-MT models with the transformers pipelines, for example:
 
 ```python
 from transformers import pipeline
 pipe = pipeline("translation", model="Helsinki-NLP/opus-mt-tc-base-gmw-gmw")
-print(pipe(">>afr<< Replace this with text in an accepted source language."))
+print(pipe(>>nld<< You need help.))
+
+# expected output: Je hebt hulp nodig.
 ```
 
 ## Benchmarks
 
 * test set translations: [opus-2021-02-23.test.txt](https://object.pouta.csc.fi/Tatoeba-MT-models/gmw-gmw/opus-2021-02-23.test.txt)
 * test set scores: [opus-2021-02-23.eval.txt](https://object.pouta.csc.fi/Tatoeba-MT-models/gmw-gmw/opus-2021-02-23.eval.txt)
+* benchmark results: [benchmarks.tsv](benchmarks.tsv)
+* benchmark output: [benchmark_translations.zip](benchmark_translations.zip)
 
-| langpair | testset | BLEU  | chr-F | #sent | #words | BP |
-|----------|---------|-------|-------|-------|--------|----|
-| afr-deu | Tatoeba-test 	| 48.5 	| 0.677 	| 1583 	| 9105 	| 1.000 |
-| afr-eng | Tatoeba-test 	| 58.7 	| 0.727 	| 1374 	| 9622 	| 0.995 |
-| afr-nld | Tatoeba-test 	| 54.7 	| 0.713 	| 1056 	| 6710 	| 0.989 |
-| deu-afr | Tatoeba-test 	| 52.4 	| 0.697 	| 1583 	| 9507 	| 1.000 |
-| deu-eng | newssyscomb2009 	| 25.4 	| 0.527 	| 502 	| 11821 	| 0.986 |
-| deu-eng | news-test2008 	| 23.9 	| 0.519 	| 2051 	| 49380 	| 0.992 |
-| deu-eng | newstest2009 	| 23.5 	| 0.517 	| 2525 	| 65402 	| 0.978 |
-| deu-eng | newstest2010 	| 26.1 	| 0.548 	| 2489 	| 61724 	| 1.000 |
-| deu-eng | newstest2011 	| 23.9 	| 0.525 	| 3003 	| 74681 	| 1.000 |
-| deu-eng | newstest2012 	| 25.0 	| 0.533 	| 3003 	| 72812 	| 1.000 |
-| deu-eng | newstest2013 	| 27.7 	| 0.549 	| 3000 	| 64505 	| 1.000 |
-| deu-eng | newstest2014-deen 	| 27.4 	| 0.549 	| 3003 	| 67337 	| 0.977 |
-| deu-eng | newstest2015-ende 	| 28.8 	| 0.554 	| 2169 	| 46443 	| 0.973 |
-| deu-eng | newstest2016-ende 	| 33.7 	| 0.598 	| 2999 	| 64126 	| 1.000 |
-| deu-eng | newstest2017-ende 	| 29.6 	| 0.562 	| 3004 	| 64399 	| 0.979 |
-| deu-eng | newstest2018-ende 	| 36.3 	| 0.611 	| 2998 	| 67013 	| 0.977 |
-| deu-eng | newstest2019-deen 	| 32.7 	| 0.585 	| 2000 	| 39282 	| 0.984 |
-| deu-eng | Tatoeba-test 	| 44.7 	| 0.629 	| 10000 	| 81233 	| 0.975 |
-| deu-nds | Tatoeba-test 	| 18.7 	| 0.444 	| 10000 	| 76144 	| 0.988 |
-| deu-nld | Tatoeba-test 	| 48.7 	| 0.672 	| 10000 	| 73546 	| 0.969 |
-| eng-afr | Tatoeba-test 	| 56.5 	| 0.735 	| 1374 	| 10317 	| 0.984 |
-| eng-deu | newssyscomb2009 	| 19.4 	| 0.503 	| 502 	| 11271 	| 0.991 |
-| eng-deu | news-test2008 	| 19.5 	| 0.493 	| 2051 	| 47427 	| 0.996 |
-| eng-deu | newstest2009 	| 18.8 	| 0.499 	| 2525 	| 62816 	| 0.993 |
-| eng-deu | newstest2010 	| 20.8 	| 0.509 	| 2489 	| 61511 	| 0.958 |
-| eng-deu | newstest2011 	| 19.2 	| 0.493 	| 3003 	| 72981 	| 0.980 |
-| eng-deu | newstest2012 	| 19.6 	| 0.494 	| 3003 	| 72886 	| 0.960 |
-| eng-deu | newstest2013 	| 22.8 	| 0.518 	| 3000 	| 63737 	| 0.974 |
-| eng-deu | newstest2015-ende 	| 25.8 	| 0.545 	| 2169 	| 44260 	| 1.000 |
-| eng-deu | newstest2016-ende 	| 30.3 	| 0.581 	| 2999 	| 62670 	| 0.989 |
-| eng-deu | newstest2017-ende 	| 24.2 	| 0.537 	| 3004 	| 61291 	| 1.000 |
-| eng-deu | newstest2018-ende 	| 35.5 	| 0.616 	| 2998 	| 64276 	| 1.000 |
-| eng-deu | newstest2019-ende 	| 31.6 	| 0.586 	| 1997 	| 48969 	| 0.973 |
-| eng-deu | Tatoeba-test 	| 37.8 	| 0.591 	| 10000 	| 83347 	| 0.991 |
-| eng-nds | Tatoeba-test 	| 16.5 	| 0.411 	| 2500 	| 18264 	| 0.992 |
-| eng-nld | Tatoeba-test 	| 50.3 	| 0.677 	| 10000 	| 71436 	| 0.979 |
-| fry-deu | Tatoeba-test 	| 28.7 	| 0.545 	| 66 	| 432 	| 1.000 |
-| fry-eng | Tatoeba-test 	| 31.9 	| 0.496 	| 205 	| 1500 	| 1.000 |
-| fry-nld | Tatoeba-test 	| 43.0 	| 0.634 	| 233 	| 1672 	| 1.000 |
-| gos-nld | Tatoeba-test 	| 15.9 	| 0.409 	| 1852 	| 9903 	| 0.959 |
-| hrx-deu | Tatoeba-test 	| 24.7 	| 0.487 	| 471 	| 2805 	| 0.984 |
-| ltz-deu | Tatoeba-test 	| 36.6 	| 0.552 	| 337 	| 2144 	| 1.000 |
-| ltz-eng | Tatoeba-test 	| 31.4 	| 0.477 	| 283 	| 1751 	| 1.000 |
-| ltz-nld | Tatoeba-test 	| 37.5 	| 0.523 	| 273 	| 1567 	| 1.000 |
-| multi-multi | Tatoeba-test 	| 37.1 	| 0.569 	| 10000 	| 73153 	| 1.000 |
-| nds-deu | Tatoeba-test 	| 34.5 	| 0.572 	| 10000 	| 74571 	| 1.000 |
-| nds-eng | Tatoeba-test 	| 29.6 	| 0.492 	| 2500 	| 17589 	| 1.000 |
-| nds-nld | Tatoeba-test 	| 42.2 	| 0.621 	| 1657 	| 11490 	| 0.994 |
-| nld-afr | Tatoeba-test 	| 59.0 	| 0.756 	| 1056 	| 6823 	| 1.000 |
-| nld-deu | Tatoeba-test 	| 50.6 	| 0.688 	| 10000 	| 72438 	| 1.000 |
-| nld-eng | Tatoeba-test 	| 54.5 	| 0.702 	| 10000 	| 69848 	| 0.975 |
-| nld-fry | Tatoeba-test 	| 23.3 	| 0.462 	| 233 	| 1679 	| 1.000 |
-| nld-nds | Tatoeba-test 	| 21.7 	| 0.462 	| 1657 	| 11711 	| 0.998 |
-| pdc-eng | Tatoeba-test 	| 24.3 	| 0.402 	| 53 	| 399 	| 1.000 |
-| yid-nld | Tatoeba-test 	| 21.3 	| 0.402 	| 55 	| 323 	| 1.000 |
+| langpair | testset | chr-F | BLEU  | #sent | #words |
+|----------|---------|-------|-------|-------|--------|
+| afr-deu | tatoeba-test-v2021-08-07 | 0.674 | 48.1 | 1583 | 9105 |
+| afr-eng | tatoeba-test-v2021-08-07 | 0.728 | 58.8 | 1374 | 9622 |
+| afr-nld | tatoeba-test-v2021-08-07 | 0.711 | 54.5 | 1056 | 6710 |
+| deu-afr | tatoeba-test-v2021-08-07 | 0.696 | 52.4 | 1583 | 9507 |
+| deu-eng | tatoeba-test-v2021-08-07 | 0.609 | 42.1 | 17565 | 149462 |
+| deu-nds | tatoeba-test-v2021-08-07 | 0.442 | 18.6 | 9999 | 76137 |
+| deu-nld | tatoeba-test-v2021-08-07 | 0.672 | 48.7 | 10218 | 75235 |
+| eng-afr | tatoeba-test-v2021-08-07 | 0.735 | 56.5 | 1374 | 10317 |
+| eng-deu | tatoeba-test-v2021-08-07 | 0.580 | 35.9 | 17565 | 151568 |
+| eng-nds | tatoeba-test-v2021-08-07 | 0.412 | 16.6 | 2500 | 18264 |
+| eng-nld | tatoeba-test-v2021-08-07 | 0.663 | 48.3 | 12696 | 91796 |
+| fry-eng | tatoeba-test-v2021-08-07 | 0.500 | 32.5 | 220 | 1573 |
+| fry-nld | tatoeba-test-v2021-08-07 | 0.633 | 43.1 | 260 | 1854 |
+| gos-nld | tatoeba-test-v2021-08-07 | 0.405 | 15.6 | 1852 | 9903 |
+| hrx-deu | tatoeba-test-v2021-08-07 | 0.484 | 24.7 | 471 | 2805 |
+| hrx-eng | tatoeba-test-v2021-08-07 | 0.362 | 20.4 | 221 | 1235 |
+| ltz-deu | tatoeba-test-v2021-08-07 | 0.556 | 37.2 | 347 | 2208 |
+| ltz-eng | tatoeba-test-v2021-08-07 | 0.485 | 32.4 | 293 | 1840 |
+| ltz-nld | tatoeba-test-v2021-08-07 | 0.534 | 39.3 | 292 | 1685 |
+| nds-deu | tatoeba-test-v2021-08-07 | 0.572 | 34.5 | 9999 | 74564 |
+| nds-eng | tatoeba-test-v2021-08-07 | 0.493 | 29.9 | 2500 | 17589 |
+| nds-nld | tatoeba-test-v2021-08-07 | 0.621 | 42.3 | 1657 | 11490 |
+| nld-afr | tatoeba-test-v2021-08-07 | 0.755 | 58.8 | 1056 | 6823 |
+| nld-deu | tatoeba-test-v2021-08-07 | 0.686 | 50.4 | 10218 | 74131 |
+| nld-eng | tatoeba-test-v2021-08-07 | 0.690 | 53.1 | 12696 | 89978 |
+| nld-fry | tatoeba-test-v2021-08-07 | 0.478 | 25.1 | 260 | 1857 |
+| nld-nds | tatoeba-test-v2021-08-07 | 0.462 | 21.4 | 1657 | 11711 |
+| afr-deu | flores101-devtest | 0.524 | 21.6 | 1012 | 25094 |
+| afr-eng | flores101-devtest | 0.693 | 46.8 | 1012 | 24721 |
+| afr-nld | flores101-devtest | 0.509 | 18.4 | 1012 | 25467 |
+| deu-afr | flores101-devtest | 0.534 | 21.4 | 1012 | 25740 |
+| deu-eng | flores101-devtest | 0.616 | 33.8 | 1012 | 24721 |
+| deu-nld | flores101-devtest | 0.516 | 19.2 | 1012 | 25467 |
+| eng-afr | flores101-devtest | 0.628 | 33.8 | 1012 | 25740 |
+| eng-deu | flores101-devtest | 0.581 | 29.1 | 1012 | 25094 |
+| eng-nld | flores101-devtest | 0.533 | 21.0 | 1012 | 25467 |
+| ltz-afr | flores101-devtest | 0.430 | 12.9 | 1012 | 25740 |
+| ltz-deu | flores101-devtest | 0.482 | 17.1 | 1012 | 25094 |
+| ltz-eng | flores101-devtest | 0.468 | 18.8 | 1012 | 24721 |
+| ltz-nld | flores101-devtest | 0.409 | 10.7 | 1012 | 25467 |
+| nld-afr | flores101-devtest | 0.494 | 16.8 | 1012 | 25740 |
+| nld-deu | flores101-devtest | 0.501 | 17.9 | 1012 | 25094 |
+| nld-eng | flores101-devtest | 0.551 | 25.6 | 1012 | 24721 |
+| deu-eng | multi30k_test_2016_flickr | 0.546 | 32.2 | 1000 | 12955 |
+| eng-deu | multi30k_test_2016_flickr | 0.582 | 28.8 | 1000 | 12106 |
+| deu-eng | multi30k_test_2017_flickr | 0.561 | 32.7 | 1000 | 11374 |
+| eng-deu | multi30k_test_2017_flickr | 0.573 | 27.6 | 1000 | 10755 |
+| deu-eng | multi30k_test_2017_mscoco | 0.499 | 25.5 | 461 | 5231 |
+| eng-deu | multi30k_test_2017_mscoco | 0.514 | 22.0 | 461 | 5158 |
+| deu-eng | multi30k_test_2018_flickr | 0.535 | 30.0 | 1071 | 14689 |
+| eng-deu | multi30k_test_2018_flickr | 0.547 | 25.3 | 1071 | 13703 |
+| deu-eng | newssyscomb2009 | 0.527 | 25.4 | 502 | 11818 |
+| eng-deu | newssyscomb2009 | 0.504 | 19.3 | 502 | 11271 |
+| deu-eng | news-test2008 | 0.518 | 23.8 | 2051 | 49380 |
+| eng-deu | news-test2008 | 0.492 | 19.3 | 2051 | 47447 |
+| deu-eng | newstest2009 | 0.516 | 23.4 | 2525 | 65399 |
+| eng-deu | newstest2009 | 0.498 | 18.8 | 2525 | 62816 |
+| deu-eng | newstest2010 | 0.546 | 25.8 | 2489 | 61711 |
+| eng-deu | newstest2010 | 0.508 | 20.7 | 2489 | 61503 |
+| deu-eng | newstest2011 | 0.524 | 23.7 | 3003 | 74681 |
+| eng-deu | newstest2011 | 0.493 | 19.2 | 3003 | 72981 |
+| deu-eng | newstest2012 | 0.532 | 24.8 | 3003 | 72812 |
+| eng-deu | newstest2012 | 0.493 | 19.5 | 3003 | 72886 |
+| deu-eng | newstest2013 | 0.548 | 27.7 | 3000 | 64505 |
+| eng-deu | newstest2013 | 0.517 | 22.5 | 3000 | 63737 |
+| deu-eng | newstest2014-deen | 0.548 | 27.3 | 3003 | 67337 |
+| eng-deu | newstest2014-deen | 0.532 | 22.0 | 3003 | 62688 |
+| deu-eng | newstest2015-deen | 0.553 | 28.6 | 2169 | 46443 |
+| eng-deu | newstest2015-ende | 0.544 | 25.7 | 2169 | 44260 |
+| deu-eng | newstest2016-deen | 0.596 | 33.3 | 2999 | 64119 |
+| eng-deu | newstest2016-ende | 0.580 | 30.0 | 2999 | 62669 |
+| deu-eng | newstest2017-deen | 0.561 | 29.5 | 3004 | 64399 |
+| eng-deu | newstest2017-ende | 0.535 | 24.1 | 3004 | 61287 |
+| deu-eng | newstest2018-deen | 0.610 | 36.1 | 2998 | 67012 |
+| eng-deu | newstest2018-ende | 0.613 | 35.4 | 2998 | 64276 |
+| deu-eng | newstest2019-deen | 0.582 | 32.3 | 2000 | 39227 |
+| eng-deu | newstest2019-ende | 0.583 | 31.2 | 1997 | 48746 |
+| deu-eng | newstest2020-deen | 0.604 | 32.0 | 785 | 38220 |
+| eng-deu | newstest2020-ende | 0.542 | 23.9 | 1418 | 52383 |
+| deu-eng | newstestB2020-deen | 0.598 | 31.2 | 785 | 37696 |
+| eng-deu | newstestB2020-ende | 0.532 | 23.3 | 1418 | 53092 |
 
 ## Acknowledgements
 
@@ -125,6 +1718,6 @@ The work is supported by the [European Language Grid](https://www.european-langu
 ## Model conversion info
 
 * transformers version: 4.12.3
-* OPUS-MT git hash: b250e2e
-* port time: Thu Jan 27 23:08:33 EET 2022
+* OPUS-MT git hash: 64dc362
+* port time: Fri Feb 11 00:08:20 EET 2022
 * port machine: LM0-400-22516.local

benchmark_translations.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a99fdfc9a2526feb7568e5e87cefd4fb1417c9cbae893733eb24928f99d351d
+size 30803803

benchmarks.tsv

@@ -0,0 +1,199 @@
+afr-deu	flores101-dev	0.516	22.2	997	24097
+afr-eng	flores101-dev	0.696	47.0	997	23555
+afr-ltz	flores101-dev	0.337	6.2	997	23906
+afr-nld	flores101-dev	0.509	18.6	997	24342
+deu-afr	flores101-dev	0.522	21.1	997	24677
+deu-eng	flores101-dev	0.617	34.5	997	23555
+deu-ltz	flores101-dev	0.381	8.8	997	23906
+deu-nld	flores101-dev	0.515	20.0	997	24342
+eng-afr	flores101-dev	0.625	33.2	997	24677
+eng-deu	flores101-dev	0.575	29.2	997	24097
+eng-ltz	flores101-dev	0.351	6.8	997	23906
+eng-nld	flores101-dev	0.535	21.6	997	24342
+ltz-afr	flores101-dev	0.430	13.7	997	24677
+ltz-deu	flores101-dev	0.485	18.5	997	24097
+ltz-eng	flores101-dev	0.474	20.1	997	23555
+ltz-nld	flores101-dev	0.412	11.5	997	24342
+nld-afr	flores101-dev	0.493	16.8	997	24677
+nld-deu	flores101-dev	0.494	19.0	997	24097
+nld-eng	flores101-dev	0.552	25.5	997	23555
+nld-ltz	flores101-dev	0.324	5.5	997	23906
+afr-deu	flores101-devtest	0.524	21.6	1012	25094
+afr-eng	flores101-devtest	0.693	46.8	1012	24721
+afr-ltz	flores101-devtest	0.338	5.6	1012	25087
+afr-nld	flores101-devtest	0.509	18.4	1012	25467
+deu-afr	flores101-devtest	0.534	21.4	1012	25740
+deu-eng	flores101-devtest	0.616	33.8	1012	24721
+deu-ltz	flores101-devtest	0.381	7.9	1012	25087
+deu-nld	flores101-devtest	0.516	19.2	1012	25467
+eng-afr	flores101-devtest	0.628	33.8	1012	25740
+eng-deu	flores101-devtest	0.581	29.1	1012	25094
+eng-ltz	flores101-devtest	0.351	6.3	1012	25087
+eng-nld	flores101-devtest	0.533	21.0	1012	25467
+ltz-afr	flores101-devtest	0.430	12.9	1012	25740
+ltz-deu	flores101-devtest	0.482	17.1	1012	25094
+ltz-eng	flores101-devtest	0.468	18.8	1012	24721
+ltz-nld	flores101-devtest	0.409	10.7	1012	25467
+nld-afr	flores101-devtest	0.494	16.8	1012	25740
+nld-deu	flores101-devtest	0.501	17.9	1012	25094
+nld-eng	flores101-devtest	0.551	25.6	1012	24721
+nld-ltz	flores101-devtest	0.324	4.7	1012	25087
+deu-eng	multi30k_task2_test_2016	0.201	3.1	5000	67382
+eng-deu	multi30k_task2_test_2016	0.258	2.3	5000	51501
+deu-eng	multi30k_test_2016_flickr	0.546	32.2	1000	12955
+eng-deu	multi30k_test_2016_flickr	0.582	28.8	1000	12106
+deu-eng	multi30k_test_2017_flickr	0.561	32.7	1000	11374
+eng-deu	multi30k_test_2017_flickr	0.573	27.6	1000	10755
+deu-eng	multi30k_test_2017_mscoco	0.499	25.5	461	5231
+eng-deu	multi30k_test_2017_mscoco	0.514	22.0	461	5158
+deu-eng	multi30k_test_2018_flickr	0.535	30.0	1071	14689
+eng-deu	multi30k_test_2018_flickr	0.547	25.3	1071	13703
+deu-eng	newssyscomb2009	0.527	25.4	502	11818
+eng-deu	newssyscomb2009	0.504	19.3	502	11271
+deu-eng	news-test2008	0.518	23.8	2051	49380
+eng-deu	news-test2008	0.492	19.3	2051	47447
+deu-eng	newstest2009	0.516	23.4	2525	65399
+eng-deu	newstest2009	0.498	18.8	2525	62816
+deu-eng	newstest2010	0.546	25.8	2489	61711
+eng-deu	newstest2010	0.508	20.7	2489	61503
+deu-eng	newstest2011	0.524	23.7	3003	74681
+eng-deu	newstest2011	0.493	19.2	3003	72981
+deu-eng	newstest2012	0.532	24.8	3003	72812
+eng-deu	newstest2012	0.493	19.5	3003	72886
+deu-eng	newstest2013	0.548	27.7	3000	64505
+eng-deu	newstest2013	0.517	22.5	3000	63737
+deu-eng	newstest2014-deen	0.548	27.3	3003	67337
+eng-deu	newstest2014-deen	0.532	22.0	3003	62688
+deu-eng	newstest2015-deen	0.553	28.6	2169	46443
+eng-deu	newstest2015-ende	0.544	25.7	2169	44260
+deu-eng	newstest2016-deen	0.596	33.3	2999	64119
+eng-deu	newstest2016-ende	0.580	30.0	2999	62669
+deu-eng	newstest2017-deen	0.561	29.5	3004	64399
+eng-deu	newstest2017-ende	0.535	24.1	3004	61287
+deu-eng	newstest2018-deen	0.610	36.1	2998	67012
+eng-deu	newstest2018-ende	0.613	35.4	2998	64276
+deu-eng	newstest2019-deen	0.582	32.3	2000	39227
+eng-deu	newstest2019-ende	0.583	31.2	1997	48746
+deu-eng	newstest2020-deen	0.604	32.0	785	38220
+eng-deu	newstest2020-ende	0.542	23.9	1418	52383
+deu-eng	newstestB2020-deen	0.598	31.2	785	37696
+eng-deu	newstestB2020-ende	0.532	23.3	1418	53092
+afr-deu	tatoeba-test-v2020-07-28	0.674	48.1	1583	9105
+afr-eng	tatoeba-test-v2020-07-28	0.728	58.8	1374	9622
+afr-nld	tatoeba-test-v2020-07-28	0.711	54.5	1056	6710
+deu-afr	tatoeba-test-v2020-07-28	0.696	52.4	1583	9507
+deu-eng	tatoeba-test-v2020-07-28	0.629	44.8	10000	81233
+deu-ltz	tatoeba-test-v2020-07-28	0.274	12.7	337	2135
+deu-nds	tatoeba-test-v2020-07-28	0.442	18.6	10000	76144
+deu-nld	tatoeba-test-v2020-07-28	0.672	48.8	10000	73546
+deu-yid	tatoeba-test-v2020-07-28	0.189	2.2	556	3425
+eng-afr	tatoeba-test-v2020-07-28	0.735	56.5	1374	10317
+eng-deu	tatoeba-test-v2020-07-28	0.591	37.7	10000	83347
+eng-fry	tatoeba-test-v2020-07-28	0.376	15.3	205	1529
+eng-ltz	tatoeba-test-v2020-07-28	0.280	13.8	283	1733
+eng-nds	tatoeba-test-v2020-07-28	0.412	16.6	2500	18264
+eng-nld	tatoeba-test-v2020-07-28	0.676	50.1	10000	71436
+eng-yid	tatoeba-test-v2020-07-28	0.199	2.2	1168	8094
+fry-eng	tatoeba-test-v2020-07-28	0.497	32.1	205	1500
+fry-nld	tatoeba-test-v2020-07-28	0.631	42.8	233	1672
+gos-eng	tatoeba-test-v2020-07-28	0.315	14.5	1152	5622
+gos-nld	tatoeba-test-v2020-07-28	0.405	15.7	1852	9903
+gsw-eng	tatoeba-test-v2020-07-28	0.312	14.0	205	990
+hrx-deu	tatoeba-test-v2020-07-28	0.484	24.7	471	2805
+hrx-eng	tatoeba-test-v2020-07-28	0.362	20.4	221	1235
+ltz-deu	tatoeba-test-v2020-07-28	0.553	36.9	337	2144
+ltz-eng	tatoeba-test-v2020-07-28	0.479	31.4	283	1751
+ltz-nld	tatoeba-test-v2020-07-28	0.518	37.4	273	1567
+nds-deu	tatoeba-test-v2020-07-28	0.572	34.5	10000	74571
+nds-eng	tatoeba-test-v2020-07-28	0.493	29.9	2500	17589
+nds-nld	tatoeba-test-v2020-07-28	0.621	42.3	1657	11490
+nld-afr	tatoeba-test-v2020-07-28	0.755	58.8	1056	6823
+nld-deu	tatoeba-test-v2020-07-28	0.687	50.4	10000	72438
+nld-eng	tatoeba-test-v2020-07-28	0.702	54.5	10000	69848
+nld-fry	tatoeba-test-v2020-07-28	0.465	23.7	233	1679
+nld-ltz	tatoeba-test-v2020-07-28	0.288	13.3	273	1532
+nld-nds	tatoeba-test-v2020-07-28	0.461	21.4	1657	11711
+swg-deu	tatoeba-test-v2020-07-28	0.336	8.6	1523	15632
+yid-deu	tatoeba-test-v2020-07-28	0.381	17.8	556	3332
+yid-eng	tatoeba-test-v2020-07-28	0.337	15.2	1168	7741
+afr-deu	tatoeba-test-v2021-03-30	0.674	48.1	1583	9105
+afr-eng	tatoeba-test-v2021-03-30	0.728	58.8	1374	9622
+afr-nld	tatoeba-test-v2021-03-30	0.711	54.5	1058	6720
+ang-eng	tatoeba-test-v2021-03-30	0.215	7.8	200	2132
+deu-afr	tatoeba-test-v2021-03-30	0.696	52.4	1583	9507
+deu-eng	tatoeba-test-v2021-03-30	0.618	43.4	12664	105121
+deu-ltz	tatoeba-test-v2021-03-30	0.278	13.0	350	2227
+deu-nds	tatoeba-test-v2021-03-30	0.442	18.6	10000	76144
+deu-nld	tatoeba-test-v2021-03-30	0.672	48.8	10124	74568
+deu-yid	tatoeba-test-v2021-03-30	0.192	2.2	830	5207
+eng-afr	tatoeba-test-v2021-03-30	0.735	56.5	1374	10317
+eng-deu	tatoeba-test-v2021-03-30	0.584	36.7	12664	107460
+eng-fry	tatoeba-test-v2021-03-30	0.386	16.7	221	1655
+eng-ltz	tatoeba-test-v2021-03-30	0.284	14.2	299	1833
+eng-nds	tatoeba-test-v2021-03-30	0.412	16.6	2500	18264
+eng-nld	tatoeba-test-v2021-03-30	0.668	49.2	11660	83811
+eng-yid	tatoeba-test-v2021-03-30	0.206	2.9	1888	12516
+frr-deu	tatoeba-test-v2021-03-30	0.199	3.0	279	1886
+fry-eng	tatoeba-test-v2021-03-30	0.498	32.2	221	1624
+fry-nld	tatoeba-test-v2021-03-30	0.635	43.4	265	1884
+gos-deu	tatoeba-test-v2021-03-30	0.369	13.2	210	1175
+gos-eng	tatoeba-test-v2021-03-30	0.314	14.4	1193	5819
+gos-nld	tatoeba-test-v2021-03-30	0.405	15.7	1852	9903
+gsw-eng	tatoeba-test-v2021-03-30	0.311	13.8	210	1021
+hrx-deu	tatoeba-test-v2021-03-30	0.484	24.7	473	2813
+hrx-eng	tatoeba-test-v2021-03-30	0.364	20.3	223	1249
+ltz-deu	tatoeba-test-v2021-03-30	0.556	37.2	350	2229
+ltz-eng	tatoeba-test-v2021-03-30	0.487	32.6	299	1845
+ltz-nld	tatoeba-test-v2021-03-30	0.531	38.9	306	1763
+nds-deu	tatoeba-test-v2021-03-30	0.572	34.5	10000	74571
+nds-eng	tatoeba-test-v2021-03-30	0.493	29.9	2500	17589
+nds-nld	tatoeba-test-v2021-03-30	0.621	42.3	1657	11490
+nld-afr	tatoeba-test-v2021-03-30	0.754	58.7	1058	6833
+nld-deu	tatoeba-test-v2021-03-30	0.687	50.5	10124	73449
+nld-eng	tatoeba-test-v2021-03-30	0.693	53.5	11660	81885
+nld-fry	tatoeba-test-v2021-03-30	0.474	24.4	265	1889
+nld-ltz	tatoeba-test-v2021-03-30	0.296	13.5	306	1735
+nld-nds	tatoeba-test-v2021-03-30	0.461	21.4	1657	11711
+swg-deu	tatoeba-test-v2021-03-30	0.336	8.6	1523	15632
+yid-deu	tatoeba-test-v2021-03-30	0.381	17.4	830	5045
+yid-eng	tatoeba-test-v2021-03-30	0.333	15.3	1888	11810
+afr-deu	tatoeba-test-v2021-08-07	0.674	48.1	1583	9105
+afr-eng	tatoeba-test-v2021-08-07	0.728	58.8	1374	9622
+afr-nld	tatoeba-test-v2021-08-07	0.711	54.5	1056	6710
+deu-afr	tatoeba-test-v2021-08-07	0.696	52.4	1583	9507
+deu-eng	tatoeba-test-v2021-08-07	0.609	42.1	17565	149462
+deu-ltz	tatoeba-test-v2021-08-07	0.282	13.4	347	2206
+deu-nds	tatoeba-test-v2021-08-07	0.442	18.6	9999	76137
+deu-nld	tatoeba-test-v2021-08-07	0.672	48.7	10218	75235
+deu-yid	tatoeba-test-v2021-08-07	0.198	2.3	853	5355
+eng-afr	tatoeba-test-v2021-08-07	0.735	56.5	1374	10317
+eng-deu	tatoeba-test-v2021-08-07	0.580	35.9	17565	151568
+eng-fry	tatoeba-test-v2021-08-07	0.389	17.2	220	1600
+eng-ltz	tatoeba-test-v2021-08-07	0.288	14.0	293	1828
+eng-nds	tatoeba-test-v2021-08-07	0.412	16.6	2500	18264
+eng-nld	tatoeba-test-v2021-08-07	0.663	48.3	12696	91796
+eng-yid	tatoeba-test-v2021-08-07	0.207	2.8	2483	16395
+frr-deu	tatoeba-test-v2021-08-07	0.198	3.0	278	1880
+fry-eng	tatoeba-test-v2021-08-07	0.500	32.5	220	1573
+fry-nld	tatoeba-test-v2021-08-07	0.633	43.1	260	1854
+gos-deu	tatoeba-test-v2021-08-07	0.370	12.7	207	1168
+gos-eng	tatoeba-test-v2021-08-07	0.316	14.6	1154	5635
+gos-nld	tatoeba-test-v2021-08-07	0.405	15.6	1852	9903
+gsw-eng	tatoeba-test-v2021-08-07	0.312	14.0	205	990
+hrx-deu	tatoeba-test-v2021-08-07	0.484	24.7	471	2805
+hrx-eng	tatoeba-test-v2021-08-07	0.362	20.4	221	1235
+ltz-deu	tatoeba-test-v2021-08-07	0.556	37.2	347	2208
+ltz-eng	tatoeba-test-v2021-08-07	0.485	32.4	293	1840
+ltz-nld	tatoeba-test-v2021-08-07	0.534	39.3	292	1685
+nds-deu	tatoeba-test-v2021-08-07	0.572	34.5	9999	74564
+nds-eng	tatoeba-test-v2021-08-07	0.493	29.9	2500	17589
+nds-nld	tatoeba-test-v2021-08-07	0.621	42.3	1657	11490
+nld-afr	tatoeba-test-v2021-08-07	0.755	58.8	1056	6823
+nld-deu	tatoeba-test-v2021-08-07	0.686	50.4	10218	74131
+nld-eng	tatoeba-test-v2021-08-07	0.690	53.1	12696	89978
+nld-fry	tatoeba-test-v2021-08-07	0.478	25.1	260	1857
+nld-ltz	tatoeba-test-v2021-08-07	0.301	14.1	292	1658
+nld-nds	tatoeba-test-v2021-08-07	0.462	21.4	1657	11711
+swg-deu	tatoeba-test-v2021-08-07	0.335	8.6	1523	15632
+yid-deu	tatoeba-test-v2021-08-07	0.381	17.6	853	5173
+yid-eng	tatoeba-test-v2021-08-07	0.324	14.2	2483	15452

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4586f60a829f1bd8331e4f877b3ddf493c333c821afe8bac9b84847681df2c7
+oid sha256:39d123b7e766718bcf6605ad74a234d47c57867b4e7b1968bede9549faf4e282
 size 161034627