model update

README.md

@@ -7,14 +7,14 @@ metrics:
 - rouge-l
 - bertscore
 - moverscore
-language: en
+language: es
 datasets:
 - lmqg/qag_esquad
 pipeline_tag: text2text-generation
 tags:
 - questions and answers generation
 widget:
-- text: "generate question and answer: Beyonce further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, Cadillac Records."
+- text: "del Ministerio de Desarrollo Urbano , Gobierno de la India."
   example_title: "Questions & Answers Generation Example 1" 
 model-index:
 - name: lmqg/mt5-small-esquad-qag
@@ -29,37 +29,37 @@ model-index:
     metrics:
     - name: BLEU4 (Question & Answer Generation)
       type: bleu4_question_answer_generation
-      value: 1.48
+      value: 1.36
     - name: ROUGE-L (Question & Answer Generation)
       type: rouge_l_question_answer_generation
-      value: 13.35
+      value: 12.76
     - name: METEOR (Question & Answer Generation)
       type: meteor_question_answer_generation
-      value: 17.68
+      value: 16.9
     - name: BERTScore (Question & Answer Generation)
       type: bertscore_question_answer_generation
-      value: 81.17
+      value: 64.09
     - name: MoverScore (Question & Answer Generation)
       type: moverscore_question_answer_generation
-      value: 54.06
+      value: 50.15
     - name: QAAlignedF1Score-BERTScore (Question & Answer Generation)
       type: qa_aligned_f1_score_bertscore_question_answer_generation
-      value: 86.93
+      value: 78.12
     - name: QAAlignedRecall-BERTScore (Question & Answer Generation)
       type: qa_aligned_recall_bertscore_question_answer_generation
-      value: 87.22
+      value: 78.27
     - name: QAAlignedPrecision-BERTScore (Question & Answer Generation)
       type: qa_aligned_precision_bertscore_question_answer_generation
-      value: 86.67
+      value: 78.0
     - name: QAAlignedF1Score-MoverScore (Question & Answer Generation)
       type: qa_aligned_f1_score_moverscore_question_answer_generation
-      value: 63.76
+      value: 53.92
     - name: QAAlignedRecall-MoverScore (Question & Answer Generation)
       type: qa_aligned_recall_moverscore_question_answer_generation
-      value: 64.15
+      value: 53.93
     - name: QAAlignedPrecision-MoverScore (Question & Answer Generation)
       type: qa_aligned_precision_moverscore_question_answer_generation
-      value: 63.41
+      value: 53.93
 ---
 
 # Model Card of `lmqg/mt5-small-esquad-qag`
@@ -68,7 +68,7 @@ This model is fine-tuned version of [google/mt5-small](https://huggingface.co/go
 
 ### Overview
 - **Language model:** [google/mt5-small](https://huggingface.co/google/mt5-small)   
-- **Language:** en  
+- **Language:** es  
 - **Training data:** [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) (default)
 - **Online Demo:** [https://autoqg.net/](https://autoqg.net/)
 - **Repository:** [https://github.com/asahi417/lm-question-generation](https://github.com/asahi417/lm-question-generation)
@@ -80,10 +80,10 @@ This model is fine-tuned version of [google/mt5-small](https://huggingface.co/go
 from lmqg import TransformersQG
 
 # initialize model
-model = TransformersQG(language="en", model="lmqg/mt5-small-esquad-qag")
+model = TransformersQG(language="es", model="lmqg/mt5-small-esquad-qag")
 
 # model prediction
-question_answer_pairs = model.generate_qa("William Turner was an English painter who specialised in watercolour landscapes")
+question_answer_pairs = model.generate_qa("a noviembre , que es también la estación lluviosa.")
 
 ```
 
@@ -92,7 +92,7 @@ question_answer_pairs = model.generate_qa("William Turner was an English painter
 from transformers import pipeline
 
 pipe = pipeline("text2text-generation", "lmqg/mt5-small-esquad-qag")
-output = pipe("generate question and answer: Beyonce further expanded her acting career, starring as blues singer Etta James in the 2008 musical biopic, Cadillac Records.")
+output = pipe("del Ministerio de Desarrollo Urbano , Gobierno de la India.")
 
 ```
 
@@ -103,20 +103,20 @@ output = pipe("generate question and answer: Beyonce further expanded her acting
 
 |                                 |   Score | Type    | Dataset                                                            |
 |:--------------------------------|--------:|:--------|:-------------------------------------------------------------------|
-| BERTScore                       |   81.17 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| Bleu_1                          |    7.99 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| Bleu_2                          |    3.81 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| Bleu_3                          |    2.22 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| Bleu_4                          |    1.48 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| METEOR                          |   17.68 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| MoverScore                      |   54.06 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| QAAlignedF1Score (BERTScore)    |   86.93 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| QAAlignedF1Score (MoverScore)   |   63.76 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| QAAlignedPrecision (BERTScore)  |   86.67 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| QAAlignedPrecision (MoverScore) |   63.41 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| QAAlignedRecall (BERTScore)     |   87.22 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| QAAlignedRecall (MoverScore)    |   64.15 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
-| ROUGE_L                         |   13.35 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| BERTScore                       |   64.09 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| Bleu_1                          |    7.43 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| Bleu_2                          |    3.51 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| Bleu_3                          |    2.04 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| Bleu_4                          |    1.36 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| METEOR                          |   16.9  | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| MoverScore                      |   50.15 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| QAAlignedF1Score (BERTScore)    |   78.12 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| QAAlignedF1Score (MoverScore)   |   53.92 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| QAAlignedPrecision (BERTScore)  |   78    | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| QAAlignedPrecision (MoverScore) |   53.93 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| QAAlignedRecall (BERTScore)     |   78.27 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| QAAlignedRecall (MoverScore)    |   53.93 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
+| ROUGE_L                         |   12.76 | default | [lmqg/qag_esquad](https://huggingface.co/datasets/lmqg/qag_esquad) |
 
 
 
@@ -127,16 +127,16 @@ The following hyperparameters were used during fine-tuning:
  - dataset_name: default
  - input_types: ['paragraph']
  - output_types: ['questions_answers']
- - prefix_types: ['qag']
+ - prefix_types: None
  - model: google/mt5-small
  - max_length: 512
  - max_length_output: 256
- - epoch: 15
+ - epoch: 14
  - batch: 8
  - lr: 0.001
  - fp16: False
  - random_seed: 1
- - gradient_accumulation_steps: 16
+ - gradient_accumulation_steps: 8
  - label_smoothing: 0.0
 
 The full configuration can be found at [fine-tuning config file](https://huggingface.co/lmqg/mt5-small-esquad-qag/raw/main/trainer_config.json).

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "lmqg_output/mt5-small-esquad-qag/best_model",
+  "_name_or_path": "lmqg_output/mt5-small-esquad-qag/model_mzgdpa/epoch_5",
   "add_prefix": false,
   "architectures": [
     "MT5ForConditionalGeneration"

eval/metric.first.answer.paragraph.questions_answers.lmqg_qag_esquad.default.json

@@ -1 +1 @@
-{"validation": {"Bleu_1": 0.2850437982146202, "Bleu_2": 0.1610693347933042, "Bleu_3": 0.09479214712562453, "Bleu_4": 0.06316072756946382, "METEOR": 0.20937507317506926, "ROUGE_L": 0.247577012309755, "BERTScore": 0.8606134602627895, "MoverScore": 0.5926754397407197, "QAAlignedF1Score (BERTScore)": 0.8818379455108741, "QAAlignedRecall (BERTScore)": 0.8702034684856624, "QAAlignedPrecision (BERTScore)": 0.8941234975356361, "QAAlignedF1Score (MoverScore)": 0.6550485268023295, "QAAlignedRecall (MoverScore)": 0.6401481289661134, "QAAlignedPrecision (MoverScore)": 0.671663012891372}, "test": {"Bleu_1": 0.07985266597997066, "Bleu_2": 0.03813330462794156, "Bleu_3": 0.022234146301146923, "Bleu_4": 0.014768244684738977, "METEOR": 0.1768493505457922, "ROUGE_L": 0.13354751168160714, "BERTScore": 0.8116741879330348, "MoverScore": 0.540645316283108, "QAAlignedF1Score (BERTScore)": 0.8693435521882494, "QAAlignedRecall (BERTScore)": 0.872202193455801, "QAAlignedPrecision (BERTScore)": 0.8666602810589459, "QAAlignedF1Score (MoverScore)": 0.6376073535235884, "QAAlignedRecall (MoverScore)": 0.641493720145074, "QAAlignedPrecision (MoverScore)": 0.6341303484358323}}
+{"validation": {"Bleu_1": 0.2718803847471669, "Bleu_2": 0.15254535887603982, "Bleu_3": 0.0894467983934925, "Bleu_4": 0.060018622693704934, "METEOR": 0.20863168631638873, "ROUGE_L": 0.2493543685540034, "BERTScore": 0.7305193155297462, "MoverScore": 0.521490296598535, "QAAlignedF1Score (BERTScore)": 0.8023877345182617, "QAAlignedRecall (BERTScore)": 0.7787815647509312, "QAAlignedPrecision (BERTScore)": 0.8284482213316031, "QAAlignedF1Score (MoverScore)": 0.5564934635971232, "QAAlignedRecall (MoverScore)": 0.5375780961020423, "QAAlignedPrecision (MoverScore)": 0.5781583103105322}, "test": {"Bleu_1": 0.07432354130344639, "Bleu_2": 0.03507385086417368, "Bleu_3": 0.0204298541841383, "Bleu_4": 0.013646333757009334, "METEOR": 0.1689782326396343, "ROUGE_L": 0.12757471978428708, "BERTScore": 0.6408908916549078, "MoverScore": 0.5014847577134893, "QAAlignedF1Score (BERTScore)": 0.7811615456479336, "QAAlignedRecall (BERTScore)": 0.7826666697304786, "QAAlignedPrecision (BERTScore)": 0.7800460044662871, "QAAlignedF1Score (MoverScore)": 0.5391780380897195, "QAAlignedRecall (MoverScore)": 0.5393083014332328, "QAAlignedPrecision (MoverScore)": 0.5392940215876131}}

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f37ea83e20747f94e7d54865e0331671c9318f43d4e901d01d0d93a340133c58
-size 1200724741
+oid sha256:8597366c1661de134170374ac795bf81c1ff381830277756abe8793299cb1014
+size 1200727429

tokenizer_config.json

@@ -2,7 +2,7 @@
   "additional_special_tokens": null,
   "eos_token": "</s>",
   "extra_ids": 0,
-  "name_or_path": "lmqg_output/mt5-small-esquad-qag/best_model",
+  "name_or_path": "lmqg_output/mt5-small-esquad-qag/model_mzgdpa/epoch_5",
   "pad_token": "<pad>",
   "sp_model_kwargs": {},
   "special_tokens_map_file": "/home/c.c2042013/.cache/huggingface/transformers/685ac0ca8568ec593a48b61b0a3c272beee9bc194a3c7241d15dcadb5f875e53.f76030f3ec1b96a8199b2593390c610e76ca8028ef3d24680000619ffb646276",

trainer_config.json

@@ -1 +1 @@
-{"dataset_path": "lmqg/qag_esquad", "dataset_name": "default", "input_types": ["paragraph"], "output_types": ["questions_answers"], "prefix_types": ["qag"], "model": "google/mt5-small", "max_length": 512, "max_length_output": 256, "epoch": 15, "batch": 8, "lr": 0.001, "fp16": false, "random_seed": 1, "gradient_accumulation_steps": 16, "label_smoothing": 0.0}
+{"dataset_path": "lmqg/qag_esquad", "dataset_name": "default", "input_types": ["paragraph"], "output_types": ["questions_answers"], "prefix_types": null, "model": "google/mt5-small", "max_length": 512, "max_length_output": 256, "epoch": 14, "batch": 8, "lr": 0.001, "fp16": false, "random_seed": 1, "gradient_accumulation_steps": 8, "label_smoothing": 0.0}