model update

README.md

@@ -14,7 +14,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.8825793650793651
+      value: 0.9067460317460317
   - task:
       name: Analogy Questions (SAT full)
       type: multiple-choice-qa
@@ -25,7 +25,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.5962566844919787
+      value: 0.6203208556149733
   - task:
       name: Analogy Questions (SAT)
       type: multiple-choice-qa
@@ -36,7 +36,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6083086053412463
+      value: 0.6261127596439169
   - task:
       name: Analogy Questions (BATS)
       type: multiple-choice-qa
@@ -47,7 +47,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.783212896053363
+      value: 0.7787659811006115
   - task:
       name: Analogy Questions (Google)
       type: multiple-choice-qa
@@ -58,7 +58,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.918
+      value: 0.934
   - task:
       name: Analogy Questions (U2)
       type: multiple-choice-qa
@@ -69,7 +69,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6271929824561403
+      value: 0.6403508771929824
   - task:
       name: Analogy Questions (U4)
       type: multiple-choice-qa
@@ -80,7 +80,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6180555555555556
+      value: 0.6157407407407407
   - task:
       name: Analogy Questions (ConceptNet Analogy)
       type: multiple-choice-qa
@@ -91,7 +91,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.3640939597315436
+      value: 0.3674496644295302
   - task:
       name: Analogy Questions (TREX Analogy)
       type: multiple-choice-qa
@@ -102,7 +102,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.5846994535519126
+      value: 0.5081967213114754
   - task:
       name: Lexical Relation Classification (BLESS)
       type: classification
@@ -113,10 +113,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9225553714027421
+      value: 0.9213500075335241
     - name: F1 (macro)
       type: f1_macro
-      value: 0.9197291498009345
+      value: 0.9170729678018987
   - task:
       name: Lexical Relation Classification (CogALexV)
       type: classification
@@ -127,10 +127,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.8448356807511737
+      value: 0.8551643192488263
     - name: F1 (macro)
       type: f1_macro
-      value: 0.6748981211013405
+      value: 0.6925965667442509
   - task:
       name: Lexical Relation Classification (EVALution)
       type: classification
@@ -141,10 +141,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.6847237269772481
+      value: 0.6966413867822319
     - name: F1 (macro)
       type: f1_macro
-      value: 0.6766012034119605
+      value: 0.6791249033004563
   - task:
       name: Lexical Relation Classification (K&H+N)
       type: classification
@@ -155,10 +155,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9536760102942199
+      value: 0.9565973429783682
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8593518348956461
+      value: 0.8699123751243014
   - task:
       name: Lexical Relation Classification (ROOT09)
       type: classification
@@ -169,10 +169,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9072391099968662
+      value: 0.8962707615167659
     - name: F1 (macro)
       type: f1_macro
-      value: 0.9056105194658931
+      value: 0.8935557085918555
 
 ---
 # relbert/relbert-roberta-large-nce-e-semeval2012
@@ -180,22 +180,22 @@ model-index:
 RelBERT based on [roberta-large](https://huggingface.co/roberta-large) fine-tuned on [relbert/semeval2012_relational_similarity](https://huggingface.co/datasets/relbert/semeval2012_relational_similarity) (see the [`relbert`](https://github.com/asahi417/relbert) for more detail of fine-tuning).
 This model achieves the following results on the relation understanding tasks:
 - Analogy Question ([dataset](https://huggingface.co/datasets/relbert/analogy_questions), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-e-semeval2012/raw/main/analogy.forward.json)):
-    - Accuracy on SAT (full): 0.5962566844919787 
-    - Accuracy on SAT: 0.6083086053412463
-    - Accuracy on BATS: 0.783212896053363
-    - Accuracy on U2: 0.6271929824561403
-    - Accuracy on U4: 0.6180555555555556
-    - Accuracy on Google: 0.918
-    - Accuracy on ConceptNet Analogy: 0.3640939597315436
-    - Accuracy on T-Rex Analogy: 0.5846994535519126
+    - Accuracy on SAT (full): 0.6203208556149733 
+    - Accuracy on SAT: 0.6261127596439169
+    - Accuracy on BATS: 0.7787659811006115
+    - Accuracy on U2: 0.6403508771929824
+    - Accuracy on U4: 0.6157407407407407
+    - Accuracy on Google: 0.934
+    - Accuracy on ConceptNet Analogy: 0.3674496644295302
+    - Accuracy on T-Rex Analogy: 0.5081967213114754
 - Lexical Relation Classification ([dataset](https://huggingface.co/datasets/relbert/lexical_relation_classification), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-e-semeval2012/raw/main/classification.json)):
-    - Micro F1 score on BLESS: 0.9225553714027421
-    - Micro F1 score on CogALexV: 0.8448356807511737
-    - Micro F1 score on EVALution: 0.6847237269772481
-    - Micro F1 score on K&H+N: 0.9536760102942199
-    - Micro F1 score on ROOT09: 0.9072391099968662
+    - Micro F1 score on BLESS: 0.9213500075335241
+    - Micro F1 score on CogALexV: 0.8551643192488263
+    - Micro F1 score on EVALution: 0.6966413867822319
+    - Micro F1 score on K&H+N: 0.9565973429783682
+    - Micro F1 score on ROOT09: 0.8962707615167659
 - Relation Mapping ([dataset](https://huggingface.co/datasets/relbert/relation_mapping), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-e-semeval2012/raw/main/relation_mapping.json)):
-    - Accuracy on Relation Mapping: 0.8825793650793651 
+    - Accuracy on Relation Mapping: 0.9067460317460317 
 
 
 ### Usage

analogy.bidirection.json

@@ -1 +1 @@
-{"sat_full/test": 0.6310160427807486, "sat/test": 0.6409495548961425, "u2/test": 0.6798245614035088, "u4/test": 0.6643518518518519, "google/test": 0.934, "bats/test": 0.8004446914952752, "t_rex_relational_similarity/test": 0.6338797814207651, "conceptnet_relational_similarity/test": 0.40687919463087246, "sat/validation": 0.5405405405405406, "u2/validation": 0.5833333333333334, "u4/validation": 0.5833333333333334, "google/validation": 1.0, "bats/validation": 0.8542713567839196, "semeval2012_relational_similarity/validation": 0.7468354430379747, "t_rex_relational_similarity/validation": 0.2782258064516129, "conceptnet_relational_similarity/validation": 0.34532374100719426}
+{"sat_full/test": 0.660427807486631, "sat/test": 0.6646884272997032, "u2/test": 0.6885964912280702, "u4/test": 0.6620370370370371, "google/test": 0.954, "bats/test": 0.8104502501389661, "t_rex_relational_similarity/test": 0.6010928961748634, "conceptnet_relational_similarity/test": 0.4312080536912752, "sat/validation": 0.6216216216216216, "u2/validation": 0.6666666666666666, "u4/validation": 0.625, "google/validation": 0.96, "bats/validation": 0.8592964824120602, "semeval2012_relational_similarity/validation": 0.6962025316455697, "t_rex_relational_similarity/validation": 0.26814516129032256, "conceptnet_relational_similarity/validation": 0.36241007194244607}

analogy.forward.json

@@ -1 +1 @@
-{"semeval2012_relational_similarity/validation": 0.7468354430379747, "sat_full/test": 0.5962566844919787, "sat/test": 0.6083086053412463, "u2/test": 0.6271929824561403, "u4/test": 0.6180555555555556, "google/test": 0.918, "bats/test": 0.783212896053363, "t_rex_relational_similarity/test": 0.5846994535519126, "conceptnet_relational_similarity/test": 0.3640939597315436, "sat/validation": 0.4864864864864865, "u2/validation": 0.7083333333333334, "u4/validation": 0.6458333333333334, "google/validation": 1.0, "bats/validation": 0.8492462311557789, "t_rex_relational_similarity/validation": 0.2540322580645161, "conceptnet_relational_similarity/validation": 0.32194244604316546}
+{"semeval2012_relational_similarity/validation": 0.7341772151898734, "sat_full/test": 0.6203208556149733, "sat/test": 0.6261127596439169, "u2/test": 0.6403508771929824, "u4/test": 0.6157407407407407, "google/test": 0.934, "bats/test": 0.7787659811006115, "t_rex_relational_similarity/test": 0.5081967213114754, "conceptnet_relational_similarity/test": 0.3674496644295302, "sat/validation": 0.5675675675675675, "u2/validation": 0.625, "u4/validation": 0.5833333333333334, "google/validation": 1.0, "bats/validation": 0.8241206030150754, "t_rex_relational_similarity/validation": 0.2399193548387097, "conceptnet_relational_similarity/validation": 0.3183453237410072}

analogy.reverse.json

@@ -1 +1 @@
-{"sat_full/test": 0.6310160427807486, "sat/test": 0.6468842729970327, "u2/test": 0.6359649122807017, "u4/test": 0.6643518518518519, "google/test": 0.936, "bats/test": 0.7448582545858811, "t_rex_relational_similarity/test": 0.6120218579234973, "conceptnet_relational_similarity/test": 0.3565436241610738, "sat/validation": 0.4864864864864865, "u2/validation": 0.5833333333333334, "u4/validation": 0.4583333333333333, "google/validation": 0.98, "bats/validation": 0.8090452261306532, "semeval2012_relational_similarity/validation": 0.6582278481012658, "t_rex_relational_similarity/validation": 0.2782258064516129, "conceptnet_relational_similarity/validation": 0.29856115107913667}
+{"sat_full/test": 0.6550802139037433, "sat/test": 0.6646884272997032, "u2/test": 0.6754385964912281, "u4/test": 0.6527777777777778, "google/test": 0.946, "bats/test": 0.773763201778766, "t_rex_relational_similarity/test": 0.5956284153005464, "conceptnet_relational_similarity/test": 0.4186241610738255, "sat/validation": 0.5675675675675675, "u2/validation": 0.5833333333333334, "u4/validation": 0.5833333333333334, "google/validation": 0.92, "bats/validation": 0.8341708542713567, "semeval2012_relational_similarity/validation": 0.6582278481012658, "t_rex_relational_similarity/validation": 0.2661290322580645, "conceptnet_relational_similarity/validation": 0.32014388489208634}

classification.json

@@ -1 +1 @@
-{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9225553714027422, "test/f1_macro": 0.9197291498009345, "test/f1_micro": 0.9225553714027421, "test/p_macro": 0.9150094905573273, "test/p_micro": 0.9225553714027422, "test/r_macro": 0.9255428117096391, "test/r_micro": 0.9225553714027422}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8448356807511737, "test/f1_macro": 0.6748981211013405, "test/f1_micro": 0.8448356807511737, "test/p_macro": 0.6910806285341508, "test/p_micro": 0.8448356807511737, "test/r_macro": 0.662819068684954, "test/r_micro": 0.8448356807511737}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6847237269772481, "test/f1_macro": 0.6766012034119605, "test/f1_micro": 0.6847237269772481, "test/p_macro": 0.6885422680797412, "test/p_micro": 0.6847237269772481, "test/r_macro": 0.6679318977886294, "test/r_micro": 0.6847237269772481}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9536760102942199, "test/f1_macro": 0.8593518348956461, "test/f1_micro": 0.9536760102942199, "test/p_macro": 0.8369706300928008, "test/p_micro": 0.9536760102942199, "test/r_macro": 0.8877934020457473, "test/r_micro": 0.9536760102942199}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9072391099968662, "test/f1_macro": 0.9056105194658931, "test/f1_micro": 0.9072391099968662, "test/p_macro": 0.9072040983082953, "test/p_micro": 0.9072391099968662, "test/r_macro": 0.9041836458174983, "test/r_micro": 0.9072391099968662}}
+{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9213500075335241, "test/f1_macro": 0.9170729678018987, "test/f1_micro": 0.9213500075335241, "test/p_macro": 0.9135189518365673, "test/p_micro": 0.9213500075335241, "test/r_macro": 0.9212177387658143, "test/r_micro": 0.9213500075335241}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8551643192488263, "test/f1_macro": 0.6925965667442509, "test/f1_micro": 0.8551643192488263, "test/p_macro": 0.7200507756798393, "test/p_micro": 0.8551643192488263, "test/r_macro": 0.6710169140522311, "test/r_micro": 0.8551643192488263}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6966413867822319, "test/f1_macro": 0.6791249033004563, "test/f1_micro": 0.6966413867822319, "test/p_macro": 0.6848713891450097, "test/p_micro": 0.6966413867822319, "test/r_macro": 0.6757594597070967, "test/r_micro": 0.6966413867822319}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9565973429783682, "test/f1_macro": 0.8699123751243014, "test/f1_micro": 0.9565973429783682, "test/p_macro": 0.8913173287320396, "test/p_micro": 0.9565973429783682, "test/r_macro": 0.8531003052992858, "test/r_micro": 0.9565973429783682}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8962707615167659, "test/f1_macro": 0.8935557085918555, "test/f1_micro": 0.8962707615167659, "test/p_macro": 0.8963813848040892, "test/p_micro": 0.8962707615167659, "test/r_macro": 0.8911779807661145, "test/r_micro": 0.8962707615167659}}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "relbert_output/ckpt/nce_semeval2012/template-e/epoch_9",
+  "_name_or_path": "roberta-large",
   "architectures": [
     "RobertaModel"
   ],

tokenizer_config.json

@@ -6,7 +6,7 @@
   "errors": "replace",
   "mask_token": "<mask>",
   "model_max_length": 512,
-  "name_or_path": "relbert_output/ckpt/nce_semeval2012/template-e/epoch_9",
+  "name_or_path": "roberta-large",
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "special_tokens_map_file": null,