Upload 17 files

result/Classification/scores_amazon_counterfactual_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.8080806321853091,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.9163090128755365,
+                "macro_f1": 0.6680366047454656
+            },
+            "logreg": {
+                "accuracy": 0.924892703862661,
+                "macro_f1": 0.7781405155410461
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.936830835117773,
+                "macro_f1": 0.8080806321853091
+            }
+        }
+    }
+}

result/Classification/scores_amazon_review_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.5680171450057119,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.4306,
+                "macro_f1": 0.42021222867279706
+            },
+            "logreg": {
+                "accuracy": 0.579,
+                "macro_f1": 0.5741023378981216
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.5726,
+                "macro_f1": 0.5680171450057119
+            }
+        }
+    }
+}

result/Classification/scores_massive_intent_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.8255898596881264,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.794392523364486,
+                "macro_f1": 0.777118788798846
+            },
+            "logreg": {
+                "accuracy": 0.8484997540580423,
+                "macro_f1": 0.820880408759503
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.8527236045729657,
+                "macro_f1": 0.8255898596881264
+            }
+        }
+    }
+}

result/Classification/scores_massive_scenario_classification.json

@@ -0,0 +1,23 @@
+{
+    "metric_name": "macro_f1",
+    "metric_value": 0.8956410349938264,
+    "details": {
+        "optimal_classifier_name": "logreg",
+        "val_scores": {
+            "knn_cosine_k_2": {
+                "accuracy": 0.868175110673881,
+                "macro_f1": 0.8608390250049474
+            },
+            "logreg": {
+                "accuracy": 0.8898180029513035,
+                "macro_f1": 0.885017174493131
+            }
+        },
+        "test_scores": {
+            "logreg": {
+                "accuracy": 0.8971082716879624,
+                "macro_f1": 0.8956410349938264
+            }
+        }
+    }
+}

result/Clustering/scores_livedoor_news.json

@@ -0,0 +1,36 @@
+{
+    "metric_name": "v_measure_score",
+    "metric_value": 0.5139491572866559,
+    "details": {
+        "optimal_clustering_model_name": "BisectingKMeans",
+        "val_scores": {
+            "MiniBatchKMeans": {
+                "v_measure_score": 0.534380650758092,
+                "homogeneity_score": 0.5317945677498351,
+                "completeness_score": 0.5369920085891019
+            },
+            "AgglomerativeClustering": {
+                "v_measure_score": 0.5087884029896779,
+                "homogeneity_score": 0.5086363161581664,
+                "completeness_score": 0.5089405807990526
+            },
+            "BisectingKMeans": {
+                "v_measure_score": 0.553601060540702,
+                "homogeneity_score": 0.5441430236373349,
+                "completeness_score": 0.56339370366749
+            },
+            "Birch": {
+                "v_measure_score": 0.5128405854529453,
+                "homogeneity_score": 0.5036525351610802,
+                "completeness_score": 0.5223700971469907
+            }
+        },
+        "test_scores": {
+            "BisectingKMeans": {
+                "v_measure_score": 0.5139491572866559,
+                "homogeneity_score": 0.5138923967226684,
+                "completeness_score": 0.5140059303906908
+            }
+        }
+    }
+}

result/Clustering/scores_mewsc16.json

@@ -0,0 +1,36 @@
+{
+    "metric_name": "v_measure_score",
+    "metric_value": 0.5225025331595674,
+    "details": {
+        "optimal_clustering_model_name": "MiniBatchKMeans",
+        "val_scores": {
+            "MiniBatchKMeans": {
+                "v_measure_score": 0.552492789911379,
+                "homogeneity_score": 0.6042939880544443,
+                "completeness_score": 0.5088713930700391
+            },
+            "AgglomerativeClustering": {
+                "v_measure_score": 0.5320671961178625,
+                "homogeneity_score": 0.5784321195121053,
+                "completeness_score": 0.4925835668838936
+            },
+            "BisectingKMeans": {
+                "v_measure_score": 0.5213239117720363,
+                "homogeneity_score": 0.5707139382787456,
+                "completeness_score": 0.47980151668476945
+            },
+            "Birch": {
+                "v_measure_score": 0.5365993974126925,
+                "homogeneity_score": 0.5836494458740594,
+                "completeness_score": 0.4965691962164629
+            }
+        },
+        "test_scores": {
+            "MiniBatchKMeans": {
+                "v_measure_score": 0.5225025331595674,
+                "homogeneity_score": 0.5674992006449608,
+                "completeness_score": 0.48411717585646613
+            }
+        }
+    }
+}

result/PairClassification/scores_paws_x_ja.json

@@ -0,0 +1,41 @@
+{
+    "metric_name": "binary_f1",
+    "metric_value": 0.6228813559322034,
+    "details": {
+        "optimal_distance_metric": "manhatten_distances",
+        "val_scores": {
+            "cosine_distances": {
+                "accuracy": 0.5725,
+                "accuracy_threshold": 0.6882385611534119,
+                "binary_f1": 0.5979670522257273,
+                "binary_f1_threshold": 1.0
+            },
+            "manhatten_distances": {
+                "accuracy": 0.605,
+                "accuracy_threshold": 28.748050689697266,
+                "binary_f1": 0.6024691358024691,
+                "binary_f1_threshold": 441.396484375
+            },
+            "euclidean_distances": {
+                "accuracy": 0.6055,
+                "accuracy_threshold": 1.1335991621017456,
+                "binary_f1": 0.6024691358024691,
+                "binary_f1_threshold": 17.27420425415039
+            },
+            "dot_similarities": {
+                "accuracy": 0.575,
+                "accuracy_threshold": 592.8396606445312,
+                "binary_f1": 0.6016949152542372,
+                "binary_f1_threshold": 467.1964111328125
+            }
+        },
+        "test_scores": {
+            "manhatten_distances": {
+                "accuracy": 0.566,
+                "accuracy_threshold": 28.748050689697266,
+                "binary_f1": 0.6228813559322034,
+                "binary_f1_threshold": 441.396484375
+            }
+        }
+    }
+}

result/Reranking/scores_esci.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.9298524733536755,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "ndcg@10": 0.9444370520979605,
+                "ndcg@20": 0.9564705129231158,
+                "ndcg@40": 0.9642100034266439
+            },
+            "dot_score": {
+                "ndcg@10": 0.9397851998624545,
+                "ndcg@20": 0.953415991071425,
+                "ndcg@40": 0.9616766756410274
+            },
+            "euclidean_distance": {
+                "ndcg@10": 0.9440129544685741,
+                "ndcg@20": 0.9562443399797927,
+                "ndcg@40": 0.9641731260066199
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "ndcg@10": 0.9298524733536755,
+                "ndcg@20": 0.9466833513010471,
+                "ndcg@40": 0.9562527070963045
+            }
+        }
+    }
+}

result/Retrieval/scores_jagovfaqs_22k.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.7667506664925435,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.6267914594910793,
+                "accuracy@3": 0.8028663351857268,
+                "accuracy@5": 0.8520035097981866,
+                "accuracy@10": 0.896753436677391,
+                "ndcg@10": 0.7669189607189386,
+                "mrr@10": 0.7247629957706004
+            },
+            "dot_score": {
+                "accuracy@1": 0.5141854343375256,
+                "accuracy@3": 0.7002047382275519,
+                "accuracy@5": 0.7627961392219947,
+                "accuracy@10": 0.8329921029540801,
+                "ndcg@10": 0.6726648184545547,
+                "mrr@10": 0.6214657121501219
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.6329336063176367,
+                "accuracy@3": 0.805791167007897,
+                "accuracy@5": 0.8546358584381398,
+                "accuracy@10": 0.8964609534951741,
+                "ndcg@10": 0.7698508124664031,
+                "mrr@10": 0.7287346156167448
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.631578947368421,
+                "accuracy@3": 0.8023391812865497,
+                "accuracy@5": 0.8467836257309942,
+                "accuracy@10": 0.8923976608187134,
+                "ndcg@10": 0.7667506664925435,
+                "mrr@10": 0.7259732664995826
+            }
+        }
+    }
+}

result/Retrieval/scores_jaqket.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.6173871224245404,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.42512562814070354,
+                "accuracy@3": 0.6412060301507537,
+                "accuracy@5": 0.7266331658291457,
+                "accuracy@10": 0.7879396984924623,
+                "ndcg@10": 0.6071501889023596,
+                "mrr@10": 0.548947914174044
+            },
+            "dot_score": {
+                "accuracy@1": 0.4,
+                "accuracy@3": 0.6180904522613065,
+                "accuracy@5": 0.6814070351758794,
+                "accuracy@10": 0.7587939698492462,
+                "ndcg@10": 0.5801732695748337,
+                "mrr@10": 0.5229046023769638
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.421105527638191,
+                "accuracy@3": 0.6311557788944724,
+                "accuracy@5": 0.7045226130653266,
+                "accuracy@10": 0.7798994974874371,
+                "ndcg@10": 0.5993385752717826,
+                "mrr@10": 0.541523889287708
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.436308926780341,
+                "accuracy@3": 0.6619859578736209,
+                "accuracy@5": 0.7211634904714143,
+                "accuracy@10": 0.7963891675025075,
+                "ndcg@10": 0.6173871224245404,
+                "mrr@10": 0.5600018308894937
+            }
+        }
+    }
+}

result/Retrieval/scores_mrtydi.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.3803302462897418,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.25323275862068967,
+                "accuracy@3": 0.4073275862068966,
+                "accuracy@5": 0.47737068965517243,
+                "accuracy@10": 0.5775862068965517,
+                "ndcg@10": 0.4050275055597,
+                "mrr@10": 0.35105406746031714
+            },
+            "dot_score": {
+                "accuracy@1": 0.18426724137931033,
+                "accuracy@3": 0.3308189655172414,
+                "accuracy@5": 0.39870689655172414,
+                "accuracy@10": 0.5021551724137931,
+                "ndcg@10": 0.3312247600613045,
+                "mrr@10": 0.2781006260262726
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.2521551724137931,
+                "accuracy@3": 0.41163793103448276,
+                "accuracy@5": 0.4827586206896552,
+                "accuracy@10": 0.5808189655172413,
+                "ndcg@10": 0.40565667279158407,
+                "mrr@10": 0.35095657156540727
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.24722222222222223,
+                "accuracy@3": 0.41944444444444445,
+                "accuracy@5": 0.5027777777777778,
+                "accuracy@10": 0.6166666666666667,
+                "ndcg@10": 0.3803302462897418,
+                "mrr@10": 0.3568121693121691
+            }
+        }
+    }
+}

result/Retrieval/scores_nlp_journal_abs_intro.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.8712459719069233,
+    "details": {
+        "optimal_distance_metric": "euclidean_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.87,
+                "accuracy@3": 0.93,
+                "accuracy@5": 0.95,
+                "accuracy@10": 0.95,
+                "ndcg@10": 0.9160310788673668,
+                "mrr@10": 0.9045000000000001
+            },
+            "dot_score": {
+                "accuracy@1": 0.79,
+                "accuracy@3": 0.89,
+                "accuracy@5": 0.93,
+                "accuracy@10": 0.94,
+                "ndcg@10": 0.870417853868121,
+                "mrr@10": 0.8474166666666666
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.87,
+                "accuracy@3": 0.94,
+                "accuracy@5": 0.95,
+                "accuracy@10": 0.95,
+                "ndcg@10": 0.916724313286633,
+                "mrr@10": 0.9053333333333333
+            }
+        },
+        "test_scores": {
+            "euclidean_distance": {
+                "accuracy@1": 0.7920792079207921,
+                "accuracy@3": 0.8960396039603961,
+                "accuracy@5": 0.9207920792079208,
+                "accuracy@10": 0.943069306930693,
+                "ndcg@10": 0.8712459719069233,
+                "mrr@10": 0.8478096023888101
+            }
+        }
+    }
+}

result/Retrieval/scores_nlp_journal_title_abs.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.9657898747088243,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.9,
+                "accuracy@3": 0.97,
+                "accuracy@5": 0.99,
+                "accuracy@10": 1.0,
+                "ndcg@10": 0.9532838532027325,
+                "mrr@10": 0.9378333333333333
+            },
+            "dot_score": {
+                "accuracy@1": 0.85,
+                "accuracy@3": 0.96,
+                "accuracy@5": 0.97,
+                "accuracy@10": 0.98,
+                "ndcg@10": 0.922904980229142,
+                "mrr@10": 0.9036666666666666
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.89,
+                "accuracy@3": 0.98,
+                "accuracy@5": 0.99,
+                "accuracy@10": 1.0,
+                "ndcg@10": 0.9515956826934274,
+                "mrr@10": 0.9353333333333333
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.9207920792079208,
+                "accuracy@3": 0.9826732673267327,
+                "accuracy@5": 0.995049504950495,
+                "accuracy@10": 1.0,
+                "ndcg@10": 0.9657898747088243,
+                "mrr@10": 0.9542491749174917
+            }
+        }
+    }
+}

result/Retrieval/scores_nlp_journal_title_intro.json

@@ -0,0 +1,43 @@
+{
+    "metric_name": "ndcg@10",
+    "metric_value": 0.779665053945222,
+    "details": {
+        "optimal_distance_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.69,
+                "accuracy@3": 0.85,
+                "accuracy@5": 0.88,
+                "accuracy@10": 0.92,
+                "ndcg@10": 0.8109556473323615,
+                "mrr@10": 0.7755
+            },
+            "dot_score": {
+                "accuracy@1": 0.64,
+                "accuracy@3": 0.76,
+                "accuracy@5": 0.8,
+                "accuracy@10": 0.87,
+                "ndcg@10": 0.7514689640047522,
+                "mrr@10": 0.7141111111111111
+            },
+            "euclidean_distance": {
+                "accuracy@1": 0.7,
+                "accuracy@3": 0.86,
+                "accuracy@5": 0.89,
+                "accuracy@10": 0.9,
+                "ndcg@10": 0.8074203858231017,
+                "mrr@10": 0.7765833333333332
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "accuracy@1": 0.6410891089108911,
+                "accuracy@3": 0.8044554455445545,
+                "accuracy@5": 0.8564356435643564,
+                "accuracy@10": 0.9207920792079208,
+                "ndcg@10": 0.779665053945222,
+                "mrr@10": 0.7346092644978786
+            }
+        }
+    }
+}

result/STS/scores_jsick.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "spearman",
+    "metric_value": 0.8199959693684533,
+    "details": {
+        "optimal_similarity_metric": "cosine_similarity",
+        "val_scores": {
+            "cosine_similarity": {
+                "pearson": 0.8396915234568699,
+                "spearman": 0.8270305329525446
+            },
+            "manhatten_distance": {
+                "pearson": 0.842469742131321,
+                "spearman": 0.8254681121166032
+            },
+            "euclidean_distance": {
+                "pearson": 0.842469742131321,
+                "spearman": 0.8254681121166032
+            },
+            "dot_score": {
+                "pearson": 0.8210183246366123,
+                "spearman": 0.7992374318058588
+            }
+        },
+        "test_scores": {
+            "cosine_similarity": {
+                "pearson": 0.8328957443401386,
+                "spearman": 0.8199959693684533
+            }
+        }
+    }
+}

result/STS/scores_jsts.json

@@ -0,0 +1,31 @@
+{
+    "metric_name": "spearman",
+    "metric_value": 0.8426164139167538,
+    "details": {
+        "optimal_similarity_metric": "manhatten_distance",
+        "val_scores": {
+            "cosine_similarity": {
+                "pearson": 0.8561010991360143,
+                "spearman": 0.816640933451772
+            },
+            "manhatten_distance": {
+                "pearson": 0.8538233899023175,
+                "spearman": 0.8173500466699406
+            },
+            "euclidean_distance": {
+                "pearson": 0.8538233899023175,
+                "spearman": 0.8173500466699406
+            },
+            "dot_score": {
+                "pearson": 0.8304879187538848,
+                "spearman": 0.7769676321807759
+            }
+        },
+        "test_scores": {
+            "manhatten_distance": {
+                "pearson": 0.8763180052177622,
+                "spearman": 0.8426164139167538
+            }
+        }
+    }
+}

result/summary.json

@@ -0,0 +1,62 @@
+{
+    "Classification": {
+        "amazon_counterfactual_classification": {
+            "macro_f1": 0.8080806321853091
+        },
+        "amazon_review_classification": {
+            "macro_f1": 0.5680171450057119
+        },
+        "massive_intent_classification": {
+            "macro_f1": 0.8255898596881264
+        },
+        "massive_scenario_classification": {
+            "macro_f1": 0.8956410349938264
+        }
+    },
+    "Reranking": {
+        "esci": {
+            "ndcg@10": 0.9298524733536755
+        }
+    },
+    "Retrieval": {
+        "jagovfaqs_22k": {
+            "ndcg@10": 0.7667506664925435
+        },
+        "jaqket": {
+            "ndcg@10": 0.6173871224245404
+        },
+        "mrtydi": {
+            "ndcg@10": 0.3803302462897418
+        },
+        "nlp_journal_abs_intro": {
+            "ndcg@10": 0.8712459719069233
+        },
+        "nlp_journal_title_abs": {
+            "ndcg@10": 0.9657898747088243
+        },
+        "nlp_journal_title_intro": {
+            "ndcg@10": 0.779665053945222
+        }
+    },
+    "STS": {
+        "jsick": {
+            "spearman": 0.8199959693684533
+        },
+        "jsts": {
+            "spearman": 0.8426164139167538
+        }
+    },
+    "Clustering": {
+        "livedoor_news": {
+            "v_measure_score": 0.5139491572866559
+        },
+        "mewsc16": {
+            "v_measure_score": 0.5225025331595674
+        }
+    },
+    "PairClassification": {
+        "paws_x_ja": {
+            "binary_f1": 0.6228813559322034
+        }
+    }
+}