disable psc_g; rag avg

src/about.py

@@ -28,7 +28,7 @@ class Tasks(Enum):
     task12 = Task("polish_ppc_multiple_choice", "acc,none", "ppc_mc", "multiple_choice", 0.419)
     task13 = Task("polish_ppc_regex", "exact_match,score-first", "ppc_g", "generate_until", 0.419)
     task14 = Task("polish_psc_multiple_choice", "f1,none", "psc_mc", "multiple_choice", 0.466)
-    task15 = Task("polish_psc_regex", "f1,score-first", "psc_g", "generate_until", 0.466)
+    # task15 = Task("polish_psc_regex", "f1,score-first", "psc_g", "generate_until", 0.466)  # disabled until recalculation
     task16 = Task("polish_cbd_multiple_choice", "f1,none", "cbd_mc", "multiple_choice", 0.149)
     task17 = Task("polish_cbd_regex", "f1,score-first", "cbd_g", "generate_until", 0.149)
     task18 = Task("polish_klej_ner_multiple_choice", "acc,none", "klej_ner_mc", "multiple_choice", 0.343)

src/display/utils.py

@@ -34,9 +34,12 @@ auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average
 auto_eval_column_dict.append(["average_old", ColumnContent, ColumnContent("Average old", "number", False)])
 auto_eval_column_dict.append(["average_g", ColumnContent, ColumnContent("Avg g", "number", True)])
 auto_eval_column_dict.append(["average_mc", ColumnContent, ColumnContent("Avg mc", "number", True)])
+
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
+auto_eval_column_dict.append(["average_rag", ColumnContent, ColumnContent("Avg RAG", "number", True)])
+
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])

src/leaderboard/read_evals.py

@@ -166,6 +166,7 @@ class EvalResult:
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         g_tasks = [task.value.benchmark for task in Tasks if task.value.type == "generate_until"]
         mc_tasks = [task.value.benchmark for task in Tasks if task.value.type == "multiple_choice"]
+        rag_tasks = ['polish_polqa_reranking_multiple_choice', 'polish_polqa_open_book']
         all_tasks = g_tasks + mc_tasks
         all_tasks_wo_polqa = [task for task in all_tasks if 'polqa' not in task]
 
@@ -188,6 +189,7 @@ class EvalResult:
         average = sum([(self.results.get(task,0) - baselines.get(task, 0)) / (100 - baselines.get(task, 0)) * 100 for task in all_tasks]) / len(all_tasks)
         average_g = sum([(self.results.get(task,0) - baselines.get(task, 0)) / (100 - baselines.get(task, 0)) * 100 for task in g_tasks]) / len(g_tasks)
         average_mc = sum([(self.results.get(task,0) - baselines.get(task, 0)) / (100 - baselines.get(task, 0)) * 100 for task in mc_tasks]) / len(mc_tasks)
+        average_rag = sum([(self.results.get(task,0) - baselines.get(task, 0)) / (100 - baselines.get(task, 0)) * 100 for task in rag_tasks]) / len(rag_tasks)
 
         data_dict = {}
         # data_dict = {
@@ -280,6 +282,11 @@ class EvalResult:
         except KeyError:
             print(f"Could not find average_mc")
 
+        try:
+            data_dict[AutoEvalColumn.average_rag.name] = average_rag
+        except KeyError:
+            print(f"Could not find average_rag")
+
         try:
             data_dict[AutoEvalColumn.license.name] = self.license
         except KeyError: