Add average checkbox for each individual category

app.py

@@ -101,17 +101,20 @@ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
         AutoEvalColumn.model.name,
     ]
 
-    # Ensure no duplicates when never_hidden and displayed_by_default are both True
+    # Ensure no duplicates and add the new average columns
     unique_columns = set(always_here_cols + columns)
 
     # We use COLS to maintain sorting
-    filtered_df = df[
-        [c for c in COLS if c in df.columns and c in unique_columns]
-    ]
+    filtered_df = df[[c for c in COLS if c in df.columns and c in unique_columns]]
+
+    # Debugging print to see if the new columns are included
+    print(f"Columns included in DataFrame: {filtered_df.columns.tolist()}")
+
     return filtered_df
 
 
 
+
 def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
     final_df = []
     if query != "":

src/display/utils.py

@@ -27,7 +27,17 @@ auto_eval_column_dict = []
 # Model Information
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, category="Model Information", never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, category="Model Information", never_hidden=True)])
+
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True, category="Model Information")])
+auto_eval_column_dict.append(["average_IE", ColumnContent, ColumnContent("Average IE ⬆️", "number", False, category="Information Extraction (IE)")])
+auto_eval_column_dict.append(["average_TA", ColumnContent, ColumnContent("Average TA ⬆️", "number", False, category="Textual Analysis (TA)")])
+auto_eval_column_dict.append(["average_QA", ColumnContent, ColumnContent("Average QA ⬆️", "number", False, category="Question Answering (QA)")])
+auto_eval_column_dict.append(["average_TG", ColumnContent, ColumnContent("Average TG ⬆️", "number", False, category="Text Generation (TG)")])
+auto_eval_column_dict.append(["average_RM", ColumnContent, ColumnContent("Average RM ⬆️", "number", False, category="Risk Management (RM)")])
+auto_eval_column_dict.append(["average_FO", ColumnContent, ColumnContent("Average FO ⬆️", "number", False, category="Forecasting (FO)")])
+auto_eval_column_dict.append(["average_DM", ColumnContent, ColumnContent("Average DM ⬆️", "number", False, category="Decision-Making (DM)")])
+auto_eval_column_dict.append(["average_Spanish", ColumnContent, ColumnContent("Average Spanish ⬆️", "number", False, category="Spanish")])
+
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False, category="Model Information")])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False, category="Model Information")])
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, category="Model Information", hidden=True)])

src/leaderboard/read_evals.py

@@ -117,8 +117,53 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
+
+        # Initialize category averages
+        category_averages = {
+            "average_IE": [],
+            "average_TA": [],
+            "average_QA": [],
+            "average_TG": [],
+            "average_RM": [],
+            "average_FO": [],
+            "average_DM": [],
+            "average_Spanish": []
+        }
+
+        # Calculate averages for each task
+        for task in Tasks:
+            score = self.results.get(task.value.benchmark)
+            if score is not None:
+                # Append score to the appropriate category
+                if task.value.category == "Information Extraction (IE)":
+                    category_averages["average_IE"].append(score)
+                elif task.value.category == "Textual Analysis (TA)":
+                    category_averages["average_TA"].append(score)
+                elif task.value.category == "Question Answering (QA)":
+                    category_averages["average_QA"].append(score)
+                elif task.value.category == "Text Generation (TG)":
+                    category_averages["average_TG"].append(score)
+                elif task.value.category == "Risk Management (RM)":
+                    category_averages["average_RM"].append(score)
+                elif task.value.category == "Forecasting (FO)":
+                    category_averages["average_FO"].append(score)
+                elif task.value.category == "Decision-Making (DM)":
+                    category_averages["average_DM"].append(score)
+                elif task.value.category == "Spanish":
+                    category_averages["average_Spanish"].append(score)
+
+        # Calculate the mean for each category and add to data_dict
+        data_dict = {}
+        for category, scores in category_averages.items():
+            average = sum(scores) / len(scores) if scores else 0
+            data_dict[category] = average
+
+        # Overall average
+        total_scores = [v for v in self.results.values() if v is not None]
+        overall_average = sum(total_scores) / len(total_scores) if total_scores else 0
+
+        # Add other columns
+        data_dict.update({
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,
@@ -127,19 +172,23 @@ class EvalResult:
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.average.name: overall_average,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
+        })
 
+        # Add task results to the data dictionary
         for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+            data_dict[task.value.col_name] = self.results.get(task.value.benchmark)
 
         return data_dict
 
 
+
+
+
 def get_request_file_for_model(requests_path, model_name, precision):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
     request_files = os.path.join(

src/populate.py

@@ -1,6 +1,5 @@
 import json
 import os
-
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
@@ -14,8 +13,23 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
+
+    # Add category average columns with default values
+    category_avg_columns = {
+        "Average IE ⬆️": "average_IE",
+        "Average TA ⬆️": "average_TA",
+        "Average QA ⬆️": "average_QA",
+        "Average TG ⬆️": "average_TG",
+        "Average RM ⬆️": "average_RM",
+        "Average FO ⬆️": "average_FO",
+        "Average DM ⬆️": "average_DM",
+        "Average Spanish ⬆️": "average_Spanish"
+    }
+
+    for display_name, internal_name in category_avg_columns.items():
+        df[display_name] = df[internal_name]
+
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
 
     # Apply the transformation for MCC values
     mcc_tasks = ["German", "Australian", "LendingClub", "ccf", "ccfraud", "polish", "taiwan", "portoseguro", "travelinsurance"]
@@ -23,14 +37,16 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
         if task in df.columns:
             df[task] = (df[task] + 100) / 2.0
 
+    # Now, select the columns that were passed to the function
+    df = df[cols].round(decimals=2)
+
     # Filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
     return raw_data, df
 
 
-
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
+    """Creates the different dataframes for the evaluation queues requests"""
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []