Merge pull request #12 from OpenGPTX/fix/belebele_fewshot

Refactorings and fixes for tab handling and few-shot selection

app.py

@@ -12,6 +12,8 @@ with demo:
         elem_classes="markdown-text",
     )
 
+    selected_tab = gr.State(value=0)
+
     with gr.Column():
         with gr.Row():
             with gr.Column():
@@ -67,10 +69,9 @@ with demo:
                         select.click(update_bar, inputs=[], outputs=langs_bar)
 
                 with gr.Row():
-                    acc_task_group_names = core.task_groups_with_task_type("accuracy")
                     shown_tasks = gr.CheckboxGroup(
-                        choices=acc_task_group_names,
-                        value=acc_task_group_names,
+                        choices=[],
+                        value=[],
                         label="Select tasks to show",
                         elem_id="column-select",
                         interactive=True,
@@ -80,15 +81,8 @@ with demo:
                         choices=[("0-Shot", False), ("Few-shot", True)],
                         value=True,
                         label="Select evaluation type",
-                        interactive=True,
                         scale=29,
                     )
-                    demo.load(
-                        core.fix_zeroshot, [shown_tasks, fewshot], shown_tasks
-                    )
-                    fewshot.change(
-                        core.fix_zeroshot, [shown_tasks, fewshot], shown_tasks
-                    )
                     clear = gr.ClearButton(
                         shown_tasks, value="Deselect all tasks", size="sm", scale=21
                     )
@@ -104,33 +98,44 @@ with demo:
                 id=1,
             ) as misc:
                 leaderboard_table_misc = gr.Dataframe()
-            acc.select(
-                lambda x: core.update_tab_tasks(0, x),
-                inputs=fewshot,
-                outputs=[shown_tasks, fewshot],
+
+        demo.load(
+            core.update_task_groups_and_fewshot,
+            [gr.State(value=0), fewshot],
+            [shown_tasks, fewshot, selected_tab],
+        )
+        fewshot.change(
+            core.update_task_groups_and_fewshot,
+            [selected_tab, fewshot],
+            [shown_tasks, fewshot, selected_tab],
+        )
+        acc.select(
+            core.update_task_groups_and_fewshot,
+            inputs=[gr.State(value=0), fewshot],
+            outputs=[shown_tasks, fewshot, selected_tab],
+        )
+        misc.select(
+            core.update_task_groups_and_fewshot,
+            inputs=[gr.State(value=1), fewshot],
+            outputs=[shown_tasks, fewshot, selected_tab],
+        )
+        for comp, fn in [
+            (search_bar, "submit"),
+            (langs_bar, "change"),
+            (shown_tasks, "change"),
+            (fewshot, "change"),
+            (model_types, "change"),
+        ]:
+            getattr(comp, fn)(
+                core.update_df,
+                [shown_tasks, search_bar, langs_bar, model_types, fewshot],
+                leaderboard_table,
             )
-            misc.select(
-                lambda x: core.update_tab_tasks(1, x),
-                inputs=fewshot,
-                outputs=[shown_tasks, fewshot],
+            getattr(comp, fn)(
+                core.update_df,
+                [shown_tasks, search_bar, langs_bar, model_types, fewshot],
+                leaderboard_table_misc,
             )
-            for comp, fn in [
-                (search_bar, "submit"),
-                (langs_bar, "change"),
-                (shown_tasks, "change"),
-                (fewshot, "change"),
-                (model_types, "change"),
-            ]:
-                getattr(comp, fn)(
-                    core.update_df,
-                    [shown_tasks, search_bar, langs_bar, model_types, fewshot],
-                    leaderboard_table,
-                )
-                getattr(comp, fn)(
-                    core.update_df,
-                    [shown_tasks, search_bar, langs_bar, model_types, fewshot],
-                    leaderboard_table_misc,
-                )
 
 
     gr.Blocks.load(

core.py

@@ -9,9 +9,8 @@ from datasets import load_dataset
 
 import style
 
-TAB_STATE = 0  # FIXME
-NO_FEWSHOT = ["BELEBELE"]  # FIXME
-NO_ZEROSHOT = ["GSM8K", "TruthfulQA"]  # FIXME
+ZERO_SHOT_ONLY = ["BELEBELE"]
+FEW_SHOT_ONLY = ["GSM8K", "TruthfulQA"]
 
 
 def init():
@@ -126,68 +125,29 @@ def update_df(
         return sort_cols(df, fewshot)
 
 
-def fix_zeroshot(tasks: list[str | int | float], fewshot: bool = False):
-    global TAB_STATE
-    selected_task_type = get_selected_task_type(TAB_STATE)
-    choices = task_groups_with_task_type(selected_task_type)
-    if not fewshot:
-        choices = [c for c in choices if c not in NO_ZEROSHOT]
-        value = [v for v in tasks if v in choices]
-        value += [t for t in NO_FEWSHOT if t not in value]
-    else:
-        if TAB_STATE == 0:
-            choices = [c for c in choices if c not in NO_FEWSHOT]
-            value = [v for v in tasks if v in choices]
-            value += [t for t in NO_ZEROSHOT if t not in value]
-        elif TAB_STATE == 1:
-            value = [v for v in tasks if v in choices]
-    shown_tasks = gr.CheckboxGroup(
-        choices=choices,
-        value=value,
-        label="Select tasks to show",
-        elem_id="column-select",
-        interactive=True,
-        scale=50,
+def update_task_groups_and_fewshot(current_selected_tab: int, is_fewshot_current: bool = False):
+    selected_task_type = get_selected_task_type(current_selected_tab)
+    available_tasks = get_available_task_groups(selected_task_type, is_fewshot_current)
+    new_selected_tasks = available_tasks.copy()
+
+    tasks_checkbox_group_update = gr.CheckboxGroup(
+        choices=available_tasks,
+        value=new_selected_tasks,
     )
-    return shown_tasks
-
-
-def update_tab_tasks(id: int, fewshot: bool = False):
-    # when the tab is changed, update the TAB_STATE accordingly
-    global TAB_STATE
-    TAB_STATE = id
-    selected_task_type = get_selected_task_type(TAB_STATE)
-    choices = task_groups_with_task_type(selected_task_type)
-    if not fewshot:
-        choices = [c for c in choices if c not in NO_ZEROSHOT]
-    values = choices.copy()
-    shown_tasks = gr.CheckboxGroup(
-        choices=choices,
-        value=values,
-        label="Select tasks to show",
-        elem_id="column-select",
-        interactive=True,
-        scale=50,
+
+    if current_selected_tab == 0:
+        is_fewshot_new = is_fewshot_current
+        fewshot_available = True
+    elif current_selected_tab == 1:
+        is_fewshot_new = False
+        fewshot_available = False
+
+    fewshot_radio_update = gr.Radio(
+        value=is_fewshot_new,
+        interactive=fewshot_available,
     )
-    if id == 0:
-        # switching to accuracy tab, default to fewshot
-        fewshot = gr.Radio(
-            choices=[("0-Shot", False), ("Few-shot", True)],
-            value=True,
-            label="Select evaluation type",
-            interactive=True,
-            scale=29,
-        )
-    elif id == 1:
-        # switching to translation tab, default to 0-shot and disable selection
-        fewshot = gr.Radio(
-            choices=[("0-Shot", False), ("Few-shot", True)],
-            value=False,
-            label="Select evaluation type",
-            interactive=False,
-            scale=29,
-        )
-    return [shown_tasks, fewshot]
+
+    return [tasks_checkbox_group_update, fewshot_radio_update, current_selected_tab]
 
 
 def get_selected_task_type(task_type_id):
@@ -196,10 +156,15 @@ def get_selected_task_type(task_type_id):
     return selected_task_type
 
 
-def task_groups_with_task_type(selected_task_type):
-    choices = [task_group_name for task_group_name, task_type in task_group_type_dict.items() if task_type == selected_task_type]
+def get_available_task_groups(selected_task_type, fewshot):
+    task_groups = [task_group_name for task_group_name, task_type in task_group_type_dict.items() if task_type == selected_task_type]
+
+    if fewshot:
+        available_tasks = [c for c in task_groups if c not in ZERO_SHOT_ONLY]
+    else:
+        available_tasks = [c for c in task_groups if c not in FEW_SHOT_ONLY]
 
-    return choices
+    return available_tasks
 
 
 init()