File size: 10,962 Bytes
e16fd64
923aff9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e16fd64
 
 
 
 
00586e5
35378f6
923aff9
 
 
 
e16fd64
923aff9
 
 
 
35378f6
 
923aff9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35378f6
 
 
 
 
 
 
 
 
923aff9
35378f6
 
 
923aff9
35378f6
 
 
923aff9
e16fd64
b345ff4
 
 
923aff9
b345ff4
 
 
923aff9
69c36b6
 
 
923aff9
69c36b6
 
 
923aff9
b345ff4
923aff9
 
 
 
 
e16fd64
35378f6
923aff9
e16fd64
 
35378f6
e16fd64
35378f6
 
 
 
923aff9
 
 
 
35378f6
923aff9
 
 
 
35378f6
 
 
 
923aff9
 
 
 
35378f6
 
923aff9
35378f6
923aff9
 
 
 
35378f6
 
e16fd64
35378f6
923aff9
 
 
 
b345ff4
 
 
 
923aff9
 
 
 
69c36b6
 
 
 
923aff9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e16fd64
35378f6
923aff9
e16fd64
 
 
 
 
923aff9
e16fd64
 
69c36b6
923aff9
 
e16fd64
 
69c36b6
e16fd64
 
69c36b6
923aff9
 
e16fd64
69c36b6
e16fd64
 
923aff9
 
 
e16fd64
923aff9
e16fd64
 
 
 
 
35378f6
923aff9
35378f6
e16fd64
 
 
35378f6
923aff9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e16fd64
923aff9
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
import gradio as gr
import os
from apscheduler.schedulers.background import BackgroundScheduler
from huggingface_hub import HfApi
from datetime import datetime, timedelta

from src.assets.text_content import TITLE, INTRODUCTION_TEXT, CLEMSCORE_TEXT, MULTIMODAL_NAME, TEXT_NAME, HF_REPO
from src.leaderboard_utils import query_search, get_github_data
from src.plot_utils import split_models, plotly_plot, get_plot_df, update_open_models, update_closed_models
from src.plot_utils import reset_show_all, reset_show_names, reset_show_legend, reset_mobile_view
from src.version_utils import get_versions_data

""" 
CONSTANTS
"""
# For restarting the gradio application every 24 Hrs
TIME = 43200  # in seconds # Reload will not work locally - requires HFToken # The app launches locally as expected - only without the reload utility
# For Leaderboard table
dataframe_height = 800  # Height of the table in pixels # Set on average considering all possible devices


"""
AUTO RESTART HF SPACE
"""
HF_TOKEN = os.environ.get("H4_TOKEN", None)
api = HfApi()

def restart_space():
    api.restart_space(repo_id=HF_REPO, token=HF_TOKEN)


"""
GITHUB UTILS
"""
github_data = get_github_data()
multimodal_leaderboard = github_data["multimodal"][0]  # Get multimodal leaderboard for its available latest version.

# Show only First 4 columns for the leaderboards
multimodal_leaderboard = multimodal_leaderboard.iloc[:, :4]
print(f"Showing the following columns for the multimodal leaderboard: {multimodal_leaderboard.columns}")


"""
VERSIONS UTILS
"""
versions_data = get_versions_data()
latest_version = versions_data['latest']  # Always show latest version in text-only benchmark
last_updated_date = versions_data['date']
version_names = list(versions_data.keys())
version_names = [v for v in version_names if v.startswith("v")]  # Remove "latest" and "date" keys

global version_df
version_df = versions_data[latest_version]
def select_version_df(name):
    return versions_data[name]

"""
MAIN APPLICATION
"""
hf_app = gr.Blocks()
with hf_app:

    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")

    with gr.Tabs(elem_classes="tab-buttons") as tabs:


        """
        #######################       SECOND TAB - MULTIMODAL LEADERBOARD     #######################
        """
        with gr.TabItem(MULTIMODAL_NAME, elem_id="mm-llm-benchmark-tab-table", id=1):
            with gr.Row():
                mm_search_bar = gr.Textbox(
                    placeholder=" πŸ” Search for models - separate multiple queries with `;` and press ENTER...",
                    show_label=False,
                    elem_id="search-bar",
                )

            mm_leaderboard_table = gr.Dataframe(
                value=multimodal_leaderboard,
                elem_id="mm-leaderboard-table",
                interactive=False,
                visible=True,
                height=dataframe_height
            )

            # Show information about the clemscore and last updated date below the table
            gr.HTML(CLEMSCORE_TEXT)
            gr.HTML(f"Last updated - {github_data['date']}")

            # Add a dummy leaderboard to handle search queries in leaderboard_table
            # This will show a temporary leaderboard based on the searched value
            mm_dummy_leaderboard_table = gr.Dataframe(
                value=multimodal_leaderboard,
                elem_id="mm-leaderboard-table-dummy",
                interactive=False,
                visible=False
            )

            # Action after submitting a query to the search bar
            mm_search_bar.submit(
                query_search,
                [mm_dummy_leaderboard_table, mm_search_bar],
                mm_leaderboard_table,
                queue=True
            )

        """
        #######################       THIRD TAB - PLOTS - %PLAYED V/S QUALITY SCORE     #######################
        """
        with gr.TabItem("πŸ“ˆ Plots", elem_id="plots", id=2):

            """
            Accordion Groups to select individual models - Hidden by default
            """
            with gr.Accordion("Select Open-weight Models 🌐", open=False):
                open_models_selection = update_open_models()
                clear_button_1 = gr.ClearButton(open_models_selection)

            with gr.Accordion("Select Commercial Models πŸ’°", open=False):
                closed_models_selection = update_closed_models()
                clear_button_2 = gr.ClearButton(closed_models_selection)

            """
            Checkbox group to control the layout of the plot 
            """
            with gr.Row():
                with gr.Column():
                    show_all = gr.CheckboxGroup(
                        ["Select All Models"],
                        label="Show plot for all models πŸ€–",
                        value=[],
                        elem_id="value-select-3",
                        interactive=True,
                    )

                with gr.Column():
                    show_names = gr.CheckboxGroup(
                        ["Show Names"],
                        label="Show names of models on the plot 🏷️",
                        value=[],
                        elem_id="value-select-4",
                        interactive=True,
                    )

                with gr.Column():
                    show_legend = gr.CheckboxGroup(
                        ["Show Legend"],
                        label="Show legend on the plot πŸ’‘",
                        value=[],
                        elem_id="value-select-5",
                        interactive=True,
                    )
                with gr.Column():
                    mobile_view = gr.CheckboxGroup(
                        ["Mobile View"],
                        label="View plot on smaller screens πŸ“±",
                        value=[],
                        elem_id="value-select-6",
                        interactive=True,
                    )

            """
            PLOT BLOCK
            """
            # Create a dummy DataFrame as an input to the plotly_plot function.
            # Uses this data to plot the %played v/s quality score
            with gr.Row():
                dummy_plot_df = gr.DataFrame(
                    value=get_plot_df(),
                    visible=False
                )

            with gr.Row():
                with gr.Column():
                    # Output block for the plot
                    plot_output = gr.Plot()

            """
            PLOT CHANGE ACTIONS
            Toggle 'Select All Models' based on the values in Accordion checkbox groups
            """
            open_models_selection.change(
                plotly_plot,
                [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend,
                 mobile_view],
                [plot_output],
                queue=True
            )

            closed_models_selection.change(
                plotly_plot,
                [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend,
                 mobile_view],
                [plot_output],
                queue=True
            )

            show_all.change(
                plotly_plot,
                [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend,
                 mobile_view],
                [plot_output],
                queue=True
            )

            show_names.change(
                plotly_plot,
                [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend,
                 mobile_view],
                [plot_output],
                queue=True
            )

            show_legend.change(
                plotly_plot,
                [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend,
                 mobile_view],
                [plot_output],
                queue=True
            )

            mobile_view.change(
                plotly_plot,
                [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend,
                 mobile_view],
                [plot_output],
                queue=True
            )


            open_models_selection.change(
                reset_show_all,
                outputs=[show_all],
                queue=True
            )

            closed_models_selection.change(
                reset_show_all,
                outputs=[show_all],
                queue=True
            )


        """
        #######################       FOURTH TAB - VERSIONS AND DETAILS     #######################
        """
        with gr.TabItem("πŸ”„ Versions and Details", elem_id="versions-details-tab", id=3):
            with gr.Row():
                version_select = gr.Dropdown(
                    version_names, label="Select Version πŸ•ΉοΈ", value=latest_version
                )
            with gr.Row():
                search_bar_prev = gr.Textbox(
                    placeholder=" πŸ” Search for models - separate multiple queries with `;` and press ENTER...",
                    show_label=False,
                    elem_id="search-bar-3",
                )

            prev_table = gr.Dataframe(
                value=version_df,
                elem_id="version-leaderboard-table",
                interactive=False,
                visible=True,
                height=dataframe_height
            )

            dummy_prev_table = gr.Dataframe(
                value=version_df,
                elem_id="version-dummy-leaderboard-table",
                interactive=False,
                visible=False
            )

            gr.HTML(CLEMSCORE_TEXT)
            gr.HTML(f"Last updated - {last_updated_date}")

            search_bar_prev.submit(
                query_search,
                [dummy_prev_table, search_bar_prev],
                prev_table,
                queue=True
            )

            version_select.change(
                select_version_df,
                [version_select],
                prev_table,
                queue=True
            )

            # Update Dummy Leaderboard, when changing versions
            version_select.change(
                select_version_df,
                [version_select],
                dummy_prev_table,
                queue=True
            )

    hf_app.load()
hf_app.queue()

# Add scheduler to auto-restart the HF space at every TIME interval and update every component each time
scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, 'interval', seconds=TIME)
scheduler.start()

# Log current start time and scheduled restart time
print(datetime.now())
print(f"Scheduled restart at {datetime.now() + timedelta(seconds=TIME)}")

hf_app.launch()