Omartificial-Intelligence-Space's picture
update app.py
1dc1a26 verified
raw
history blame
7.29 kB
# app.py
import gradio as gr
from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
import pandas as pd
from apscheduler.schedulers.background import BackgroundScheduler
from huggingface_hub import snapshot_download
from src.about import (
CITATION_BUTTON_LABEL,
CITATION_BUTTON_TEXT,
EVALUATION_QUEUE_TEXT,
INTRODUCTION_TEXT,
LLM_BENCHMARKS_TEXT,
TITLE,
)
from src.display.css_html_js import custom_css
from src.display.utils import (
COLUMNS,
COLS,
BENCHMARK_COLS,
EVAL_COLS,
EVAL_TYPES,
ModelType,
WeightType,
Precision
)
from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
from src.populate import get_evaluation_queue_df, get_leaderboard_df
from src.submission.submit import add_new_eval
def restart_space():
API.restart_space(repo_id=REPO_ID)
### Space initialization
try:
print(EVAL_REQUESTS_PATH)
snapshot_download(
repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
)
except Exception:
restart_space()
try:
print(EVAL_RESULTS_PATH)
snapshot_download(
repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
)
except Exception:
restart_space()
LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
print("LEADERBOARD_DF Shape:", LEADERBOARD_DF.shape) # Debug
print("LEADERBOARD_DF Columns:", LEADERBOARD_DF.columns.tolist()) # Debug
(
finished_eval_queue_df,
running_eval_queue_df,
pending_eval_queue_df,
) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
demo = gr.Blocks(css=custom_css)
with demo:
gr.HTML(TITLE)
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
with gr.Tabs(elem_classes="tab-buttons") as tabs:
with gr.TabItem("πŸ… LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
if LEADERBOARD_DF.empty:
gr.Markdown("No evaluations have been performed yet. The leaderboard is currently empty.")
else:
default_selection = [col.name for col in COLUMNS if col.displayed_by_default]
print("Default Selection before ensuring 'model':", default_selection) # Debug
# Ensure "model" is included
if "model" not in default_selection:
default_selection.insert(0, "model")
print("Default Selection after ensuring 'model':", default_selection) # Debug
leaderboard = Leaderboard(
value=LEADERBOARD_DF,
datatype=[col.type for col in COLUMNS],
select_columns=SelectColumns(
default_selection=default_selection,
cant_deselect=[col.name for col in COLUMNS if col.never_hidden],
label="Select Columns to Display:",
),
search_columns=[col.name for col in COLUMNS if col.name in ["model", "license"]],
hide_columns=[col.name for col in COLUMNS if col.hidden],
filter_columns=[
ColumnFilter("model_type", type="checkboxgroup", label="Model types"),
ColumnFilter("precision", type="checkboxgroup", label="Precision"),
ColumnFilter(
"still_on_hub", type="boolean", label="Deleted/incomplete", default=True
),
],
bool_checkboxgroup_label="Hide models",
interactive=False,
)
# No need to call leaderboard.render() since it's created within the Gradio context
with gr.TabItem("πŸ“ About", elem_id="llm-benchmark-tab-table", id=2):
gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
with gr.TabItem("πŸš€ Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
with gr.Column():
with gr.Row():
gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
# Since the evaluation queues are empty, display a message
with gr.Column():
gr.Markdown("Evaluations are performed immediately upon submission. There are no pending or running evaluations.")
with gr.Row():
gr.Markdown("# βœ‰οΈβœ¨ Submit your model here!", elem_classes="markdown-text")
with gr.Row():
with gr.Column():
model_name_textbox = gr.Textbox(label="Model name")
revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
model_type = gr.Dropdown(
choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
label="Model type",
multiselect=False,
value=None,
interactive=True,
)
# Removed num_examples_input since we're using a fixed number
# num_examples_input = gr.Number(
# label="Number of Examples per Subject (e.g., 10)",
# value=10,
# precision=0
# )
with gr.Column():
precision = gr.Dropdown(
choices=[i.value for i in Precision if i != Precision.Unknown],
label="Precision",
multiselect=False,
value="float16",
interactive=True,
)
weight_type = gr.Dropdown(
choices=[i.value for i in WeightType],
label="Weights type",
multiselect=False,
value="Original",
interactive=True,
)
base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
submit_button = gr.Button("Submit Eval")
submission_result = gr.Markdown()
submit_button.click(
add_new_eval,
[
model_name_textbox,
base_model_name_textbox,
revision_name_textbox,
precision,
weight_type,
model_type,
# num_examples_input # Removed
],
submission_result,
)
with gr.Row():
with gr.Accordion("πŸ“™ Citation", open=False):
citation_button = gr.Textbox(
value=CITATION_BUTTON_TEXT,
label=CITATION_BUTTON_LABEL,
lines=20,
elem_id="citation-button",
show_copy_button=True,
)
scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=1800)
scheduler.start()
demo.queue(default_concurrency_limit=40).launch()