Spaces:
Runtime error
Runtime error
File size: 5,496 Bytes
5d28865 728a44a 5d28865 728a44a 5d28865 728a44a 5d28865 728a44a 5d28865 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 |
import json
import os
from datetime import datetime, timezone
import gradio as gr
import numpy as np
import pandas as pd
from apscheduler.schedulers.background import BackgroundScheduler
from huggingface_hub import HfApi
from src.assets.text_content import *
from src.elo_leaderboard.load_results import get_elo_plots, get_elo_results_dicts
from src.assets.css_html_js import custom_css, get_window_url_params # left in case you need them
from src.utils_display import EloEvalColumn, fields, styled_error, styled_warning, styled_message
from src.init import load_all_info_from_hub
# clone / pull the lmeh eval data
H4_TOKEN = os.environ.get("H4_TOKEN", None)
HUMAN_EVAL_REPO = "HuggingFaceH4/scale-human-eval"
GPT_4_EVAL_REPO = "HuggingFaceH4/open_llm_leaderboard_oai_evals"
IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
ADD_PLOTS = False
EVAL_REQUESTS_PATH = "auto_evals/eval_requests"
api = HfApi()
def restart_space():
api.restart_space(
repo_id="HuggingFaceH4/open_llm_leaderboard", token=H4_TOKEN
)
human_eval_repo, gpt_4_eval_repo = load_all_info_from_hub(HUMAN_EVAL_REPO, GPT_4_EVAL_REPO)
ELO_COLS = [c.name for c in fields(EloEvalColumn)]
ELO_TYPES = [c.type for c in fields(EloEvalColumn)]
ELO_SORT_COL = EloEvalColumn.gpt4.name
def has_no_nan_values(df, columns):
return df[columns].notna().all(axis=1)
def has_nan_values(df, columns):
return df[columns].isna().any(axis=1)
def get_elo_leaderboard(df_instruct, df_code_instruct, tie_allowed=False):
if human_eval_repo:
print("Pulling human_eval_repo changes")
human_eval_repo.git_pull()
all_data = get_elo_results_dicts(df_instruct, df_code_instruct, tie_allowed)
dataframe = pd.DataFrame.from_records(all_data)
dataframe = dataframe.sort_values(by=ELO_SORT_COL, ascending=False)
dataframe = dataframe[ELO_COLS]
return dataframe
def get_elo_elements():
df_instruct = pd.read_json("human_evals/without_code.json")
df_code_instruct = pd.read_json("human_evals/with_code.json")
elo_leaderboard = get_elo_leaderboard(
df_instruct, df_code_instruct, tie_allowed=False
)
elo_leaderboard_with_tie_allowed = get_elo_leaderboard(
df_instruct, df_code_instruct, tie_allowed=True
)
plot_1, plot_2, plot_3, plot_4 = get_elo_plots(
df_instruct, df_code_instruct, tie_allowed=False
)
return (
elo_leaderboard,
elo_leaderboard_with_tie_allowed,
plot_1,
plot_2,
plot_3,
plot_4,
)
(
elo_leaderboard,
elo_leaderboard_with_tie_allowed,
plot_1,
plot_2,
plot_3,
plot_4,
) = get_elo_elements()
demo = gr.Blocks(css=custom_css)
with demo:
gr.HTML(TITLE)
with gr.Row():
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
with gr.Column():
with gr.Row():
with gr.Column(scale=2):
gr.Markdown(HUMAN_GPT_EVAL_TEXT, elem_classes="markdown-text")
with gr.Column(scale=1):
gr.Image(
"src/assets/scale-hf-logo.png", elem_id="scale-logo", show_label=False
)
gr.Markdown("## No tie allowed")
elo_leaderboard_table = gr.components.Dataframe(
value=elo_leaderboard,
headers=ELO_COLS,
datatype=ELO_TYPES,
max_rows=5,
)
gr.Markdown("## Tie allowed*")
elo_leaderboard_table_with_tie_allowed = gr.components.Dataframe(
value=elo_leaderboard_with_tie_allowed,
headers=ELO_COLS,
datatype=ELO_TYPES,
max_rows=5,
)
gr.Markdown(
"\* Results when the scores of 4 and 5 were treated as ties.",
elem_classes="markdown-text",
)
gr.Markdown(
"Let us know in [this discussion](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/65) which models we should add!",
elem_id="models-to-add-text",
)
if ADD_PLOTS:
with gr.Box():
visualization_title = gr.HTML(VISUALIZATION_TITLE)
with gr.Row():
with gr.Column():
gr.Markdown(f"#### Figure 1: {PLOT_1_TITLE}")
plot_1 = gr.Plot(plot_1, show_label=False)
with gr.Column():
gr.Markdown(f"#### Figure 2: {PLOT_2_TITLE}")
plot_2 = gr.Plot(plot_2, show_label=False)
with gr.Row():
with gr.Column():
gr.Markdown(f"#### Figure 3: {PLOT_3_TITLE}")
plot_3 = gr.Plot(plot_3, show_label=False)
with gr.Column():
gr.Markdown(f"#### Figure 4: {PLOT_4_TITLE}")
plot_4 = gr.Plot(plot_4, show_label=False)
with gr.Row():
with gr.Column():
with gr.Accordion("📙 Citation", open=False):
citation_button = gr.Textbox(
value=CITATION_BUTTON_TEXT,
label=CITATION_BUTTON_LABEL,
elem_id="citation-button",
).style(show_copy_button=True)
with gr.Column():
with gr.Accordion("✨ CHANGELOG", open=False):
changelog = gr.Markdown(CHANGELOG_TEXT, elem_id="changelog-text")
scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=3600)
scheduler.start()
demo.queue(concurrency_count=40).launch()
|