|
import re |
|
import streamlit as st |
|
import requests |
|
import pandas as pd |
|
from io import StringIO |
|
import plotly.graph_objs as go |
|
from huggingface_hub import HfApi |
|
from huggingface_hub.utils import RepositoryNotFoundError, RevisionNotFoundError |
|
|
|
from yall import create_yall |
|
|
|
|
|
|
|
def convert_markdown_table_to_dataframe(md_content): |
|
""" |
|
Converts markdown table to Pandas DataFrame, handling special characters and links, |
|
extracts Hugging Face URLs, and adds them to a new column. |
|
""" |
|
|
|
cleaned_content = re.sub(r'\|\s*$', '', re.sub(r'^\|\s*', '', md_content, flags=re.MULTILINE), flags=re.MULTILINE) |
|
|
|
|
|
df = pd.read_csv(StringIO(cleaned_content), sep="\|", engine='python') |
|
|
|
|
|
df = df.drop(0, axis=0) |
|
|
|
|
|
df.columns = df.columns.str.strip() |
|
|
|
|
|
model_link_pattern = r'\[(.*?)\]\((.*?)\)\s*\[.*?\]\(.*?\)' |
|
df['URL'] = df['Model'].apply(lambda x: re.search(model_link_pattern, x).group(2) if re.search(model_link_pattern, x) else None) |
|
|
|
|
|
df['Model'] = df['Model'].apply(lambda x: re.sub(model_link_pattern, r'\1', x)) |
|
|
|
return df |
|
|
|
@st.cache_data |
|
def get_model_info(df): |
|
api = HfApi() |
|
|
|
|
|
df['Likes'] = None |
|
df['Tags'] = None |
|
|
|
|
|
for index, row in df.iterrows(): |
|
model = row['Model'].strip() |
|
try: |
|
model_info = api.model_info(repo_id=str(model)) |
|
df.loc[index, 'Likes'] = model_info.likes |
|
df.loc[index, 'Tags'] = ', '.join(model_info.tags) |
|
|
|
except (RepositoryNotFoundError, RevisionNotFoundError): |
|
df.loc[index, 'Likes'] = -1 |
|
df.loc[index, 'Tags'] = '' |
|
|
|
return df |
|
|
|
|
|
|
|
def create_bar_chart(df, category): |
|
"""Create and display a bar chart for a given category.""" |
|
st.write(f"### {category} Scores") |
|
|
|
|
|
sorted_df = df[['Model', category]].sort_values(by=category, ascending=True) |
|
|
|
|
|
fig = go.Figure(go.Bar( |
|
x=sorted_df[category], |
|
y=sorted_df['Model'], |
|
orientation='h', |
|
marker=dict(color=sorted_df[category], colorscale='Inferno') |
|
)) |
|
|
|
|
|
fig.update_layout( |
|
margin=dict(l=20, r=20, t=20, b=20) |
|
) |
|
|
|
|
|
st.plotly_chart(fig, use_container_width=True, height=35) |
|
|
|
|
|
|
|
|
|
|
|
def main(): |
|
st.set_page_config(page_title="YALL - Yet Another LLM Leaderboard", layout="wide") |
|
|
|
st.title("π YALL - Yet Another LLM Leaderboard") |
|
st.markdown("Leaderboard made with π§ [LLM AutoEval](https://github.com/mlabonne/llm-autoeval) using [Nous](https://huggingface.co/NousResearch) benchmark suite.") |
|
content = create_yall() |
|
tab1, tab2 = st.tabs(["π Leaderboard", "π About"]) |
|
|
|
|
|
with tab1: |
|
if content: |
|
try: |
|
score_columns = ['Average', 'AGIEval', 'GPT4All', 'TruthfulQA', 'Bigbench'] |
|
|
|
|
|
full_df = convert_markdown_table_to_dataframe(content) |
|
for col in score_columns: |
|
|
|
full_df[col] = pd.to_numeric(full_df[col].str.strip(), errors='coerce') |
|
full_df = get_model_info(full_df) |
|
full_df['Tags'] = full_df['Tags'].fillna('') |
|
df = pd.DataFrame(columns=full_df.columns) |
|
|
|
|
|
col1, col2, col3, col4 = st.columns(4) |
|
with col1: |
|
show_phi = st.checkbox("Phi (2.8B)", value=False) |
|
with col2: |
|
show_mistral = st.checkbox("Mistral (7B)", value=False) |
|
with col3: |
|
show_llama = st.checkbox("Llama 3 (8B)", value=True) |
|
with col4: |
|
show_other = st.checkbox("Other", value=False) |
|
|
|
|
|
dfs_to_concat = [] |
|
|
|
if show_phi: |
|
dfs_to_concat.append(full_df[full_df['Tags'].str.lower().str.contains('phi,|phi-msft,')]) |
|
if show_mistral: |
|
dfs_to_concat.append(full_df[full_df['Tags'].str.lower().str.contains('mistral,')]) |
|
if show_llama: |
|
dfs_to_concat.append(full_df[full_df['Tags'].str.lower().str.contains('llama,')]) |
|
if show_other: |
|
other_df = full_df[~full_df['Tags'].str.lower().str.contains('phi,|phi-msft,|mistral,')] |
|
dfs_to_concat.append(other_df) |
|
|
|
|
|
if dfs_to_concat: |
|
df = pd.concat(dfs_to_concat, ignore_index=True) |
|
|
|
|
|
df = df.sort_values(by='Average', ascending=False) |
|
|
|
|
|
search_query = st.text_input("Search models", "") |
|
|
|
|
|
if search_query: |
|
df = df[df['Model'].str.contains(search_query, case=False)] |
|
|
|
|
|
st.dataframe( |
|
df[['Model'] + score_columns + ['Likes', 'URL']], |
|
use_container_width=True, |
|
column_config={ |
|
"Likes": st.column_config.NumberColumn( |
|
"Likes", |
|
help="Number of likes on Hugging Face", |
|
format="%d β€οΈ", |
|
), |
|
"URL": st.column_config.LinkColumn("URL"), |
|
}, |
|
hide_index=True, |
|
height=int(len(df) * 36.2), |
|
) |
|
|
|
|
|
selected_models = st.multiselect('Select models to compare', df['Model'].unique()) |
|
comparison_df = df[df['Model'].isin(selected_models)] |
|
st.dataframe( |
|
comparison_df, |
|
use_container_width=True, |
|
column_config={ |
|
"Likes": st.column_config.NumberColumn( |
|
"Likes", |
|
help="Number of likes on Hugging Face", |
|
format="%d β€οΈ", |
|
), |
|
"URL": st.column_config.LinkColumn("URL"), |
|
}, |
|
hide_index=True, |
|
) |
|
|
|
|
|
if st.button("Export to CSV"): |
|
|
|
csv_data = df.to_csv(index=False) |
|
|
|
|
|
st.download_button( |
|
label="Download CSV", |
|
data=csv_data, |
|
file_name="leaderboard.csv", |
|
key="download-csv", |
|
help="Click to download the CSV file", |
|
) |
|
|
|
|
|
create_bar_chart(df, score_columns[0]) |
|
|
|
|
|
col1, col2 = st.columns(2) |
|
with col1: |
|
create_bar_chart(df, score_columns[1]) |
|
with col2: |
|
create_bar_chart(df, score_columns[2]) |
|
|
|
|
|
col3, col4 = st.columns(2) |
|
with col3: |
|
create_bar_chart(df, score_columns[3]) |
|
with col4: |
|
create_bar_chart(df, score_columns[4]) |
|
|
|
|
|
except Exception as e: |
|
st.error("An error occurred while processing the markdown table.") |
|
st.error(str(e)) |
|
else: |
|
st.error("Failed to download the content from the URL provided.") |
|
|
|
|
|
with tab2: |
|
st.markdown(''' |
|
### Nous benchmark suite |
|
|
|
Popularized by [Teknium](https://huggingface.co/teknium) and [NousResearch](https://huggingface.co/NousResearch), this benchmark suite aggregates four benchmarks: |
|
|
|
* [**AGIEval**](https://arxiv.org/abs/2304.06364) (0-shot): `agieval_aqua_rat,agieval_logiqa_en,agieval_lsat_ar,agieval_lsat_lr,agieval_lsat_rc,agieval_sat_en,agieval_sat_en_without_passage,agieval_sat_math` |
|
* **GPT4ALL** (0-shot): `hellaswag,openbookqa,winogrande,arc_easy,arc_challenge,boolq,piqa` |
|
* [**TruthfulQA**](https://arxiv.org/abs/2109.07958) (0-shot): `truthfulqa_mc` |
|
* [**Bigbench**](https://arxiv.org/abs/2206.04615) (0-shot): `bigbench_causal_judgement,bigbench_date_understanding,bigbench_disambiguation_qa,bigbench_geometric_shapes,bigbench_logical_deduction_five_objects,bigbench_logical_deduction_seven_objects,bigbench_logical_deduction_three_objects,bigbench_movie_recommendation,bigbench_navigate,bigbench_reasoning_about_colored_objects,bigbench_ruin_names,bigbench_salient_translation_error_detection,bigbench_snarks,bigbench_sports_understanding,bigbench_temporal_sequences,bigbench_tracking_shuffled_objects_five_objects,bigbench_tracking_shuffled_objects_seven_objects,bigbench_tracking_shuffled_objects_three_objects` |
|
|
|
### Reproducibility |
|
|
|
You can easily reproduce these results using π§ [LLM AutoEval](https://github.com/mlabonne/llm-autoeval/tree/master), a colab notebook that automates the evaluation process (benchmark: `nous`). This will upload the results to GitHub as gists. You can find the entire table with the links to the detailed results [here](https://gist.github.com/mlabonne/90294929a2dbcb8877f9696f28105fdf). |
|
|
|
### Clone this space |
|
|
|
You can create your own leaderboard with your LLM AutoEval results on GitHub Gist. You just need to clone this space and specify two variables: |
|
|
|
* Change the `gist_id` in [yall.py](https://huggingface.co/spaces/mlabonne/Yet_Another_LLM_Leaderboard/blob/main/yall.py#L126). |
|
* Create "New Secret" in Settings > Variables and secrets (name: "github", value: [your GitHub token](https://github.com/settings/tokens)) |
|
|
|
A special thanks to [gblazex](https://huggingface.co/gblazex) for providing many evaluations and [CultriX](https://huggingface.co/CultriX) for the CSV export and search bar. |
|
''') |
|
|
|
if __name__ == "__main__": |
|
main() |
|
|