davanstrien's picture
davanstrien HF staff
Update app.py
f37a266 verified
raw
history blame contribute delete
No virus
6.05 kB
import os
from datetime import datetime, timedelta, timezone
from typing import Any, Dict
import gradio as gr
import pandas as pd
from cachetools import TTLCache, cached
from dotenv import load_dotenv
from httpx import Client
from huggingface_hub import DatasetCard, hf_hub_url, list_datasets
from tqdm.auto import tqdm
from tqdm.contrib.concurrent import thread_map
load_dotenv()
LIMIT = None
CACHE_TIME = 60 * 60 * 12 # 12 hours
REMOVE_ORGS = {
"HuggingFaceM4",
"HuggingFaceBR4",
"open-llm-leaderboard",
"TrainingDataPro",
}
HF_TOKEN = os.getenv("HF_TOKEN")
USER_AGENT = os.getenv("USER_AGENT")
headers = {"authorization": f"Bearer ${HF_TOKEN}", "user-agent": USER_AGENT}
client = Client(
headers=headers,
timeout=120,
)
# LOCAL = False
# if platform == "darwin":
# LOCAL = True
# cache_dir = "cache" if LOCAL else "/data/diskcache"
# cache = Cache(cache_dir)
cache = TTLCache(maxsize=10, ttl=CACHE_TIME)
def get_three_months_ago():
now = datetime.now(timezone.utc)
return now - timedelta(days=90)
def add_created_data(dataset):
_id = dataset._id
created = dataset.created_at
dataset_dict = dataset.__dict__
dataset_dict["createdAt"] = created
return dataset_dict
def get_readme_len(dataset: Dict[str, Any]):
try:
url = hf_hub_url(dataset["id"], "README.md", repo_type="dataset")
resp = client.get(url)
if resp.status_code == 200:
card = DatasetCard(resp.text)
dataset["len"] = len(card.text)
return dataset
except Exception as e:
print(e)
return None
def check_ds_server_valid(id):
url = f"https://datasets-server.huggingface.co/is-valid?dataset={id}"
response = client.get(url)
if response.status_code != 200:
return False
try:
data = response.json()
preview = data.get("preview")
return preview is not None
except Exception as e:
print(e)
return False
def has_server_preview(dataset):
dataset["server_preview"] = check_ds_server_valid(dataset["id"])
return dataset
def render_model_hub_link(hub_id):
link = f"https://huggingface.co/datasets/{hub_id}"
return (
f'<a target="_blank" href="{link}" style="color: var(--link-text-color);'
f' text-decoration: underline;text-decoration-style: dotted;">{hub_id}</a>'
)
@cached(cache)
def get_datasets():
return list(
tqdm(
iter(
list_datasets(limit=LIMIT, full=True, sort="createdAt", direction=-1)
)
)
)
@cached(cache)
def load_data():
datasets = get_datasets()
datasets = [add_created_data(dataset) for dataset in tqdm(datasets)]
# datasets = [dataset.__dict__ for dataset in tqdm(datasets)]
filtered = [ds for ds in datasets if ds["createdAt"] > get_three_months_ago()]
ds_with_len = thread_map(get_readme_len, filtered)
ds_with_len = [ds for ds in ds_with_len if ds is not None]
ds_with_valid_status = thread_map(has_server_preview, ds_with_len)
ds_with_valid_status = [ds for ds in ds_with_valid_status if ds is not None]
return ds_with_valid_status
columns_to_drop = [
"cardData",
"gated",
"sha",
"tags",
"description",
"siblings",
"disabled",
"_id",
"private",
"author",
# "citation",
"lastModified",
]
def prep_dataframe(remove_orgs_and_users=REMOVE_ORGS, columns_to_drop=columns_to_drop):
ds_with_len = load_data()
if remove_orgs_and_users:
ds_with_len = [
ds for ds in ds_with_len if ds["author"] not in remove_orgs_and_users
]
df = pd.DataFrame(ds_with_len)
df["id"] = df["id"].apply(render_model_hub_link)
if columns_to_drop:
df = df.drop(columns=columns_to_drop)
df = df.sort_values(by=["likes", "downloads", "len"], ascending=False)
return df
def filter_df_by_max_age(df, max_age_days=None):
df = df.dropna(subset=["createdAt"])
now = datetime.now(timezone.utc)
if max_age_days is not None:
max_date = now - timedelta(days=max_age_days)
df = df[df["createdAt"] >= max_date]
return df
def filter_by_readme_len(df, min_len=None):
if min_len is not None:
df = df[df["len"] >= min_len]
return df
def filter_df(max_age_days=None, min_len=None, needs_server_preview: bool = False):
df = prep_dataframe()
if needs_server_preview:
df = df[df["server_preview"] == True]
if max_age_days is not None:
df = filter_df_by_max_age(df, max_age_days=max_age_days)
if min_len is not None:
df = filter_by_readme_len(df, min_len=min_len)
df = df.sort_values(by=["likes", "downloads", "len"], ascending=False)
return df
with gr.Blocks() as demo:
gr.Markdown("# Recent Datasets on the Hub")
gr.Markdown(
"Datasets added in the past 90 days with a README.md and some metadata."
)
with gr.Row():
max_age_days = gr.Slider(
label="Max Age (days)",
value=7,
minimum=0,
maximum=90,
step=1,
interactive=True,
)
min_len = gr.Slider(
label="Minimum README Length",
value=300,
minimum=0,
maximum=1000,
step=50,
interactive=True,
)
needs_server_preview = gr.Checkbox(
label="Exclude datasets without datasets-server preview?",
value=False,
interactive=True,
)
output = gr.DataFrame(filter_df, datatype="markdown", min_width=160 * 2.5, height=1000)
max_age_days.input(
filter_df,
inputs=[max_age_days, min_len, needs_server_preview],
outputs=[output],
)
min_len.input(
filter_df,
inputs=[max_age_days, min_len, needs_server_preview],
outputs=[output],
)
needs_server_preview.change(
filter_df,
inputs=[max_age_days, min_len, needs_server_preview],
outputs=[output],
)
demo.launch()