app.py

import os
from functools import lru_cache
from typing import Dict, List

import plotly.express as px
import streamlit as st

from datasets import Dataset, get_dataset_infos, load_dataset

BASE_DATASET: str = "lion-ai/pl_med_data"
read_key = os.environ.get('HF_TOKEN', None)


dataset_names_map: Dict[str, str] = {
    "znany_lekarz": "Porady - pytania i odpowiedzi",
    "kor_epikryzy_qa": "Dokumentacja medyczna - pytania i odpowiedzi",
    "wikipedia": "Ogólna wiedza medyczna - pytania i opowiedzi",
}

reverse_dataset_names_map: Dict[str, str] = {v: k for k, v in dataset_names_map.items()}


@st.cache_resource
def list_datasets() -> Dict[str, Dataset]:
    """
    Retrieves a list of dataset information.

    Returns:
        List[Dict[str, str]]: A list of dataset information.
    """
    return get_dataset_infos(BASE_DATASET)


def show_examples(dataset_name: str, split: str) -> None:
    dataset_name = reverse_dataset_names_map.get(dataset_name, dataset_name)

    dataset: Dataset = load_dataset(BASE_DATASET, dataset_name, split=f"{split}[:10]", use_auth_token=read_key)
    st.data_editor(dataset.to_pandas(), use_container_width=True)


def count_all_examples(datasets: Dict[str, Dataset]) -> None:
    count: int = 0
    for dataset_name, dataset_info in datasets.items():
        count += dataset_info.num_examples
    st.metric(label="Total no. of instructions", value=f"{count:,}")


def filter_splits(dataset: Dict[str, Dataset], split: str) -> Dict[str, Dataset]:
    """
    Filter the dataset based on the specified split.

    Args:
        dataset (Dict[str, Dataset]): A dictionary containing dataset information.
        split (str): The split to filter the dataset by.

    Returns:
        Dict[str, Dataset]: A dictionary containing the filtered dataset splits.
    """

    dataset_splits: Dict[str, Dataset] = {}
    for dataset_name, dataset_info in dataset.items():
        if split in dataset_info.splits:
            dataset_name = dataset_names_map.get(dataset_name, dataset_name)
            dataset_splits[dataset_name] = dataset_info.splits[split]
    return dataset_splits


split: str = st.selectbox("splits", ["raw", "processed"])

datasets: Dict[str, Dataset] = list_datasets()
# st.write(datasets)

filtered_datasets: Dict[str, Dataset] = filter_splits(datasets, split)
# st.write(filtered_datasets)

count_all_examples(filtered_datasets)

# Create a pie chart showing the number of examples per dataset
fig = px.pie(
    values=[split.num_examples for split in filtered_datasets.values()],
    names=list(filtered_datasets.keys()),
    # title=f"Number of Examples per Dataset ({split} split)",
    labels={"label": "Dataset", "value": "Number of Examples"},
)

# Update layout for better readability
fig.update_traces(textposition="inside", textinfo="value+label")
fig.update_layout(legend_title_text="Datasets", uniformtext_minsize=12, uniformtext_mode="hide")

chart = st.plotly_chart(fig, use_container_width=True)


dataset_name = st.selectbox("Select a dataset", list(filtered_datasets.keys()))

show_examples(dataset_name, split)