Upload folder using huggingface_hub

README.md

@@ -0,0 +1,12 @@
+---
+title: Giskard Evaluator
+emoji: 🐢🔍
+colorFrom: blue
+colorTo: indigo
+sdk: gradio
+sdk_version: 4.7.1
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,27 @@
+import atexit
+
+import gradio as gr
+
+from app_debug import get_demo as get_demo_debug
+from app_leaderboard import get_demo as get_demo_leaderboard
+from app_text_classification import get_demo as get_demo_text_classification
+from run_jobs import start_process_run_job, stop_thread
+
+try:
+    with gr.Blocks(theme=gr.themes.Soft(primary_hue="green")) as demo:
+        with gr.Tab("Text Classification"):
+            get_demo_text_classification()
+        with gr.Tab("Leaderboard") as leaderboard_tab:
+            get_demo_leaderboard(leaderboard_tab)
+        with gr.Tab("Logs(Debug)"):
+            get_demo_debug()
+
+    start_process_run_job()
+
+    demo.queue(max_size=1000)
+    demo.launch(share=False)
+    atexit.register(stop_thread)
+
+except Exception as e:
+    print("stop background thread: ", e)
+    stop_thread()

app_debug.py

@@ -0,0 +1,85 @@
+from os import listdir
+from os.path import isfile, join
+import html
+
+import gradio as gr
+
+import pipe
+from io_utils import get_logs_file
+
+LOG_PATH = "./tmp"
+CONFIG_PATH = "./cicd/configs/"
+MAX_FILES_NUM = 20
+
+
+def get_accordions_of_files(path, files):
+    components = [None for _ in range(0, MAX_FILES_NUM)]
+    for i in range(0, len(files)):
+        if i >= MAX_FILES_NUM:
+            break
+        with open(join(path, files[i]), "r") as f:
+            components[i] = f.read()
+    return components
+
+
+def get_accordions_of_log_files():
+    log_files = [
+        f for f in listdir(LOG_PATH) if isfile(join(LOG_PATH, f)) and f.endswith("_log")
+    ]
+    return get_accordions_of_files(LOG_PATH, log_files)
+
+
+def get_accordions_of_config_files():
+    config_files = [
+        f
+        for f in listdir(CONFIG_PATH)
+        if isfile(join(CONFIG_PATH, f)) and f.endswith(".yaml")
+    ]
+    return get_accordions_of_files(CONFIG_PATH, config_files)
+
+
+def get_config_files():
+    config_files = [
+        join(CONFIG_PATH, f)
+        for f in listdir(CONFIG_PATH)
+        if isfile(join(CONFIG_PATH, f)) and f.endswith(".yaml")
+    ]
+    return config_files
+
+
+def get_log_files():
+    return [
+        join(LOG_PATH, f)
+        for f in listdir(LOG_PATH)
+        if isfile(join(LOG_PATH, f)) and f.endswith("log")
+    ]
+
+
+def get_jobs_info_in_queue():
+    return [f"⌛️job id {html.escape(job[0])}: {html.escape(job[2])}<br/>" for job in pipe.jobs]
+
+def get_queue_status():
+    if len(pipe.jobs) > 0 or pipe.current is not None:
+        current = pipe.current
+        if current is None:
+            current = "None"
+        return f'<div style="padding-top: 5%">Current job: {html.escape(current)} <br/> Jobs in queue: <br/> {"".join(get_jobs_info_in_queue())}</div>'
+    else:
+        return '<div style="padding-top: 5%">No jobs in queue, please submit an evaluation task from another tab.</div>'
+
+
+def get_demo():
+    with gr.Row():
+        gr.HTML(        
+            value=get_queue_status,
+            every=5,
+        )
+    with gr.Accordion(label="Log Files", open=False):
+        with gr.Row():
+            gr.Files(value=get_log_files, label="Log Files", every=10)
+        with gr.Row():
+            gr.Textbox(
+                value=get_logs_file, every=0.5, lines=10, visible=True, label="Current Log File"
+            )
+    with gr.Accordion(label="Config Files", open=False):
+        gr.Files(value=get_config_files, label="Config Files", every=10)

app_env.py

@@ -0,0 +1,9 @@
+
+HF_REPO_ID = "HF_REPO_ID"
+HF_SPACE_ID = "SPACE_ID"
+HF_WRITE_TOKEN = "HF_WRITE_TOKEN"
+HF_GSK_HUB_URL = "GSK_HUB_URL"
+HF_GSK_HUB_PROJECT_KEY = "GSK_HUB_PROJECT_KEY"
+HF_GSK_HUB_KEY = "GSK_API_KEY"
+HF_GSK_HUB_HF_TOKEN = "GSK_HF_TOKEN"
+HF_GSK_HUB_UNLOCK_TOKEN = "GSK_HUB_UNLOCK_TOKEN"

app_leaderboard.py

@@ -0,0 +1,163 @@
+import logging
+
+import datasets
+import gradio as gr
+import pandas as pd
+import datetime
+
+from fetch_utils import (check_dataset_and_get_config,
+                         check_dataset_and_get_split)
+
+import leaderboard
+logger = logging.getLogger(__name__)
+global update_time 
+update_time = datetime.datetime.fromtimestamp(0)
+
+def get_records_from_dataset_repo(dataset_id):
+    dataset_config = check_dataset_and_get_config(dataset_id)
+
+    logger.info(f"Dataset {dataset_id} has configs {dataset_config}")
+    dataset_split = check_dataset_and_get_split(dataset_id, dataset_config[0])
+    logger.info(f"Dataset {dataset_id} has splits {dataset_split}")
+
+    try:
+        ds = datasets.load_dataset(dataset_id, dataset_config[0])[dataset_split[0]]
+        df = ds.to_pandas()
+        return df
+    except Exception as e:
+        logger.warning(
+            f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}"
+        )
+        return pd.DataFrame()
+
+    
+def get_model_ids(ds):
+    logging.info(f"Dataset {ds} column names: {ds['model_id']}")
+    models = ds["model_id"].tolist()
+    # return unique elements in the list model_ids
+    model_ids = list(set(models))
+    model_ids.insert(0, "Any")
+    return model_ids
+
+
+def get_dataset_ids(ds):
+    logging.info(f"Dataset {ds} column names: {ds['dataset_id']}")
+    datasets = ds["dataset_id"].tolist()
+    dataset_ids = list(set(datasets))
+    dataset_ids.insert(0, "Any")
+    return dataset_ids
+
+
+def get_types(ds):
+    # set types for each column
+    types = [str(t) for t in ds.dtypes.to_list()]
+    types = [t.replace("object", "markdown") for t in types]
+    types = [t.replace("float64", "number") for t in types]
+    types = [t.replace("int64", "number") for t in types]
+    return types
+
+
+def get_display_df(df):
+    # style all elements in the model_id column
+    display_df = df.copy()
+    columns = display_df.columns.tolist()
+    if "model_id" in columns:
+        display_df["model_id"] = display_df["model_id"].apply(
+            lambda x: f'<a href="https://huggingface.co/{x}" target="_blank" style="color:blue">🔗{x}</a>'
+        )
+    # style all elements in the dataset_id column
+    if "dataset_id" in columns:
+        display_df["dataset_id"] = display_df["dataset_id"].apply(
+            lambda x: f'<a href="https://huggingface.co/datasets/{x}" target="_blank" style="color:blue">🔗{x}</a>'
+        )
+    # style all elements in the report_link column
+    if "report_link" in columns:
+        display_df["report_link"] = display_df["report_link"].apply(
+            lambda x: f'<a href="{x}" target="_blank" style="color:blue">🔗{x}</a>'
+        )
+    return display_df
+
+def get_demo(leaderboard_tab):
+    global update_time
+    update_time = datetime.datetime.now()
+    logger.info("Loading leaderboard records")
+    leaderboard.records = get_records_from_dataset_repo(leaderboard.LEADERBOARD)
+    records = leaderboard.records
+
+    model_ids = get_model_ids(records)
+    dataset_ids = get_dataset_ids(records)
+
+    column_names = records.columns.tolist()
+    default_columns = ["model_id", "dataset_id", "total_issues", "report_link"]
+    default_df = records[default_columns]  # extract columns selected
+    types = get_types(default_df)
+    display_df = get_display_df(default_df)  # the styled dataframe to display
+
+    with gr.Row():
+        task_select = gr.Dropdown(
+            label="Task",
+            choices=["text_classification", "tabular"],
+            value="text_classification",
+            interactive=True,
+        )
+        model_select = gr.Dropdown(
+            label="Model id", choices=model_ids, value=model_ids[0], interactive=True
+        )
+        dataset_select = gr.Dropdown(
+            label="Dataset id",
+            choices=dataset_ids,
+            value=dataset_ids[0],
+            interactive=True,
+        )
+
+    with gr.Row():
+        columns_select = gr.CheckboxGroup(
+            label="Show columns",
+            choices=column_names,
+            value=default_columns,
+            interactive=True,
+        )
+
+    with gr.Row():
+        leaderboard_df = gr.DataFrame(display_df, datatype=types, interactive=False)
+
+    def update_leaderboard_records(model_id, dataset_id, columns, task):
+        global update_time
+        if datetime.datetime.now() - update_time < datetime.timedelta(minutes=10):
+            return gr.update()
+        update_time = datetime.datetime.now()
+        logger.info("Updating leaderboard records")
+        leaderboard.records = get_records_from_dataset_repo(leaderboard.LEADERBOARD)
+        return filter_table(model_id, dataset_id, columns, task)
+
+    leaderboard_tab.select(
+        fn=update_leaderboard_records, 
+        inputs=[model_select, dataset_select, columns_select, task_select], 
+        outputs=[leaderboard_df])
+
+    @gr.on(
+        triggers=[
+            model_select.change,
+            dataset_select.change,
+            columns_select.change,
+            task_select.change,
+        ],
+        inputs=[model_select, dataset_select, columns_select, task_select],
+        outputs=[leaderboard_df],
+    )
+    def filter_table(model_id, dataset_id, columns, task):
+        logger.info("Filtering leaderboard records")
+        records = leaderboard.records
+        # filter the table based on task
+        df = records[(records["task"] == task)]
+        # filter the table based on the model_id and dataset_id
+        if model_id and model_id != "Any":
+            df = df[(df["model_id"] == model_id)]
+        if dataset_id and dataset_id != "Any":
+            df = df[(df["dataset_id"] == dataset_id)]
+
+        # filter the table based on the columns
+        df = df[columns]
+        types = get_types(df)
+        display_df = get_display_df(df)
+        return gr.update(value=display_df, datatype=types, interactive=False)

app_legacy.py

@@ -0,0 +1,556 @@
+import json
+import logging
+import os
+import subprocess
+import time
+
+import datasets
+import gradio as gr
+import huggingface_hub
+from transformers.pipelines import TextClassificationPipeline
+
+from io_utils import (
+    convert_column_mapping_to_json,
+    read_inference_type,
+    read_scanners,
+    write_inference_type,
+    write_scanners,
+)
+from text_classification import (
+    check_column_mapping_keys_validity,
+    text_classification_fix_column_mapping,
+)
+from wordings import CONFIRM_MAPPING_DETAILS_FAIL_MD, CONFIRM_MAPPING_DETAILS_MD
+
+HF_REPO_ID = "HF_REPO_ID"
+HF_SPACE_ID = "SPACE_ID"
+HF_WRITE_TOKEN = "HF_WRITE_TOKEN"
+
+
+def check_model(model_id):
+    try:
+        task = huggingface_hub.model_info(model_id).pipeline_tag
+    except Exception:
+        return None, None
+
+    try:
+        from transformers import pipeline
+
+        ppl = pipeline(task=task, model=model_id)
+
+        return model_id, ppl
+    except Exception as e:
+        return model_id, e
+
+
+def check_dataset(dataset_id, dataset_config="default", dataset_split="test"):
+    try:
+        configs = datasets.get_dataset_config_names(dataset_id)
+    except Exception:
+        # Dataset may not exist
+        return None, dataset_config, dataset_split
+
+    if dataset_config not in configs:
+        # Need to choose dataset subset (config)
+        return dataset_id, configs, dataset_split
+
+    ds = datasets.load_dataset(dataset_id, dataset_config)
+
+    if isinstance(ds, datasets.DatasetDict):
+        # Need to choose dataset split
+        if dataset_split not in ds.keys():
+            return dataset_id, None, list(ds.keys())
+    elif not isinstance(ds, datasets.Dataset):
+        # Unknown type
+        return dataset_id, None, None
+    return dataset_id, dataset_config, dataset_split
+
+
+def try_validate(
+    m_id, ppl, dataset_id, dataset_config, dataset_split, column_mapping="{}"
+):
+    # Validate model
+    if m_id is None:
+        gr.Warning(
+            "Model is not accessible. Please set your HF_TOKEN if it is a private model."
+        )
+        return (
+            gr.update(interactive=False),  # Submit button
+            gr.update(visible=True),  # Loading row
+            gr.update(visible=False),  # Preview row
+            gr.update(visible=False),  # Model prediction input
+            gr.update(visible=False),  # Model prediction preview
+            gr.update(visible=False),  # Label mapping preview
+            gr.update(visible=False),  # feature mapping preview
+        )
+    if isinstance(ppl, Exception):
+        gr.Warning(f'Failed to load model": {ppl}')
+        return (
+            gr.update(interactive=False),  # Submit button
+            gr.update(visible=True),  # Loading row
+            gr.update(visible=False),  # Preview row
+            gr.update(visible=False),  # Model prediction input
+            gr.update(visible=False),  # Model prediction preview
+            gr.update(visible=False),  # Label mapping preview
+            gr.update(visible=False),  # feature mapping preview
+        )
+
+    # Validate dataset
+    d_id, config, split = check_dataset(
+        dataset_id=dataset_id,
+        dataset_config=dataset_config,
+        dataset_split=dataset_split,
+    )
+
+    dataset_ok = False
+    if d_id is None:
+        gr.Warning(
+            f'Dataset "{dataset_id}" is not accessible. Please set your HF_TOKEN if it is a private dataset.'
+        )
+    elif isinstance(config, list):
+        gr.Warning(
+            f'Dataset "{dataset_id}" does not have "{dataset_config}" config. Please choose a valid config.'
+        )
+        config = gr.update(choices=config, value=config[0])
+    elif isinstance(split, list):
+        gr.Warning(
+            f'Dataset "{dataset_id}" does not have "{dataset_split}" split. Please choose a valid split.'
+        )
+        split = gr.update(choices=split, value=split[0])
+    else:
+        dataset_ok = True
+
+    if not dataset_ok:
+        return (
+            gr.update(interactive=False),  # Submit button
+            gr.update(visible=True),  # Loading row
+            gr.update(visible=False),  # Preview row
+            gr.update(visible=False),  # Model prediction input
+            gr.update(visible=False),  # Model prediction preview
+            gr.update(visible=False),  # Label mapping preview
+            gr.update(visible=False),  # feature mapping preview
+        )
+
+    # TODO: Validate column mapping by running once
+    prediction_result = None
+    id2label_df = None
+    if isinstance(ppl, TextClassificationPipeline):
+        try:
+            column_mapping = json.loads(column_mapping)
+        except Exception:
+            column_mapping = {}
+
+        (
+            column_mapping,
+            prediction_input,
+            prediction_result,
+            id2label_df,
+            feature_df,
+        ) = text_classification_fix_column_mapping(
+            column_mapping, ppl, d_id, config, split
+        )
+
+        column_mapping = json.dumps(column_mapping, indent=2)
+
+    if prediction_result is None and id2label_df is not None:
+        gr.Warning(
+            'The model failed to predict with the first row in the dataset. Please provide feature mappings in "Advance" settings.'
+        )
+        return (
+            gr.update(interactive=False),  # Submit button
+            gr.update(visible=False),  # Loading row
+            gr.update(CONFIRM_MAPPING_DETAILS_MD, visible=True),  # Preview row
+            gr.update(
+                value=f"**Sample Input**: {prediction_input}", visible=True
+            ),  # Model prediction input
+            gr.update(visible=False),  # Model prediction preview
+            gr.update(
+                value=id2label_df, visible=True, interactive=True
+            ),  # Label mapping preview
+            gr.update(
+                value=feature_df, visible=True, interactive=True
+            ),  # feature mapping preview
+        )
+    elif id2label_df is None:
+        gr.Warning(
+            'The prediction result does not conform the labels in the dataset. Please provide label mappings in "Advance" settings.'
+        )
+        return (
+            gr.update(interactive=False),  # Submit button
+            gr.update(visible=False),  # Loading row
+            gr.update(CONFIRM_MAPPING_DETAILS_MD, visible=True),  # Preview row
+            gr.update(
+                value=f"**Sample Input**: {prediction_input}", visible=True
+            ),  # Model prediction input
+            gr.update(
+                value=prediction_result, visible=True
+            ),  # Model prediction preview
+            gr.update(visible=True, interactive=True),  # Label mapping preview
+            gr.update(visible=True, interactive=True),  # feature mapping preview
+        )
+
+    gr.Info(
+        "Model and dataset validations passed. Your can submit the evaluation task."
+    )
+
+    return (
+        gr.update(interactive=True),  # Submit button
+        gr.update(visible=False),  # Loading row
+        gr.update(CONFIRM_MAPPING_DETAILS_MD, visible=True),  # Preview row
+        gr.update(
+            value=f"**Sample Input**: {prediction_input}", visible=True
+        ),  # Model prediction input
+        gr.update(value=prediction_result, visible=True),  # Model prediction preview
+        gr.update(
+            value=id2label_df, visible=True, interactive=True
+        ),  # Label mapping preview
+        gr.update(
+            value=feature_df, visible=True, interactive=True
+        ),  # feature mapping preview
+    )
+
+
+def try_submit(
+    m_id,
+    d_id,
+    config,
+    split,
+    id2label_mapping_dataframe,
+    feature_mapping_dataframe,
+    local,
+):
+    label_mapping = {}
+    for i, label in id2label_mapping_dataframe["Model Prediction Labels"].items():
+        label_mapping.update({str(i): label})
+
+    feature_mapping = {}
+    for i, feature in feature_mapping_dataframe["Dataset Features"].items():
+        feature_mapping.update(
+            {feature_mapping_dataframe["Model Input Features"][i]: feature}
+        )
+
+    # TODO: Set column mapping for some dataset such as `amazon_polarity`
+
+    if local:
+        command = [
+            "giskard_scanner",
+            "--loader",
+            "huggingface",
+            "--model",
+            m_id,
+            "--dataset",
+            d_id,
+            "--dataset_config",
+            config,
+            "--dataset_split",
+            split,
+            "--hf_token",
+            os.environ.get(HF_WRITE_TOKEN),
+            "--discussion_repo",
+            os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID),
+            "--output_format",
+            "markdown",
+            "--output_portal",
+            "huggingface",
+            "--feature_mapping",
+            json.dumps(feature_mapping),
+            "--label_mapping",
+            json.dumps(label_mapping),
+            "--scan_config",
+            "../config.yaml",
+        ]
+
+        eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
+        start = time.time()
+        logging.info(f"Start local evaluation on {eval_str}")
+
+        evaluator = subprocess.Popen(
+            command,
+            stderr=subprocess.STDOUT,
+        )
+        result = evaluator.wait()
+
+        logging.info(
+            f"Finished local evaluation exit code {result} on {eval_str}: {time.time() - start:.2f}s"
+        )
+
+        gr.Info(
+            f"Finished local evaluation exit code {result} on {eval_str}: {time.time() - start:.2f}s"
+        )
+    else:
+        gr.Info("TODO: Submit task to an endpoint")
+
+    return gr.update(interactive=True)  # Submit button
+
+
+def get_demo():
+    # gr.themes.Soft(
+    #     primary_hue="green",
+    # )
+
+    def check_dataset_and_get_config(dataset_id):
+        try:
+            configs = datasets.get_dataset_config_names(dataset_id)
+            return gr.Dropdown(configs, value=configs[0], visible=True)
+        except Exception:
+            # Dataset may not exist
+            pass
+
+    def check_dataset_and_get_split(dataset_config, dataset_id):
+        try:
+            splits = list(datasets.load_dataset(dataset_id, dataset_config).keys())
+            return gr.Dropdown(splits, value=splits[0], visible=True)
+        except Exception as e:
+            # Dataset may not exist
+            gr.Warning(
+                f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}"
+            )
+
+    def clear_column_mapping_tables():
+        return [
+            gr.update(CONFIRM_MAPPING_DETAILS_FAIL_MD, visible=True),
+            gr.update(value=[], visible=False, interactive=True),
+            gr.update(value=[], visible=False, interactive=True),
+        ]
+
+    def gate_validate_btn(
+        model_id,
+        dataset_id,
+        dataset_config,
+        dataset_split,
+        id2label_mapping_dataframe=None,
+        feature_mapping_dataframe=None,
+    ):
+        column_mapping = "{}"
+        _, ppl = check_model(model_id=model_id)
+
+        if id2label_mapping_dataframe is not None:
+            labels = convert_column_mapping_to_json(
+                id2label_mapping_dataframe.value, label="data"
+            )
+            features = convert_column_mapping_to_json(
+                feature_mapping_dataframe.value, label="text"
+            )
+            column_mapping = json.dumps({**labels, **features}, indent=2)
+
+        if check_column_mapping_keys_validity(column_mapping, ppl) is False:
+            gr.Warning("Label mapping table has invalid contents. Please check again.")
+            return (
+                gr.update(interactive=False),
+                gr.update(CONFIRM_MAPPING_DETAILS_FAIL_MD, visible=True),
+                gr.update(),
+                gr.update(),
+                gr.update(),
+                gr.update(),
+                gr.update(),
+            )
+        else:
+            if model_id and dataset_id and dataset_config and dataset_split:
+                return try_validate(
+                    model_id,
+                    ppl,
+                    dataset_id,
+                    dataset_config,
+                    dataset_split,
+                    column_mapping,
+                )
+            else:
+                return (
+                    gr.update(interactive=False),
+                    gr.update(visible=True),
+                    gr.update(visible=False),
+                    gr.update(visible=False),
+                    gr.update(visible=False),
+                    gr.update(visible=False),
+                    gr.update(visible=False),
+                )
+
+    with gr.Row():
+        gr.Markdown(CONFIRM_MAPPING_DETAILS_MD)
+    with gr.Row():
+        run_local = gr.Checkbox(value=True, label="Run in this Space")
+        use_inference = read_inference_type("./config.yaml") == "hf_inference_api"
+        run_inference = gr.Checkbox(value=use_inference, label="Run with Inference API")
+
+    with gr.Row():
+        selected = read_scanners("./config.yaml")
+        scan_config = selected + ["data_leakage"]
+        scanners = gr.CheckboxGroup(
+            choices=scan_config, value=selected, label="Scan Settings", visible=True
+        )
+
+    with gr.Row():
+        model_id_input = gr.Textbox(
+            label="Hugging Face model id",
+            placeholder="cardiffnlp/twitter-roberta-base-sentiment-latest",
+        )
+
+        dataset_id_input = gr.Textbox(
+            label="Hugging Face Dataset id",
+            placeholder="tweet_eval",
+        )
+    with gr.Row():
+        dataset_config_input = gr.Dropdown(label="Dataset Config", visible=False)
+        dataset_split_input = gr.Dropdown(label="Dataset Split", visible=False)
+
+    with gr.Row(visible=True) as loading_row:
+        gr.Markdown(
+            """
+                    <p style="text-align: center;">
+                    🚀🐢Please validate your model and dataset first...
+                    </p>
+                    """
+        )
+
+    with gr.Row(visible=False) as preview_row:
+        gr.Markdown(
+            """
+            <h1 style="text-align: center;">
+            Confirm Pre-processing Details
+            </h1>
+            Base on your model and dataset, we inferred this label mapping and feature mapping. <b>If the mapping is incorrect, please modify it in the table below.</b>
+            """
+        )
+
+    with gr.Row():
+        id2label_mapping_dataframe = gr.DataFrame(
+            label="Preview of label mapping", interactive=True, visible=False
+        )
+        feature_mapping_dataframe = gr.DataFrame(
+            label="Preview of feature mapping", interactive=True, visible=False
+        )
+    with gr.Row():
+        example_input = gr.Markdown("Sample Input: ", visible=False)
+
+    with gr.Row():
+        example_labels = gr.Label(label="Model Prediction Sample", visible=False)
+
+    run_btn = gr.Button(
+        "Get Evaluation Result",
+        variant="primary",
+        interactive=False,
+        size="lg",
+    )
+
+    model_id_input.blur(
+        clear_column_mapping_tables,
+        outputs=[id2label_mapping_dataframe, feature_mapping_dataframe],
+    )
+
+    dataset_id_input.blur(
+        check_dataset_and_get_config, dataset_id_input, dataset_config_input
+    )
+    dataset_id_input.submit(
+        check_dataset_and_get_config, dataset_id_input, dataset_config_input
+    )
+
+    dataset_config_input.change(
+        check_dataset_and_get_split,
+        inputs=[dataset_config_input, dataset_id_input],
+        outputs=[dataset_split_input],
+    )
+
+    dataset_id_input.blur(
+        clear_column_mapping_tables,
+        outputs=[id2label_mapping_dataframe, feature_mapping_dataframe],
+    )
+    # model_id_input.blur(gate_validate_btn,
+    #                         inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
+    #                         outputs=[run_btn, loading_row, preview_row, example_input, example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+    # dataset_id_input.blur(gate_validate_btn,
+    #                         inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input],
+    # outputs=[run_btn, loading_row, preview_row, example_input,  example_labels, id2label_mapping_dataframe, feature_mapping_dataframe])
+    dataset_config_input.change(
+        gate_validate_btn,
+        inputs=[
+            model_id_input,
+            dataset_id_input,
+            dataset_config_input,
+            dataset_split_input,
+        ],
+        outputs=[
+            run_btn,
+            loading_row,
+            preview_row,
+            example_input,
+            example_labels,
+            id2label_mapping_dataframe,
+            feature_mapping_dataframe,
+        ],
+    )
+    dataset_split_input.change(
+        gate_validate_btn,
+        inputs=[
+            model_id_input,
+            dataset_id_input,
+            dataset_config_input,
+            dataset_split_input,
+        ],
+        outputs=[
+            run_btn,
+            loading_row,
+            preview_row,
+            example_input,
+            example_labels,
+            id2label_mapping_dataframe,
+            feature_mapping_dataframe,
+        ],
+    )
+    id2label_mapping_dataframe.input(
+        gate_validate_btn,
+        inputs=[
+            model_id_input,
+            dataset_id_input,
+            dataset_config_input,
+            dataset_split_input,
+            id2label_mapping_dataframe,
+            feature_mapping_dataframe,
+        ],
+        outputs=[
+            run_btn,
+            loading_row,
+            preview_row,
+            example_input,
+            example_labels,
+            id2label_mapping_dataframe,
+            feature_mapping_dataframe,
+        ],
+    )
+    feature_mapping_dataframe.input(
+        gate_validate_btn,
+        inputs=[
+            model_id_input,
+            dataset_id_input,
+            dataset_config_input,
+            dataset_split_input,
+            id2label_mapping_dataframe,
+            feature_mapping_dataframe,
+        ],
+        outputs=[
+            run_btn,
+            loading_row,
+            preview_row,
+            example_input,
+            example_labels,
+            id2label_mapping_dataframe,
+            feature_mapping_dataframe,
+        ],
+    )
+    scanners.change(write_scanners, inputs=scanners)
+    run_inference.change(write_inference_type, inputs=[run_inference])
+
+    run_btn.click(
+        try_submit,
+        inputs=[
+            model_id_input,
+            dataset_id_input,
+            dataset_config_input,
+            dataset_split_input,
+            id2label_mapping_dataframe,
+            feature_mapping_dataframe,
+            run_local,
+        ],
+        outputs=[
+            run_btn,
+        ],
+    )

app_text_classification.py

@@ -0,0 +1,258 @@
+import uuid
+
+import gradio as gr
+
+from io_utils import get_logs_file, read_scanners, write_scanners
+from text_classification_ui_helpers import (
+    get_related_datasets_from_leaderboard,
+    align_columns_and_show_prediction,
+    check_dataset,
+    precheck_model_ds_enable_example_btn,
+    try_submit,
+    write_column_mapping_to_config,
+)
+from wordings import CONFIRM_MAPPING_DETAILS_MD, INTRODUCTION_MD, USE_INFERENCE_API_TIP
+
+MAX_LABELS = 40
+MAX_FEATURES = 20
+
+EXAMPLE_MODEL_ID = "cardiffnlp/twitter-roberta-base-sentiment-latest"
+CONFIG_PATH = "./config.yaml"
+
+
+def get_demo():
+    with gr.Row():
+        gr.Markdown(INTRODUCTION_MD)
+        uid_label = gr.Textbox(
+            label="Evaluation ID:", value=uuid.uuid4, visible=False, interactive=False
+        )
+    with gr.Row():
+        model_id_input = gr.Textbox(
+            label="Hugging Face model id",
+            placeholder=EXAMPLE_MODEL_ID + " (press enter to confirm)",
+        )
+
+        with gr.Column():
+            dataset_id_input = gr.Dropdown(
+                choices=[],
+                value="",
+                allow_custom_value=True,
+                label="Hugging Face Dataset id",
+            )
+
+    with gr.Row():
+        dataset_config_input = gr.Dropdown(label="Dataset Config", visible=False, allow_custom_value=True)
+        dataset_split_input = gr.Dropdown(label="Dataset Split", visible=False, allow_custom_value=True)
+
+    with gr.Row():
+        first_line_ds = gr.DataFrame(label="Dataset preview", visible=False)
+    with gr.Row():
+        loading_status = gr.HTML(visible=True)
+    with gr.Row():
+        example_btn = gr.Button(
+            "Validate model & dataset",
+            visible=True,
+            variant="primary",
+            interactive=False,
+        )
+
+    with gr.Row():
+        example_input = gr.HTML(visible=False)
+    with gr.Row():
+        example_prediction = gr.Label(label="Model Prediction Sample", visible=False)
+
+    with gr.Row():
+        with gr.Accordion(
+            label="Label and Feature Mapping", visible=False, open=False
+        ) as column_mapping_accordion:
+            with gr.Row():
+                gr.Markdown(CONFIRM_MAPPING_DETAILS_MD)
+            column_mappings = []
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("# Label Mapping")
+                    for _ in range(MAX_LABELS):
+                        column_mappings.append(gr.Dropdown(visible=False))
+                with gr.Column():
+                    gr.Markdown("# Feature Mapping")
+                    for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
+                        column_mappings.append(gr.Dropdown(visible=False))
+
+    with gr.Accordion(label="Model Wrap Advance Config", open=True):
+        gr.HTML(USE_INFERENCE_API_TIP)
+
+        run_inference = gr.Checkbox(value=True, label="Run with Inference API")
+        inference_token = gr.Textbox(
+            placeholder="hf-xxxxxxxxxxxxxxxxxxxx",
+            value="",
+            label="HF Token for Inference API",
+            visible=True,
+            interactive=True,
+        )
+
+    with gr.Accordion(label="Scanner Advance Config (optional)", open=False):
+        scanners = gr.CheckboxGroup(label="Scan Settings", visible=True)
+
+        @gr.on(triggers=[uid_label.change], inputs=[uid_label], outputs=[scanners])
+        def get_scanners(uid):
+            selected = read_scanners(uid)
+            # currently we remove data_leakage from the default scanners
+            # Reason: data_leakage barely raises any issues and takes too many requests
+            # when using inference API, causing rate limit error
+            scan_config = selected + ["data_leakage"]
+            return gr.update(
+                choices=scan_config, value=selected, label="Scan Settings", visible=True
+            )
+
+    with gr.Row():
+        run_btn = gr.Button(
+            "Get Evaluation Result",
+            variant="primary",
+            interactive=False,
+            size="lg",
+        )
+
+    with gr.Row():
+        logs = gr.Textbox(
+            value=get_logs_file,
+            label="Giskard Bot Evaluation Log:",
+            visible=False,
+            every=0.5,
+        )
+
+    
+    scanners.change(write_scanners, inputs=[scanners, uid_label])
+
+    gr.on(
+        triggers=[model_id_input.change],
+        fn=get_related_datasets_from_leaderboard,
+        inputs=[model_id_input],
+        outputs=[dataset_id_input],
+    ).then(
+        fn=check_dataset, 
+        inputs=[dataset_id_input], 
+        outputs=[dataset_config_input, dataset_split_input, loading_status]
+    )
+    
+    gr.on(
+        triggers=[dataset_id_input.input],
+        fn=check_dataset,
+        inputs=[dataset_id_input],
+        outputs=[dataset_config_input, dataset_split_input, loading_status]
+    )
+
+    gr.on(
+        triggers=[label.change for label in column_mappings],
+        fn=write_column_mapping_to_config,
+        inputs=[
+            uid_label,
+            *column_mappings,
+        ],
+    )
+
+    # label.change sometimes does not pass the changed value
+    gr.on(
+        triggers=[label.input for label in column_mappings],
+        fn=write_column_mapping_to_config,
+        inputs=[
+            uid_label,
+            *column_mappings,
+        ],
+    )
+
+    gr.on(
+        triggers=[
+            model_id_input.change,
+            dataset_id_input.change,
+            dataset_config_input.change,
+            dataset_split_input.change,
+        ],
+        fn=precheck_model_ds_enable_example_btn,
+        inputs=[
+            model_id_input,
+            dataset_id_input,
+            dataset_config_input,
+            dataset_split_input,
+        ],
+        outputs=[example_btn, first_line_ds, loading_status],
+    )
+
+    gr.on(
+        triggers=[
+            example_btn.click,
+        ],
+        fn=align_columns_and_show_prediction,
+        inputs=[
+            model_id_input,
+            dataset_id_input,
+            dataset_config_input,
+            dataset_split_input,
+            uid_label,
+            run_inference,
+            inference_token,
+        ],
+        outputs=[
+            example_input,
+            example_prediction,
+            column_mapping_accordion,
+            run_btn,
+            loading_status,
+            *column_mappings,
+        ],
+    )
+
+    gr.on(
+        triggers=[
+            run_btn.click,
+        ],
+        fn=try_submit,
+        inputs=[
+            model_id_input,
+            dataset_id_input,
+            dataset_config_input,
+            dataset_split_input,
+            run_inference,
+            inference_token,
+            uid_label,
+        ],
+        outputs=[run_btn, logs, uid_label],
+    )
+
+    def enable_run_btn(run_inference, inference_token, model_id, dataset_id, dataset_config, dataset_split):
+        if not run_inference or inference_token == "":
+            return gr.update(interactive=False)
+        if model_id == "" or dataset_id == "" or dataset_config == "" or dataset_split == "":
+            return gr.update(interactive=False)
+        return gr.update(interactive=True)
+
+    gr.on(
+        triggers=[
+            run_inference.input,
+            inference_token.input,
+            scanners.input,
+        ],
+        fn=enable_run_btn,
+        inputs=[
+            run_inference, 
+            inference_token, 
+            model_id_input, 
+            dataset_id_input, 
+            dataset_config_input, 
+            dataset_split_input
+        ],
+        outputs=[run_btn],
+    )
+
+    gr.on(
+        triggers=[label.input for label in column_mappings],
+        fn=enable_run_btn,
+        inputs=[
+            run_inference, 
+            inference_token, 
+            model_id_input, 
+            dataset_id_input, 
+            dataset_config_input, 
+            dataset_split_input
+        ],  # FIXME
+        outputs=[run_btn],
+    )

config.yaml

@@ -0,0 +1,13 @@
+configuration:
+  ethical_bias:
+    threshold: 0.05
+  performance:
+    alpha: 0.05
+detectors:
+  - ethical_bias
+  - text_perturbation
+  - robustness
+  - performance
+  - underconfidence
+  - overconfidence
+  - spurious_correlation

fetch_utils.py

@@ -0,0 +1,32 @@
+import logging
+
+import datasets
+
+
+def check_dataset_and_get_config(dataset_id):
+    try:
+        configs = datasets.get_dataset_config_names(dataset_id)
+        return configs
+    except Exception:
+        # Dataset may not exist
+        return None
+
+
+def check_dataset_and_get_split(dataset_id, dataset_config):
+    try:
+        ds = datasets.load_dataset(dataset_id, dataset_config)
+    except Exception as e:
+        # Dataset may not exist
+        logging.warning(
+            f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}"
+        )
+        return None
+    try:
+        splits = list(ds.keys())
+        return splits
+    except Exception as e:
+        # Dataset has no splits
+        logging.warning(
+            f"Dataset {dataset_id} with config {dataset_config} has no splits: {e}"
+        )
+        return None

index.html

@@ -0,0 +1,19 @@
+<!doctype html>
+<html>
+	<head>
+		<meta charset="utf-8" />
+		<meta name="viewport" content="width=device-width" />
+		<title>My static Space</title>
+		<link rel="stylesheet" href="style.css" />
+	</head>
+	<body>
+		<div class="card">
+			<h1>Welcome to your static Space!</h1>
+			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
+			<p>
+				Also don't forget to check the
+				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
+			</p>
+		</div>
+	</body>
+</html>

io_utils.py

@@ -0,0 +1,122 @@
+import os
+
+import yaml
+
+YAML_PATH = "./cicd/configs"
+LOG_FILE = "temp_log"
+
+
+class Dumper(yaml.Dumper):
+    def increase_indent(self, flow=False, *args, **kwargs):
+        return super().increase_indent(flow=flow, indentless=False)
+
+
+def get_yaml_path(uid):
+    if not os.path.exists(YAML_PATH):
+        os.makedirs(YAML_PATH)
+    if not os.path.exists(f"{YAML_PATH}/{uid}_config.yaml"):
+        os.system(f"cp config.yaml {YAML_PATH}/{uid}_config.yaml")
+    return f"{YAML_PATH}/{uid}_config.yaml"
+
+
+# read scanners from yaml file
+# return a list of scanners
+def read_scanners(uid):
+    scanners = []
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        scanners = config.get("detectors", [])
+    return scanners
+
+
+# convert a list of scanners to yaml file
+def write_scanners(scanners, uid):
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        if config:
+            config["detectors"] = scanners
+    # save scanners to detectors in yaml
+    with open(get_yaml_path(uid), "w") as f:
+        yaml.dump(config, f, Dumper=Dumper)
+
+
+# read model_type from yaml file
+def read_inference_type(uid):
+    inference_type = ""
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        inference_type = config.get("inference_type", "")
+    return inference_type
+
+
+# write model_type to yaml file
+def write_inference_type(use_inference, inference_token, uid):
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        if use_inference:
+            config["inference_type"] = "hf_inference_api"
+            config["inference_token"] = inference_token
+        else:
+            config["inference_type"] = "hf_pipeline"
+            # FIXME: A quick and temp fix for missing token
+            config["inference_token"] = ""
+    # save inference_type to inference_type in yaml
+    with open(get_yaml_path(uid), "w") as f:
+        yaml.dump(config, f, Dumper=Dumper)
+
+
+# read column mapping from yaml file
+def read_column_mapping(uid):
+    column_mapping = {}
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        if config:
+            column_mapping = config.get("column_mapping", dict())
+    return column_mapping
+
+
+# write column mapping to yaml file
+def write_column_mapping(mapping, uid):
+    with open(get_yaml_path(uid), "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+
+    if config is None:
+        return
+    if mapping is None and "column_mapping" in config.keys():
+        del config["column_mapping"]
+    else:
+        config["column_mapping"] = mapping
+    with open(get_yaml_path(uid), "w") as f:
+        # yaml Dumper will by default sort the keys
+        yaml.dump(config, f, Dumper=Dumper, sort_keys=False)
+
+
+# convert column mapping dataframe to json
+def convert_column_mapping_to_json(df, label=""):
+    column_mapping = {}
+    column_mapping[label] = []
+    for _, row in df.iterrows():
+        column_mapping[label].append(row.tolist())
+    return column_mapping
+
+
+def get_log_file_with_uid(uid):
+    try:
+        print(f"Loading {uid}.log")
+        with open(f"./tmp/{uid}.log", "a") as file:
+            return file.read()
+    except Exception:
+        return "Log file does not exist"
+
+
+def get_logs_file():
+    try:
+        with open(LOG_FILE, "r") as file:
+            return file.read()
+    except Exception:
+        return "Log file does not exist"
+
+
+def write_log_to_user_file(task_id, log):
+    with open(f"./tmp/{task_id}.log", "a") as f:
+        f.write(log)

isolated_env.py

@@ -0,0 +1,34 @@
+import os
+import subprocess
+
+from io_utils import write_log_to_user_file
+
+
+def prepare_venv(execution_id, deps):
+    python_executable = "python"
+    venv_base = f"tmp/venvs/{execution_id}"
+
+    pip_executable = os.path.join(venv_base, "bin", "pip")
+    # Check pyver
+    write_log_to_user_file(execution_id, "Checking Python version\n")
+    p = subprocess.run([python_executable, "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    write_log_to_user_file(execution_id, p.stdout.decode())
+    if p.returncode != 0:
+        raise RuntimeError(f"{p.args} ended with {p.returncode}")
+    # Create venv
+    write_log_to_user_file(execution_id, "Creating virtual environment\n")
+    p = subprocess.run([python_executable, "-m", "venv", venv_base, "--clear"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    write_log_to_user_file(execution_id, p.stdout.decode())
+    if p.returncode != 0:
+        raise RuntimeError(f"{p.args} ended with {p.returncode}")
+    # Output requirements.txt
+    requirement_file = os.path.join(venv_base, "requirements.txt")
+    with open(requirement_file, "w") as f:
+        f.writelines(deps)
+    # Install deps
+    write_log_to_user_file(execution_id, "Installing dependencies\n")
+    p = subprocess.run([pip_executable, "install", "-r", requirement_file], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    write_log_to_user_file(execution_id, p.stdout.decode())
+    if p.returncode != 0:
+        raise RuntimeError(f"{p.args} ended with {p.returncode}")
+    return os.path.join(venv_base, "bin", "giskard_scanner")

leaderboard.py

@@ -0,0 +1,5 @@
+import pandas as pd
+
+records = pd.DataFrame()
+
+LEADERBOARD = "giskard-bot/evaluator-leaderboard"

mlflow_test.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+from mlflow.utils.environment import _PythonEnv
+from mlflow.utils.virtualenv import (
+    _PYENV_ROOT_DIR,
+    _VIRTUALENV_ENVS_DIR,
+    _create_virtualenv,
+    _get_mlflow_virtualenv_root,
+    _get_virtualenv_extra_env_vars,
+    _get_virtualenv_name,
+    _install_python,
+)
+
+
+_create_virtualenv(
+    "/Users/inoki/giskard-home/projects/credit/models/2a2b6a9c-4050-4bb6-9024-00bf15651262",
+    Path("/opt/homebrew/bin/python3.10"),
+    Path("/Users/inoki/giskard-home/mlflow-venv1"),
+    _PythonEnv()
+)
+

pipe.py

@@ -0,0 +1,3 @@
+
+jobs = list()
+current = None

requirements.txt

@@ -0,0 +1,7 @@
+giskard
+huggingface_hub
+hf-transfer
+torch==2.0.1
+transformers
+datasets
+-e git+https://github.com/Giskard-AI/cicd.git#egg=giskard-cicd

run_jobs.py

@@ -0,0 +1,186 @@
+import json
+import logging
+import os
+import subprocess
+import threading
+import time
+from pathlib import Path
+
+import pipe
+from app_env import (
+    HF_GSK_HUB_HF_TOKEN,
+    HF_GSK_HUB_KEY,
+    HF_GSK_HUB_PROJECT_KEY,
+    HF_GSK_HUB_UNLOCK_TOKEN,
+    HF_GSK_HUB_URL,
+    HF_REPO_ID,
+    HF_SPACE_ID,
+    HF_WRITE_TOKEN,
+)
+from io_utils import LOG_FILE, get_yaml_path, write_log_to_user_file
+from isolated_env import prepare_venv
+from leaderboard import LEADERBOARD
+
+is_running = False
+
+logger = logging.getLogger(__file__)
+
+
+def start_process_run_job():
+    try:
+        logging.debug("Running jobs in thread")
+        global thread, is_running
+        thread = threading.Thread(target=run_job)
+        thread.daemon = True
+        is_running = True
+        thread.start()
+
+    except Exception as e:
+        print("Failed to start thread: ", e)
+
+
+def stop_thread():
+    logging.debug("Stop thread")
+    global is_running
+    is_running = False
+
+
+def prepare_env_and_get_command(
+    m_id,
+    d_id,
+    config,
+    split,
+    inference,
+    inference_token,
+    uid,
+    label_mapping,
+    feature_mapping,
+):
+    leaderboard_dataset = None
+    if os.environ.get("SPACE_ID") == "giskardai/giskard-evaluator":
+        leaderboard_dataset = LEADERBOARD
+
+    inference_type = "hf_pipeline"
+    if inference and inference_token:
+        inference_type = "hf_inference_api"
+
+    executable = "giskard_scanner"
+    try:
+        # Copy the current requirements (might be changed)
+        with open("requirements.txt", "r") as f:
+            executable = prepare_venv(
+                uid,
+                "\n".join(f.readlines()),
+            )
+        logger.info(f"Using {executable} as executable")
+    except Exception as e:
+        logger.warn(f"Create env failed due to {e}, using the current env as fallback.")
+        executable = "giskard_scanner"
+
+    command = [
+        executable,
+        "--loader",
+        "huggingface",
+        "--model",
+        m_id,
+        "--dataset",
+        d_id,
+        "--dataset_config",
+        config,
+        "--dataset_split",
+        split,
+        "--output_format",
+        "markdown",
+        "--output_portal",
+        "huggingface",
+        "--feature_mapping",
+        json.dumps(feature_mapping),
+        "--label_mapping",
+        json.dumps(label_mapping),
+        "--scan_config",
+        get_yaml_path(uid),
+        "--inference_type",
+        inference_type,
+        "--inference_api_token",
+        inference_token,
+    ]
+    # The token to publish post
+    if os.environ.get(HF_WRITE_TOKEN):
+        command.append("--hf_token")
+        command.append(os.environ.get(HF_WRITE_TOKEN))
+
+    # The repo to publish post
+    if os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID):
+        command.append("--discussion_repo")
+        # TODO: Replace by the model id
+        command.append(os.environ.get(HF_REPO_ID) or os.environ.get(HF_SPACE_ID))
+
+    # The repo to publish for ranking
+    if leaderboard_dataset:
+        command.append("--leaderboard_dataset")
+        command.append(leaderboard_dataset)
+
+    # The info to upload to Giskard hub
+    if os.environ.get(HF_GSK_HUB_KEY):
+        command.append("--giskard_hub_api_key")
+        command.append(os.environ.get(HF_GSK_HUB_KEY))
+        if os.environ.get(HF_GSK_HUB_URL):
+            command.append("--giskard_hub_url")
+            command.append(os.environ.get(HF_GSK_HUB_URL))
+        if os.environ.get(HF_GSK_HUB_PROJECT_KEY):
+            command.append("--giskard_hub_project_key")
+            command.append(os.environ.get(HF_GSK_HUB_PROJECT_KEY))
+        if os.environ.get(HF_GSK_HUB_HF_TOKEN):
+            command.append("--giskard_hub_hf_token")
+            command.append(os.environ.get(HF_GSK_HUB_HF_TOKEN))
+        if os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN):
+            command.append("--giskard_hub_unlock_token")
+            command.append(os.environ.get(HF_GSK_HUB_UNLOCK_TOKEN))
+
+    eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
+
+    write_log_to_user_file(
+        uid,
+        f"Start local evaluation on {eval_str}. Please wait for your job to start...\n",
+    )
+
+    return command
+
+
+def save_job_to_pipe(task_id, job, description, lock):
+    with lock:
+        pipe.jobs.append((task_id, job, description))
+
+
+def pop_job_from_pipe():
+    if len(pipe.jobs) == 0:
+        return
+    job_info = pipe.jobs.pop()
+    pipe.current = job_info[2]
+    task_id = job_info[0]
+
+    # Link to LOG_FILE
+    log_file_path = Path(LOG_FILE)
+    if log_file_path.exists():
+        log_file_path.unlink()
+    os.symlink(f"./tmp/{task_id}.log", LOG_FILE)
+
+    write_log_to_user_file(task_id, f"Running job id {task_id}\n")
+    command = prepare_env_and_get_command(*job_info[1])
+
+    with open(f"./tmp/{task_id}.log", "a") as log_file:
+        p = subprocess.Popen(command, stdout=log_file, stderr=subprocess.STDOUT)
+        p.wait()
+    pipe.current = None
+
+
+def run_job():
+    global is_running
+    while is_running:
+        try:
+            pop_job_from_pipe()
+            time.sleep(10)
+        except KeyboardInterrupt:
+            logging.debug("KeyboardInterrupt stop background thread")
+            is_running = False
+            break

scan_config.yaml

@@ -0,0 +1,8 @@
+detectors:
+  - ethical_bias
+  - text_perturbation
+  - robustness
+  - performance
+  - underconfidence
+  - overconfidence
+  - spurious_correlation

style.css

@@ -0,0 +1,28 @@
+body {
+	padding: 2rem;
+	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
+}
+
+h1 {
+	font-size: 16px;
+	margin-top: 0;
+}
+
+p {
+	color: rgb(107, 114, 128);
+	font-size: 15px;
+	margin-bottom: 10px;
+	margin-top: 5px;
+}
+
+.card {
+	max-width: 620px;
+	margin: 0 auto;
+	padding: 16px;
+	border: 1px solid lightgray;
+	border-radius: 16px;
+}
+
+.card p:last-child {
+	margin-bottom: 0;
+}

text_classification.py

@@ -0,0 +1,384 @@
+import json
+import logging
+
+import datasets
+import huggingface_hub
+import pandas as pd
+from transformers import pipeline
+import requests
+import os
+
+logger = logging.getLogger(__name__)
+HF_WRITE_TOKEN = "HF_WRITE_TOKEN"
+
+logger = logging.getLogger(__file__)
+
+class HuggingFaceInferenceAPIResponse:
+    def __init__(self, message):
+        self.message = message
+
+
+def get_labels_and_features_from_dataset(ds):
+    try:
+        dataset_features = ds.features
+        label_keys = [i for i in dataset_features.keys() if i.startswith('label')]
+        if len(label_keys) == 0: # no labels found
+            # return everything for post processing
+            return list(dataset_features.keys()), list(dataset_features.keys())
+        if not isinstance(dataset_features[label_keys[0]], datasets.ClassLabel):
+            if hasattr(dataset_features[label_keys[0]], 'feature'):
+                label_feat = dataset_features[label_keys[0]].feature
+                labels = label_feat.names
+        else:
+            labels = dataset_features[label_keys[0]].names
+        features = [f for f in dataset_features.keys() if not f.startswith("label")]
+        return labels, features
+    except Exception as e:
+        logging.warning(
+            f"Get Labels/Features Failed for dataset: {e}"
+        )
+        return None, None
+
+def check_model_task(model_id):
+    # check if model is valid on huggingface
+    try:
+        task = huggingface_hub.model_info(model_id).pipeline_tag
+        if task is None:
+            return None
+        return task
+    except Exception:
+        return None
+
+def get_model_labels(model_id, example_input):
+    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
+    payload = {"inputs": example_input, "options": {"use_cache": True}}
+    response = hf_inference_api(model_id, hf_token, payload)
+    if "error" in response:
+        return None
+    return extract_from_response(response, "label")
+
+def extract_from_response(data, key):
+    results = []
+
+    if isinstance(data, dict):
+        res = data.get(key)
+        if res is not None:
+            results.append(res)
+
+        for value in data.values():
+            results.extend(extract_from_response(value, key))
+
+    elif isinstance(data, list):
+        for element in data:
+            results.extend(extract_from_response(element, key))
+
+    return results
+
+def hf_inference_api(model_id, hf_token, payload):
+    hf_inference_api_endpoint = os.environ.get(
+        "HF_INFERENCE_ENDPOINT", default="https://api-inference.huggingface.co"
+    )
+    url = f"{hf_inference_api_endpoint}/models/{model_id}"
+    headers = {"Authorization": f"Bearer {hf_token}"}
+    response = requests.post(url, headers=headers, json=payload)
+    if not hasattr(response, "status_code") or response.status_code != 200:
+        logger.warning(f"Request to inference API returns {response}")
+    try:
+        return response.json()
+    except Exception:
+        return {"error": response.content}
+    
+def preload_hf_inference_api(model_id):
+    payload = {"inputs": "This is a test", "options": {"use_cache": True, }}
+    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
+    hf_inference_api(model_id, hf_token, payload)
+
+def check_model_pipeline(model_id):
+    try:
+        task = huggingface_hub.model_info(model_id).pipeline_tag
+    except Exception:
+        return None
+
+    try:
+        ppl = pipeline(task=task, model=model_id)
+
+        return ppl
+    except Exception:
+        return None
+
+
+def text_classificaiton_match_label_case_unsensative(id2label_mapping, label):
+    for model_label in id2label_mapping.keys():
+        if model_label.upper() == label.upper():
+            return model_label, label
+    return None, label
+
+
+def text_classification_map_model_and_dataset_labels(id2label, dataset_features):
+    id2label_mapping = {id2label[k]: None for k in id2label.keys()}
+    dataset_labels = None
+    for feature in dataset_features.values():
+        if not isinstance(feature, datasets.ClassLabel):
+            continue
+        if len(feature.names) != len(id2label_mapping.keys()):
+            continue
+
+        dataset_labels = feature.names
+        # Try to match labels
+        for label in feature.names:
+            if label in id2label_mapping.keys():
+                model_label = label
+            else:
+                # Try to find case unsensative
+                model_label, label = text_classificaiton_match_label_case_unsensative(
+                    id2label_mapping, label
+                )
+            if model_label is not None:
+                id2label_mapping[model_label] = label
+            else:
+                print(f"Label {label} is not found in model labels")
+
+    return id2label_mapping, dataset_labels
+
+
+"""
+params:
+    column_mapping: dict
+    example: {
+        "text": "sentences",
+        "label": {
+            "label0": "LABEL_0",
+            "label1": "LABEL_1"
+        }
+    }
+    ppl: pipeline
+"""
+
+
+def check_column_mapping_keys_validity(column_mapping, ppl):
+    # get the element in all the list elements
+    column_mapping = json.loads(column_mapping)
+    if "data" not in column_mapping.keys():
+        return True
+    user_labels = set([pair[0] for pair in column_mapping["data"]])
+    model_labels = set([pair[1] for pair in column_mapping["data"]])
+
+    id2label = ppl.model.config.id2label
+    original_labels = set(id2label.values())
+
+    return user_labels == model_labels == original_labels
+
+
+"""
+params:
+    column_mapping: dict
+    dataset_features: dict
+    example: {
+        'text': Value(dtype='string', id=None), 
+        'label': ClassLabel(names=['negative', 'neutral', 'positive'], id=None)
+    }
+"""
+
+
+def infer_text_input_column(column_mapping, dataset_features):
+    # Check whether we need to infer the text input column
+    infer_text_input_column = True
+    feature_map_df = None
+
+    if "text" in column_mapping.keys():
+        dataset_text_column = column_mapping["text"]
+        if dataset_text_column in dataset_features.keys():
+            infer_text_input_column = False
+        else:
+            logging.warning(f"Provided {dataset_text_column} is not in Dataset columns")
+
+    if infer_text_input_column:
+        # Try to retrieve one
+        candidates = [
+            f for f in dataset_features if dataset_features[f].dtype == "string"
+        ]
+        feature_map_df = pd.DataFrame(
+            {"Dataset Features": [candidates[0]], "Model Input Features": ["text"]}
+        )
+        if len(candidates) > 0:
+            logging.debug(f"Candidates are {candidates}")
+            column_mapping["text"] = candidates[0]
+
+    return column_mapping, feature_map_df
+
+
+"""
+params:
+    column_mapping: dict
+    id2label_mapping: dict
+    example:
+    id2label_mapping: {
+        'negative': 'negative', 
+        'neutral': 'neutral', 
+        'positive': 'positive'
+        }
+"""
+
+
+def infer_output_label_column(
+    column_mapping, id2label_mapping, id2label, dataset_labels
+):
+    # Check whether we need to infer the output label column
+    if "data" in column_mapping.keys():
+        if isinstance(column_mapping["data"], list):
+            # Use the column mapping passed by user
+            for user_label, model_label in column_mapping["data"]:
+                id2label_mapping[model_label] = user_label
+    elif None in id2label_mapping.values():
+        column_mapping["label"] = {i: None for i in id2label.keys()}
+        return column_mapping, None
+
+    if "data" not in column_mapping.keys():
+        # Column mapping should contain original model labels
+        column_mapping["label"] = {
+            str(i): id2label_mapping[label]
+            for i, label in zip(id2label.keys(), dataset_labels)
+        }
+
+    id2label_df = pd.DataFrame(
+        {
+            "Dataset Labels": dataset_labels,
+            "Model Prediction Labels": [
+                id2label_mapping[label] for label in dataset_labels
+            ],
+        }
+    )
+
+    return column_mapping, id2label_df
+
+
+def check_dataset_features_validity(d_id, config, split):
+    # We assume dataset is ok here
+    ds = datasets.load_dataset(d_id, config)[split]
+    try:
+        dataset_features = ds.features
+    except AttributeError:
+        # Dataset does not have features, need to provide everything
+        return None, None
+        # Load dataset as DataFrame
+    df = ds.to_pandas()
+
+    return df, dataset_features
+
+def select_the_first_string_column(ds):
+    for feature in ds.features.keys():
+        if isinstance(ds[0][feature], str):
+            return feature
+    return None
+
+
+def get_example_prediction(model_id, dataset_id, dataset_config, dataset_split):
+    # get a sample prediction from the model on the dataset
+    prediction_input = None
+    prediction_result = None
+    try:
+        # Use the first item to test prediction
+        ds = datasets.load_dataset(dataset_id, dataset_config)[dataset_split]
+        if "text" not in ds.features.keys():
+            # Dataset does not have text column
+            prediction_input = ds[0][select_the_first_string_column(ds)]
+        else:
+            prediction_input = ds[0]["text"]
+        
+        hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
+        payload = {"inputs": prediction_input, "options": {"use_cache": True}}
+        results = hf_inference_api(model_id, hf_token, payload)
+
+        if isinstance(results, dict) and "error" in results.keys():
+            if "estimated_time" in results.keys():
+                return prediction_input, HuggingFaceInferenceAPIResponse(
+                    f"Estimated time: {int(results['estimated_time'])}s. Please try again later.")
+            return prediction_input, HuggingFaceInferenceAPIResponse(
+                f"Inference Error: {results['error']}.")
+        
+        while isinstance(results, list):
+            if isinstance(results[0], dict):
+                break
+            results = results[0]
+        prediction_result = {
+            f'{result["label"]}': result["score"] for result in results
+        }
+    except Exception as e:
+        # inference api prediction failed, show the error message
+        logger.error(f"Get example prediction failed {e}")
+        return prediction_input, None
+
+    return prediction_input, prediction_result
+
+
+def get_sample_prediction(ppl, df, column_mapping, id2label_mapping):
+    # get a sample prediction from the model on the dataset
+    prediction_input = None
+    prediction_result = None
+    try:
+        # Use the first item to test prediction
+        prediction_input = df.head(1).at[0, column_mapping["text"]]
+        results = ppl({"text": prediction_input}, top_k=None)
+        prediction_result = {
+            f'{result["label"]}': result["score"] for result in results
+        }
+    except Exception:
+        # Pipeline prediction failed, need to provide labels
+        return prediction_input, None
+
+    # Display results in original label and mapped label
+    prediction_result = {
+        f'{result["label"]}(original) - {id2label_mapping[result["label"]]}(mapped)': result[
+            "score"
+        ]
+        for result in results
+    }
+    return prediction_input, prediction_result
+
+
+def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split):
+    # load dataset as pd DataFrame
+    # get features column from dataset
+    df, dataset_features = check_dataset_features_validity(d_id, config, split)
+
+    column_mapping, feature_map_df = infer_text_input_column(
+        column_mapping, dataset_features
+    )
+    if feature_map_df is None:
+        # dataset does not have any features
+        return None, None, None, None, None
+
+    # Retrieve all labels
+    id2label = ppl.model.config.id2label
+
+    # Infer labels
+    id2label_mapping, dataset_labels = text_classification_map_model_and_dataset_labels(
+        id2label, dataset_features
+    )
+    column_mapping, id2label_df = infer_output_label_column(
+        column_mapping, id2label_mapping, id2label, dataset_labels
+    )
+    if id2label_df is None:
+        # does not able to infer output label column
+        return column_mapping, None, None, None, feature_map_df
+
+    # Get a sample prediction
+    prediction_input, prediction_result = get_sample_prediction(
+        ppl, df, column_mapping, id2label_mapping
+    )
+    if prediction_result is None:
+        # does not able to get a sample prediction
+        return column_mapping, prediction_input, None, id2label_df, feature_map_df
+
+    return (
+        column_mapping,
+        prediction_input,
+        prediction_result,
+        id2label_df,
+        feature_map_df,
+    )
+
+def strip_model_id_from_url(model_id):
+    if model_id.startswith("https://huggingface.co/"):
+        return "/".join(model_id.split("/")[-2])
+    return model_id

text_classification_ui_helpers.py

@@ -0,0 +1,351 @@
+import collections
+import logging
+import threading
+import uuid
+
+import datasets
+import gradio as gr
+import pandas as pd
+
+import leaderboard
+from io_utils import read_column_mapping, write_column_mapping
+from run_jobs import save_job_to_pipe
+from text_classification import (
+    strip_model_id_from_url,
+    check_model_task,
+    preload_hf_inference_api,
+    get_example_prediction,
+    get_labels_and_features_from_dataset,
+    HuggingFaceInferenceAPIResponse,
+)
+from wordings import (
+    CHECK_CONFIG_OR_SPLIT_RAW,
+    CONFIRM_MAPPING_DETAILS_FAIL_RAW,
+    MAPPING_STYLED_ERROR_WARNING,
+    NOT_TEXT_CLASSIFICATION_MODEL_RAW,
+    get_styled_input,
+)
+
+MAX_LABELS = 40
+MAX_FEATURES = 20
+
+ds_dict = None
+ds_config = None
+
+def get_related_datasets_from_leaderboard(model_id):
+    records = leaderboard.records
+    model_id = strip_model_id_from_url(model_id)
+    model_records = records[records["model_id"] == model_id]
+    datasets_unique = list(model_records["dataset_id"].unique())
+
+    if len(datasets_unique) == 0:
+        return gr.update(choices=[], value="")
+    
+    return gr.update(choices=datasets_unique, value=datasets_unique[0])
+
+
+logger = logging.getLogger(__file__)
+
+
+def check_dataset(dataset_id):
+    logger.info(f"Loading {dataset_id}")
+    try:
+        configs = datasets.get_dataset_config_names(dataset_id)
+        if len(configs) == 0:
+            return (
+                gr.update(),
+                gr.update(),
+                ""
+            )
+        splits = list(
+                    datasets.load_dataset(
+                        dataset_id, configs[0]
+                    ).keys()
+                )
+        return (
+            gr.update(choices=configs, value=configs[0], visible=True),
+            gr.update(choices=splits, value=splits[0], visible=True),
+            ""
+        )
+    except Exception as e:
+        logger.warn(f"Check your dataset {dataset_id}: {e}")
+        return (
+            gr.update(),
+            gr.update(),
+            ""
+        )
+
+
+
+def write_column_mapping_to_config(uid, *labels):
+    # TODO: Substitute 'text' with more features for zero-shot
+    # we are not using ds features because we only support "text" for now
+    all_mappings = read_column_mapping(uid)
+
+    if labels is None:
+        return
+    all_mappings = export_mappings(all_mappings, "labels", None, labels[:MAX_LABELS])
+    all_mappings = export_mappings(
+        all_mappings,
+        "features",
+        ["text"],
+        labels[MAX_LABELS : (MAX_LABELS + MAX_FEATURES)],
+    )
+
+    write_column_mapping(all_mappings, uid)
+
+
+def export_mappings(all_mappings, key, subkeys, values):
+    if key not in all_mappings.keys():
+        all_mappings[key] = dict()
+    if subkeys is None:
+        subkeys = list(all_mappings[key].keys())
+
+    if not subkeys: 
+        logging.debug(f"subkeys is empty for {key}")
+        return all_mappings
+
+    for i, subkey in enumerate(subkeys):
+        if subkey:
+            all_mappings[key][subkey] = values[i % len(values)]
+    return all_mappings
+
+
+def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels, uid):
+    all_mappings = read_column_mapping(uid)
+    # For flattened raw datasets with no labels
+    # check if there are shared labels between model and dataset
+    shared_labels = set(model_labels).intersection(set(ds_labels))
+    if shared_labels:
+        ds_labels = list(shared_labels)
+    if len(ds_labels) > MAX_LABELS:
+        ds_labels = ds_labels[:MAX_LABELS]
+        gr.Warning(f"The number of labels is truncated to length {MAX_LABELS}")
+
+    # sort labels to make sure the order is consistent
+    # prediction gives the order based on probability
+    ds_labels.sort()
+    model_labels.sort()
+
+    lables = [
+        gr.Dropdown(
+            label=f"{label}",
+            choices=model_labels,
+            value=model_labels[i % len(model_labels)],
+            interactive=True,
+            visible=True,
+        )
+        for i, label in enumerate(ds_labels)
+    ]
+    lables += [gr.Dropdown(visible=False) for _ in range(MAX_LABELS - len(lables))]
+    all_mappings = export_mappings(all_mappings, "labels", ds_labels, model_labels)
+
+    # TODO: Substitute 'text' with more features for zero-shot
+    features = [
+        gr.Dropdown(
+            label=f"{feature}",
+            choices=ds_features,
+            value=ds_features[0],
+            interactive=True,
+            visible=True,
+        )
+        for feature in ["text"]
+    ]
+    features += [
+        gr.Dropdown(visible=False) for _ in range(MAX_FEATURES - len(features))
+    ]
+    all_mappings = export_mappings(all_mappings, "features", ["text"], ds_features)
+    write_column_mapping(all_mappings, uid)
+
+    return lables + features
+
+
+def precheck_model_ds_enable_example_btn(
+    model_id, dataset_id, dataset_config, dataset_split
+):
+    model_id = strip_model_id_from_url(model_id)
+    model_task = check_model_task(model_id)
+    preload_hf_inference_api(model_id)
+    if model_task is None or model_task != "text-classification":
+        gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
+        return (gr.update(), gr.update(),"")
+
+    if dataset_config is None or dataset_split is None or len(dataset_config) == 0:
+        return (gr.update(), gr.update(), "")
+    
+    try:
+        ds = datasets.load_dataset(dataset_id, dataset_config)
+        df: pd.DataFrame = ds[dataset_split].to_pandas().head(5)
+        ds_labels, ds_features = get_labels_and_features_from_dataset(ds[dataset_split])
+
+        if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
+            gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
+            return (gr.update(interactive=False), gr.update(value=df, visible=True), "")
+
+        return (gr.update(interactive=True), gr.update(value=df, visible=True), "")
+    except Exception as e:
+        # Config or split wrong
+        gr.Warning(f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}")
+        return (gr.update(interactive=False), gr.update(value=pd.DataFrame(), visible=False), "")
+
+
+def align_columns_and_show_prediction(
+    model_id,
+    dataset_id,
+    dataset_config,
+    dataset_split,
+    uid,
+    run_inference,
+    inference_token,
+):
+    model_id = strip_model_id_from_url(model_id)
+    model_task = check_model_task(model_id)
+    if model_task is None or model_task != "text-classification":
+        gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
+        return (
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False, open=False),
+            gr.update(interactive=False),
+            "",
+            *[gr.update(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)],
+        )
+
+    dropdown_placement = [
+        gr.Dropdown(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)
+    ]
+
+    prediction_input, prediction_response = get_example_prediction(
+        model_id, dataset_id, dataset_config, dataset_split
+    )
+
+    if prediction_input is None or prediction_response is None:
+        return (
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False, open=False),
+            gr.update(interactive=False),
+            "",
+            *dropdown_placement,
+        )
+
+    if isinstance(prediction_response, HuggingFaceInferenceAPIResponse):
+        return (
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False, open=False),
+            gr.update(interactive=False),
+            f"Hugging Face Inference API is loading your model. {prediction_response.message}",
+            *dropdown_placement,
+        )
+
+    model_labels = list(prediction_response.keys())
+    
+    ds = datasets.load_dataset(dataset_id, dataset_config)[dataset_split]
+    ds_labels, ds_features = get_labels_and_features_from_dataset(ds)
+
+    # when dataset does not have labels or features
+    if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
+        gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
+        return (
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False, open=False),
+            gr.update(interactive=False),
+            "",
+            *dropdown_placement,
+        )
+
+    column_mappings = list_labels_and_features_from_dataset(
+        ds_labels,
+        ds_features,
+        model_labels,
+        uid,
+    )
+
+    # when labels or features are not aligned
+    # show manually column mapping
+    if (
+        collections.Counter(model_labels) != collections.Counter(ds_labels)
+        or ds_features[0] != "text"
+    ):
+        return (
+            gr.update(value=MAPPING_STYLED_ERROR_WARNING, visible=True),
+            gr.update(visible=False),
+            gr.update(visible=True, open=True),
+            gr.update(interactive=(run_inference and inference_token != "")),
+            "",
+            *column_mappings,
+        )
+
+    return (
+        gr.update(value=get_styled_input(prediction_input), visible=True),
+        gr.update(value=prediction_response, visible=True),
+        gr.update(visible=True, open=False),
+        gr.update(interactive=(run_inference and inference_token != "")),
+        "",
+        *column_mappings,
+    )
+
+
+def check_column_mapping_keys_validity(all_mappings):
+    if all_mappings is None:
+        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        return (gr.update(interactive=True), gr.update(visible=False))
+
+    if "labels" not in all_mappings.keys():
+        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        return (gr.update(interactive=True), gr.update(visible=False))
+
+
+def construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features):
+    label_mapping = {}
+    if len(all_mappings["labels"].keys()) != len(ds_labels):
+        gr.Warning("Label mapping corrupted: " + CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+      
+    if len(all_mappings["features"].keys()) != len(ds_features):
+        gr.Warning("Feature mapping corrupted: " + CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+
+    for i, label in zip(range(len(ds_labels)),  ds_labels):
+        # align the saved labels with dataset labels order
+        label_mapping.update({str(i): all_mappings["labels"][label]})
+
+    if "features" not in all_mappings.keys():
+        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+    feature_mapping = all_mappings["features"]
+    return label_mapping, feature_mapping
+
+
+def try_submit(m_id, d_id, config, split, inference, inference_token, uid):
+    all_mappings = read_column_mapping(uid)
+    check_column_mapping_keys_validity(all_mappings)
+
+    # get ds labels and features again for alignment
+    ds = datasets.load_dataset(d_id, config)[split]
+    ds_labels, ds_features = get_labels_and_features_from_dataset(ds)
+    label_mapping, feature_mapping = construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features)
+
+    eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
+    save_job_to_pipe(
+        uid,
+        (
+            m_id,
+            d_id,
+            config,
+            split,
+            inference,
+            inference_token,
+            uid,
+            label_mapping,
+            feature_mapping,
+        ),
+        eval_str,
+        threading.Lock(),
+    )
+    gr.Info("Your evaluation has been submitted")
+
+    return (
+        gr.update(interactive=False),  # Submit button
+        gr.update(lines=5, visible=True, interactive=False),
+        uuid.uuid4(),  # Allocate a new uuid
+    )

utils.py

@@ -0,0 +1,29 @@
+import sys
+
+import yaml
+
+
+# read scanners from yaml file
+# return a list of scanners
+def read_scanners(path):
+    scanners = []
+    with open(path, "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        scanners = config.get("detectors", None)
+    return scanners
+
+
+# convert a list of scanners to yaml file
+def write_scanners(scanners):
+    with open("./scan_config.yaml", "w") as f:
+        # save scanners to detectors in yaml
+        yaml.dump({"detectors": scanners}, f)
+
+
+# convert column mapping dataframe to json
+def convert_column_mapping_to_json(df, label=""):
+    column_mapping = {}
+    column_mapping[label] = []
+    for _, row in df.iterrows():
+        column_mapping[label].append(row.tolist())
+    return column_mapping

validate_queue.py

@@ -0,0 +1,23 @@
+import random
+import time
+
+import gradio as gr
+
+
+def sleep_a_while():
+    seconds = random.randint(5, 10)
+    print(f"Working for {seconds} seconds")
+    start = time.time()
+    while start + seconds > time.time():
+        continue
+    return str(seconds)
+
+
+with gr.Blocks() as iface:
+    text = gr.Textbox(label="Slept second")
+
+    run_btn = gr.Button("Run")
+    run_btn.click(sleep_a_while, queue=False, outputs=text, concurrency_limit=1)
+
+if __name__ == "__main__":
+    iface.queue(max_size=2, default_concurrency_limit=2).launch()

wordings.py

@@ -0,0 +1,67 @@
+INTRODUCTION_MD = """
+                <h1 style="text-align: center;">
+                🐢Giskard Evaluator
+                </h1>
+                Welcome to Giskard Evaluator Space! Get your report immediately by simply input your model id and dataset id below. Follow our leads and improve your model in no time.
+                """
+CONFIRM_MAPPING_DETAILS_MD = """
+                            <h1 style="text-align: center;">
+                            Confirm Pre-processing Details
+                            </h1>
+                            Please confirm the pre-processing details below. Align the column names of your model in the <b>dropdown</b> menu to your dataset's. If you are not sure, please double check your model and dataset.
+                            """
+CONFIRM_MAPPING_DETAILS_FAIL_MD = """
+                            <h1 style="text-align: center;">
+                            Confirm Pre-processing Details
+                            </h1>
+                            Sorry, we cannot align the input/output of your dataset with the model. <b>Pleaser double check your model and dataset.</b>
+                            """
+
+CONFIRM_MAPPING_DETAILS_FAIL_RAW = """
+                            Sorry, we cannot align the input/output of your dataset with the model. Pleaser double check your model and dataset.
+                            """
+
+CHECK_CONFIG_OR_SPLIT_RAW = """
+                            Please check your dataset config or split.
+                            """
+
+PREDICTION_SAMPLE_MD = """
+                        <h1 style="text-align: center;">
+                        Model Prediction Sample
+                        </h1>
+                        Here is a sample prediction from your model based on your dataset.
+                        """
+
+MAPPING_STYLED_ERROR_WARNING = """
+                        <h3 style="text-align: center;color: coral; background-color: #fff0f3; border-radius: 8px; padding: 10px; ">
+                        Sorry, we cannot auto-align the labels/features of your dataset and model. Please double check.
+                        </h3>
+                        """
+
+NOT_TEXT_CLASSIFICATION_MODEL_RAW = """
+                        Your model does not fall under the category of text classification. This page is specifically designated for the evaluation of text classification models.  
+                      """
+
+USE_INFERENCE_API_TIP = """
+                We recommend to use
+                <a href="https://huggingface.co/docs/api-inference/detailed_parameters#text-classification-task">
+                    Hugging Face Inference API
+                </a>
+                for the evaluation,
+                which requires your <a href="https://huggingface.co/settings/tokens">HF token</a>.
+                <br/>
+                Otherwise, an
+                <a href="https://huggingface.co/docs/transformers/main_classes/pipelines#transformers.TextClassificationPipeline">
+                    HF pipeline 
+                </a>
+                will be created and run in this Space. It takes more time to get the result.
+                <br/>
+                <b>
+                Do not worry, your HF token is only used in this Space for your evaluation.
+                </b>
+            """
+
+def get_styled_input(input):
+    return f"""<h3 style="text-align: center;color: #4ca154; background-color: #e2fbe8; border-radius: 8px; padding: 10px; ">
+            Sample input: {input}
+            </h3>"""